add package.json

.gitignore

@@ -0,0 +1,3 @@
+node_modules/
+web-ext-artifacts/
+*.swp

README.md

@@ -17,17 +17,25 @@ I'm so fucking tired of retards recommending libraries that are so easily detect
 ## Installation
 Install the 4play extension on a **CLEAN** Firefox install. Do **NOT** use your main profile, it will mess up your tabs and containers. Enter credentials in the extension and connect.
 
+Just go get the extension on the [Firefox store](https://addons.mozilla.org/en-US/firefox/addon/4play/), or run it for development like so:
+
+```sh
+npm install --global web-ext
+cd ext
+web-ext run # this will launch a new firefox instance
+```
+
 Then, start the server:
 ```bash
-cd server
+mkdir server && cd server
-npm install http ws
+npm install @lawlers/4play
 node hello-world.js
 ```
 
 ## Example server script
 
 ```js
-const fplay = require("./fplay.js");
+const fplay = require("@lawlers/4play");
 
 var port = 3030;
 var timeout = 30000;
@@ -103,6 +111,14 @@ Gets the browser's user agent string.
 
 **Returns:** `string` on success, `false` on failure.
 
+### `fplay.web_response_whitelist(sources)`
+
+Outputs the raw body of incoming pages for `fplay.event.on("web_response", ...)`. `sources` defines what data sources will be returned: `main_frame`, `sub_frame`, `stylesheet`, `script`, `image`, `object`, `xmlhttprequest`, `ping`, `font`, `media`, `websocket`, `csp_report`, `imageset`, `web_manifest`, `speculative` and `other`
+
+Default is `main_frame`, `xmlhttprequest`.
+
+**Returns:** `boolean` `true`.
+
 ### `fplay.wait_random(min, max)`
 
 Waits for an inclusive amount of time in miliseconds.
@@ -280,8 +296,9 @@ An `EventEmitter` instance that emits the following events:
 | `browser_connect` | `ws` | Browser connected |
 | `browser_disconnect` | `{}` | Browser disconnected |
 | `dom_ready` | `{ id, index, status, active, title, url, container }` | A tab finished loading |
-| `dom_load_fail` | `{ id }` | A tab failed to load |
+| `dom_load_fail` | `{ id, url, error }` | A tab failed to load (Network error, `<=400` code) |
-| `web_request` | `{ id, url, status, origin, type, method, container, headers }` | A main-frame request was sent |
+| `web_request` | `{ id, url, status, origin, type, method, container, headers }` | A request was sent |
+| `web_response` | `{ id, url, status, origin, type, method, container, body }` | A response was received |
 
 ---
 

ext/bg.js

@@ -13,6 +13,12 @@ var container_count = 0;
 
 var proxy_map = {};
 
+var web_response_whitelist = ["main_frame", "xmlhttprequest"];
+
+var webreq_completed = new Map();
+
+const log_debug = true;
+
 browser.browserAction.setBadgeBackgroundColor({
 	color: [0, 0, 0, 0]
 });
@@ -80,14 +86,15 @@ function send(ws, seqid, msg = {}){
 	
 	msg.seqid = seqid;
 	var msg = JSON.stringify(msg);
-	console.log("-> " + msg);
+	
+	if(log_debug){ console.log("-> " + msg); }
 	ws.send(msg);
 }
 
 function send_event(ws, msg = {}){
 	
 	var msg = JSON.stringify(msg);
-	console.log("-> " + msg);
+	if(log_debug){ console.log("-> " + msg) };
 	ws.send(msg);
 }
 
@@ -200,7 +207,7 @@ function attach_ws_events(ws){
 	
 	ws.addEventListener("message", async function(e){
 		
-		console.log("<- " + e.data);
+		if(log_debug){ console.log("<- " + e.data); }
 		
 		var msg = JSON.parse(e.data);
 		var seqid = msg.seqid;
@@ -216,6 +223,14 @@ function attach_ws_events(ws){
 				});
 				break;
 			
+			case "web_response_whitelist":
+				web_response_whitelist = msg.list;
+				
+				send(ws, seqid, {
+					"status": true
+				})
+				break;
+			
 			//
 			// Tabs
 			//
@@ -226,11 +241,25 @@ function attach_ws_events(ws){
 				break;
 			
 			case "tab_open":
-				var tab =
+				
-					await browser.tabs.create({
+				// sometimes opening tabs like about:blank with a
-						url: msg.url,
+				// container will fail.
-						...(typeof msg.container == "string" && { cookieStoreId: msg.container })
+				var tab = null;
+				
+				try{
+					tab =
+						await browser.tabs.create({
+							url: msg.url,
+							...(typeof msg.container == "string" && { cookieStoreId: msg.container })
+						});
+				}catch(err){
+					
+					// send error: we attempted to open tab
+					send(ws, seqid, {
+						data: false
 					});
+					break;
+				}
 				
 				// immediately return even if its not loaded yet
 				send(ws, seqid, {
@@ -309,7 +338,7 @@ function attach_ws_events(ws){
 					send(ws, seqid, {"status": true, "result": result});
 				}catch(err){
 					
-					send(ws, seqid, {"status": err.name + ": " + err.message});
+					send(ws, seqid, {"status": err.name + ": " + err.message, "result": null});
 				}
 				break;
 			
@@ -430,6 +459,22 @@ function attach_ws_events(ws){
 				
 				if(status){
 					
+					if(
+						msg.proxy.type == "http" ||
+						msg.proxy.type == "https"
+					){
+						
+						if(typeof msg.proxy.proxyDNS == "boolean"){
+							
+							delete msg.proxy.proxyDNS;
+						}
+					}
+					
+					if(typeof msg.proxy.port == "string"){
+						
+						msg.proxy.port = parseInt(msg.proxy.port);
+					}
+					
 					proxy_map[msg.id] = msg.proxy;
 					send(ws, seqid, {status: true});
 					return;
@@ -460,6 +505,7 @@ function attach_ws_events(ws){
 	ws.addEventListener("close", async function(e){
 		
 		console.log("ws: offline");
+		webreq_completed = new Map();
 		await set_status(STATUS_OFFLINE);
 	});
 }
@@ -467,9 +513,12 @@ function attach_ws_events(ws){
 //
 // Page events
 //
+// log requests before they're sent
 browser.webRequest.onSendHeaders.addListener(
 	function(details){
 		
+		if(global_ws === null){ return; }
+		
 		var headers = [];
 		
 		for(const header of details.requestHeaders){
@@ -494,10 +543,186 @@ browser.webRequest.onSendHeaders.addListener(
 			}
 		);
 	},
-	{urls: ["<all_urls>"], types: ["main_frame"]},
+	{urls: ["<all_urls>"]/*, types: ["main_frame"]*/},
 	["requestHeaders"]
 );
 
+// forward response body
+async function buff2b64(uint8Array) {
+	return new Promise(function(resolve, reject){
+		const blob = new Blob([uint8Array]);
+		const reader = new FileReader();
+
+		reader.onload = function(){
+			const base64 = reader.result.split(",")[1];
+			resolve(base64);
+		};
+
+		reader.onerror = reject;
+		reader.readAsDataURL(blob);
+	});
+}
+
+// store incoming data
+browser.webRequest.onBeforeRequest.addListener(
+	function(details){
+		
+		if(global_ws === null){ return; }
+		
+		if(!web_response_whitelist.includes(details.type)){
+			
+			return;
+		}
+		
+		if(
+			web_response_whitelist.length === 0 ||
+			global_ws === null
+		){
+			
+			return;
+		}
+		
+		var resolve_func = null;
+		
+		webreq_completed.set(
+			details.requestId,
+			new Promise(
+				function(resolve){
+					
+					resolve_func = resolve;
+				}
+			)
+			
+		);
+		
+		var filter = null;
+		
+		try{
+			
+			filter = browser.webRequest.filterResponseData(details.requestId);
+		}catch(err){
+			
+			resolve_func({success: false});
+			webreq_completed.delete(details.requestId);
+			return;
+		}	
+		
+		var chunks = [];
+		
+		filter.ondata = function(event){
+			
+			chunks.push(event.data);
+			
+			// forward response to browser untouched
+			filter.write(event.data);
+		};
+		
+		filter.onstop = function(){
+			
+			// we got the full response data
+			var len = 0;
+			for(const c of chunks){
+				
+				len += c.byteLength;
+			}
+			
+			const merged = new Uint8Array(len);
+			
+			let offset = 0;
+			for(const c of chunks){
+				
+				merged.set(new Uint8Array(c), offset);
+				offset += c.byteLength;
+			}
+			
+			resolve_func({success: true, data: merged});
+			filter.close();
+		};
+		
+		filter.onerror = function(){
+			
+			resolve_func({success: false});
+			webreq_completed.delete(details.requestId);
+		}
+	},
+	{ urls: ["<all_urls>"] },
+	["blocking"]
+);
+
+// detect page complete and send payloads
+browser.webRequest.onCompleted.addListener(
+	async function(details){
+		
+		if(global_ws === null){ return; }
+		
+		var pending_req = null;
+		
+		pending_req = await webreq_completed.get(details.requestId);
+		
+		if(
+			// not in capture group
+			!pending_req ||
+			// some fuckass error occured
+			pending_req.success === false
+		){
+			
+			return;
+		}
+		
+		var b64 = await buff2b64(pending_req.data);
+		
+		send_event(
+			global_ws,
+			{
+				action: "web_response",
+				data: {
+					id: details.tabId,
+					url: details.url,
+					status: details.statusCode,
+					origin: details.originUrl,
+					type: details.type,
+					method: details.method,
+					container: details.cookieStoreId,
+					body: b64
+				}
+			}
+		);
+		
+		// clean up
+		webreq_completed.delete(details.requestId);
+	},
+	{ urls: ["<all_urls>"] }
+);
+
+// shit out an error when the request fails
+browser.webRequest.onErrorOccurred.addListener(
+	async function(page){
+		
+		if(global_ws === null){ return; }
+		
+		if(!web_response_whitelist.includes(page.type)){
+			
+			webreq_completed.delete(page.requestId);
+			return;
+		}
+		
+		send_event(
+			global_ws,
+			{
+				"action": "dom_load_fail",
+				"data": {
+					id: page.tabId,
+					url: page.url,
+					error: page.error
+				}
+			}
+		);
+		
+		webreq_completed.delete(page.requestId);
+	},
+	{ urls: ["<all_urls>"] }
+);
+
 browser.proxy.onRequest.addListener(function(request){
 	
 	const proxy_config = proxy_map[request.cookieStoreId];
@@ -515,9 +740,47 @@ browser.proxy.onRequest.addListener(function(request){
 	urls: ["<all_urls>"]
 });
 
+// Automatically handle and detect all Proxy Authentication challenges
+browser.webRequest.onAuthRequired.addListener(
+	function(details){
+		
+		// Only intercept proxy auth requests, ignore normal website auth
+		if(!details.isProxy){
+			return {};
+		}
+
+		const proxy_config = proxy_map[details.cookieStoreId];
+		
+		// Check if the assigned proxy configuration for this container includes credentials
+		if(
+			proxy_config &&
+			(
+				proxy_config.type == "http" ||
+				proxy_config.type == "https"
+			) &&
+			proxy_config.username &&
+			proxy_config.password
+		){
+			
+			return {
+				authCredentials: {
+					username: proxy_config.username,
+					password: proxy_config.password
+				}
+			};
+		}
+		
+		// If the proxy requires auth and we have no credentials, silently cancel 
+		// to prevent hanging the browser with auth popup dialogs.
+		return { cancel: true };
+	},
+	{ urls: ["<all_urls>"] },
+	["blocking"]
+);
+
 browser.tabs.onUpdated.addListener(function(tabid, event, tab){
 	
-	if(connected === false){ return; }
+	if(global_ws === null){ return; }
 	
 	if(event.status === "complete"){
 		
@@ -539,20 +802,49 @@ browser.tabs.onUpdated.addListener(function(tabid, event, tab){
 	}
 });
 
-browser.webNavigation.onErrorOccurred.addListener(function(page){
+browser.webRequest.onHeadersReceived.addListener(
+	function(page){
 		
-	if(connected === false){ return; }
+		if(global_ws === null){ return; }
 		
-	send_event(
+		if(!web_response_whitelist.includes(page.type)){
-		global_ws,
+			
-		{
+			return
-			"action": "dom_load_fail",
+		};
-			"data": {
+		
-				id: page.tabId
+		// send error on 4xx code
+		if(page.statusCode >= 400){
+			
+			const proxy_config = proxy_map[page.cookieStoreId];
+			
+			if(
+				proxy_config &&
+				(
+					proxy_config.type == "http" ||
+					proxy_config.type == "https"
+				) &&
+				page.statusCode === 407
+			){
+				
+				// ignore http proxy auth error
+				return;
 			}
+			
+			send_event(
+				global_ws,
+				{
+					"action": "dom_load_fail",
+					"data": {
+						id: page.tabId,
+						url: page.url,
+						error: page.statusLine.trim()
+					}
+				}
+			);
 		}
-	);
+	},
-});
+	{ urls: ["<all_urls>"] }
+);
 
 (async function(){
 	await ws_init();

ext/manifest.json

@@ -1,10 +1,10 @@
 {
 	"manifest_version": 2,
 	"name": "4play",
-	"version": "1.0",
+	"version": "1.8",
 	"description": "4play & dominate",
 	"icons": {
-		"48": "icon.png"
+		"32": "icon.png"
 	},
 	"browser_action": {
 		"default_icon": "icon.png",
@@ -26,5 +26,13 @@
 	],
 	"background": {
 		"scripts": ["bg.js"]
+	},
+	"browser_specific_settings": {
+		"gecko": {
+			"strict_min_version": "102.0",
+			"data_collection_permissions": {
+				"required": ["none"]
+			}
+		}
 	}
 }

server/hello-world.js

@@ -2,7 +2,7 @@ const fplay = require("./lib/fplay.js");
 
 var port = 3030;
 var timeout = 30000;
-var password = "cnc";
+var password = process.env.FOURPLAY_PASSWORD || "cnc";
 
 fplay.event.on("server_ready", function(){
 	
@@ -18,6 +18,9 @@ fplay.event.on("browser_connect", async function(ws){
 	const blanktab = await fplay.close_all_tabs(ws);
 	await fplay.delete_all_containers(ws);
 	
+	// only return responses for these data sources
+	fplay.web_response_whitelist(ws, ["main_frame", "xmlhttprequest"]);
+	
 	// create container
 	const container = await fplay.container_create(ws);
 	console.log(container);
@@ -47,4 +50,14 @@ fplay.event.on("browser_connect", async function(ws){
 	console.log(result);
 });
 
+fplay.event.on("web_request", async function(request){
+	
+	//console.log(request);
+});
+
+fplay.event.on("web_response", async function(response){
+	
+	console.log(response);
+});
+
 fplay.init(port, password, timeout);

server/lib/fplay.js

@@ -1,11 +1,13 @@
 const http = require("http");
 const { WebSocketServer } = require("ws");
 
+const {FOURPLAY_PORT = 3000, FOURPLAY_COMMAND_TIMEOUT = 5000, FOURPLAY_PASSWORD = "cnc"} = process.env;
+
 var fplay = {};
 
-fplay.PORT = 3030;
+fplay.PORT = FOURPLAY_PORT;
-fplay.COMMAND_TIMEOUT = 5000;
+fplay.COMMAND_TIMEOUT = FOURPLAY_COMMAND_TIMEOUT;
-fplay.PASSWORD = "cnc";
+fplay.PASSWORD = FOURPLAY_PASSWORD;
 
 const { EventEmitter } = require("events");
 fplay.event = new EventEmitter();
@@ -25,8 +27,6 @@ fplay.server = http.createServer(function(req, res){
 	}
 });
 
-fplay.wss = new WebSocketServer({ server: fplay.server, path: "/" + fplay.PASSWORD });
-
 fplay.seqid = 0;
 fplay.pending = new Map(); // promise map
 fplay.pending_dom_ready = new Map();
@@ -286,6 +286,18 @@ fplay.tab_inject_js = async function(ws, tabid, js, isolated = false){
 	};
 }
 
+fplay.web_response_whitelist = async function(ws, sources = ["main_frame", "xmlhttprequest"]){
+	
+	var whitelist_status = await fplay.send(ws, "web_response_whitelist", {"list": sources});
+	
+	if(typeof whitelist_status.status === "boolean"){
+		
+		return whitelist_status.status;
+	}
+	
+	return false;
+}
+
 
 
 //
@@ -471,6 +483,11 @@ fplay.event.on("dom_ready", function(data){
         promise.resolve(data);
     }
 });
+/*
+fplay.event.on("web_response", function(data){
+	
+	console.log(data);
+});*/
 
 fplay.event.on("dom_load_fail", function(data){
 	
@@ -492,37 +509,6 @@ fplay.event.on("web_request", function(page){
 
 fplay.browser_connected = false;
 
-fplay.wss.on("connection", async function(ws){
-	
-	fplay.browser_connected = true;
-	fplay.event.emit("browser_connect", ws);
-	
-	// register user events
-	ws.on("message", async function(msg){
-		
-		msg = JSON.parse(msg.toString());
-		
-		//console.log(msg);
-		
-		if(typeof msg.seqid == "number"){
-			
-			// resolve promises
-			fplay.pending.get(msg.seqid).resolve(msg);
-			fplay.pending.delete(msg.seqid);
-			return;
-		}
-		
-		// any other message should be an unsolicited event
-		fplay.event.emit(msg.action, msg.data);
-	});
-
-	ws.on("close", async function(){
-		
-		fplay.browser_connected = false;
-		fplay.event.emit("browser_disconnect", {});
-	});
-});
-
 fplay.init = function(port = 3030, password = "cnc", command_timeout = 5000){
 	
 	fplay.PORT = port;
@@ -531,8 +517,51 @@ fplay.init = function(port = 3030, password = "cnc", command_timeout = 5000){
 	
 	fplay.server.listen(fplay.PORT, function(){
 		
+		fplay.wss = new WebSocketServer({ server: fplay.server, path: "/" + fplay.PASSWORD });
+		
+		fplay.wss.on("connection", async function(ws){
+			
+			fplay.browser_connected = true;
+			fplay.event.emit("browser_connect", ws);
+			
+			// register user events
+			ws.on("message", async function(msg){
+				
+				msg = JSON.parse(msg.toString());
+				
+				//console.log(msg);
+				
+				if(typeof msg.seqid == "number"){
+					
+					// resolve promises
+					fplay.pending.get(msg.seqid).resolve(msg);
+					fplay.pending.delete(msg.seqid);
+					return;
+				}
+				
+				if(msg.action == "web_response"){
+					
+					// decode base64 as a buffer
+					msg.data.body = Buffer.from(msg.data.body, "base64");
+				}
+				
+				// any other message should be an unsolicited event
+				fplay.event.emit(msg.action, msg.data);
+			});
+			
+			ws.on("close", async function(){
+				
+				fplay.browser_connected = false;
+				fplay.event.emit("browser_disconnect", {});
+			});
+		});
+		
 		fplay.event.emit("server_ready", {});
 	});
 }
 
+process.on('SIGINT', function() {
+        process.exit();
+});
+
 module.exports = fplay;

server/package-lock.json

@@ -0,0 +1,37 @@
+{
+  "name": "@lawlers/4play",
+  "version": "1.2.5",
+  "lockfileVersion": 3,
+  "requires": true,
+  "packages": {
+    "": {
+      "name": "@lawlers/4play",
+      "version": "1.2.5",
+      "license": "AGPL-3.0-only",
+      "dependencies": {
+        "ws": "^8.21.1"
+      }
+    },
+    "node_modules/ws": {
+      "version": "8.21.2",
+      "resolved": "https://registry.npmjs.org/ws/-/ws-8.21.2.tgz",
+      "integrity": "sha512-54dMVAo4WIe6SKy3vBgN+9bJZqqQ8IMRevAkOLQALhi49qkkQDQfWdAZ8KQlXiEabw88ARXXdUrlvtbKQX+aKw==",
+      "license": "MIT",
+      "engines": {
+        "node": ">=10.0.0"
+      },
+      "peerDependencies": {
+        "bufferutil": "^4.0.1",
+        "utf-8-validate": ">=5.0.2"
+      },
+      "peerDependenciesMeta": {
+        "bufferutil": {
+          "optional": true
+        },
+        "utf-8-validate": {
+          "optional": true
+        }
+      }
+    }
+  }
+}

server/package.json

@@ -0,0 +1,29 @@
+{
+  "name": "@lawlers/4play",
+  "version": "1.2.5",
+  "description": "4play and dominate",
+  "keywords": [
+    "4play",
+    "fplay",
+    "foreplay",
+    "browser-automation",
+    "puppeteer",
+    "selenium",
+    "playwright",
+    "firefox"
+  ],
+  "repository": {
+    "type": "git",
+    "url": "https://git.lolcat.ca/lolcat/4play"
+  },
+  "license": "AGPL-3.0-only",
+  "author": "lolcat",
+  "type": "commonjs",
+  "main": "fplay.js",
+  "scripts": {
+    "test": "echo \"Error: no test specified\" && exit 1"
+  },
+  "dependencies": {
+    "ws": "^8.21.1"
+  }
+}