added web_response

.gitignore

@@ -0,0 +1,3 @@
+node_modules/
+web-ext-artifacts/
+*.swp

README.md

@@ -103,6 +103,31 @@ Gets the browser's user agent string.
 
 **Returns:** `string` on success, `false` on failure.
 
+## `fplay.web_response_whitelist(sources)`
+
+Outputs the response body for specific data sources. `sources` can is an array that can contain:
+
+- `main_frame`
+- `sub_frame`
+- `stylesheet`
+- `script`
+- `image`
+- `object`
+- `xmlhttprequest`
+- `ping`
+- `font`
+- `media`
+- `websocket`
+- `csp_report`
+- `imageset`
+- `web_manifest`
+- `speculative`
+- `other`
+
+Default is `main_frame`, `xmlhttprequest`.
+
+**Returns:** `boolean` `true`.
+
 ### `fplay.wait_random(min, max)`
 
 Waits for an inclusive amount of time in miliseconds.
@@ -281,7 +306,8 @@ An `EventEmitter` instance that emits the following events:
 | `browser_disconnect` | `{}` | Browser disconnected |
 | `dom_ready` | `{ id, index, status, active, title, url, container }` | A tab finished loading |
 | `dom_load_fail` | `{ id }` | A tab failed to load |
-| `web_request` | `{ id, url, status, origin, type, method, container, headers }` | A main-frame request was sent |
+| `web_request` | `{ id, url, status, origin, type, method, container, headers }` | A request was sent |
+| `web_response` | `{ id, url, status, origin, type, method, container, body }` | A response was received |
 
 ---
 

ext/bg.js

@@ -13,6 +13,10 @@ var container_count = 0;
 
 var proxy_map = {};
 
+var web_response_whitelist = ["main_frame", "xmlhttprequest"];
+
+const log_debug = true;
+
 browser.browserAction.setBadgeBackgroundColor({
 	color: [0, 0, 0, 0]
 });
@@ -80,14 +84,15 @@ function send(ws, seqid, msg = {}){
 	
 	msg.seqid = seqid;
 	var msg = JSON.stringify(msg);
-	console.log("-> " + msg);
+	
+	if(log_debug){ console.log("-> " + msg); }
 	ws.send(msg);
 }
 
 function send_event(ws, msg = {}){
 	
 	var msg = JSON.stringify(msg);
-	console.log("-> " + msg);
+	if(log_debug){ console.log("-> " + msg) };
 	ws.send(msg);
 }
 
@@ -200,7 +205,7 @@ function attach_ws_events(ws){
 	
 	ws.addEventListener("message", async function(e){
 		
-		console.log("<- " + e.data);
+		if(log_debug){ console.log("<- " + e.data); }
 		
 		var msg = JSON.parse(e.data);
 		var seqid = msg.seqid;
@@ -216,6 +221,14 @@ function attach_ws_events(ws){
 				});
 				break;
 			
+			case "web_response_whitelist":
+				web_response_whitelist = msg.list;
+				
+				send(ws, seqid, {
+					"status": true
+				})
+				break;
+			
 			//
 			// Tabs
 			//
@@ -467,9 +480,12 @@ function attach_ws_events(ws){
 //
 // Page events
 //
+// log requests before they're sent
 browser.webRequest.onSendHeaders.addListener(
 	function(details){
 		
+		if(global_ws === null){ return; }
+		
 		var headers = [];
 		
 		for(const header of details.requestHeaders){
@@ -494,10 +510,93 @@ browser.webRequest.onSendHeaders.addListener(
 			}
 		);
 	},
-	{urls: ["<all_urls>"], types: ["main_frame"]},
+	{urls: ["<all_urls>"]/*, types: ["main_frame"]*/},
 	["requestHeaders"]
 );
 
+// forward response body
+async function buff2b64(uint8Array) {
+	return new Promise(function(resolve, reject){
+		const blob = new Blob([uint8Array]);
+		const reader = new FileReader();
+
+		reader.onload = function(){
+			const base64 = reader.result.split(",")[1];
+			resolve(base64);
+		};
+
+		reader.onerror = reject;
+		reader.readAsDataURL(blob);
+	});
+}
+
+browser.webRequest.onBeforeRequest.addListener(
+	async function(details){
+		
+		if(
+			web_response_whitelist.length === 0 ||
+			global_ws === null
+		){ return; }
+		
+		const filter = browser.webRequest.filterResponseData(
+			details.requestId
+		);
+		
+		var chunks = [];
+		
+		filter.ondata = async function(event){
+			
+			chunks.push(event.data);
+			
+			// forward response to browser untouched
+			filter.write(event.data);
+		};
+		
+		filter.onstop = async function(){
+			
+			// we got the full response data
+			var len = 0;
+			for(const c of chunks){
+				
+				len += c.byteLength;
+			}
+			
+			const merged = new Uint8Array(len);
+			
+			let offset = 0;
+			for(const c of chunks){
+				
+				merged.set(new Uint8Array(c), offset);
+				offset += c.byteLength;
+			}
+			
+			var b64 = await buff2b64(merged);
+			
+			send_event(
+				global_ws,
+				{
+					action: "web_response",
+					data: {
+						id: details.tabId,
+						url: details.url,
+						status: details.statusCode,
+						origin: details.originUrl,
+						type: details.type,
+						method: details.method,
+						container: details.cookieStoreId,
+						url: details.url,
+						body: b64
+					}
+				}
+			);
+			
+			filter.close();
+		};
+	},
+	{ urls: ["<all_urls>"], types: web_response_whitelist },
+	["blocking"]
+);
+
 browser.proxy.onRequest.addListener(function(request){
 	
 	const proxy_config = proxy_map[request.cookieStoreId];

ext/manifest.json

@@ -4,7 +4,7 @@
 	"version": "1.0",
 	"description": "4play & dominate",
 	"icons": {
-		"48": "icon.png"
+		"32": "icon.png"
 	},
 	"browser_action": {
 		"default_icon": "icon.png",
@@ -26,5 +26,13 @@
 	],
 	"background": {
 		"scripts": ["bg.js"]
+	},
+	"browser_specific_settings": {
+		"gecko": {
+			"strict_min_version": "102.0",
+			"data_collection_permissions": {
+				"required": ["none"]
+			}
+		}
 	}
 }

server/hello-world.js

@@ -18,6 +18,9 @@ fplay.event.on("browser_connect", async function(ws){
 	const blanktab = await fplay.close_all_tabs(ws);
 	await fplay.delete_all_containers(ws);
 	
+	// only return responses for these data sources
+	fplay.web_response_whitelist(ws, ["main_frame", "xmlhttprequest"]);
+	
 	// create container
 	const container = await fplay.container_create(ws);
 	console.log(container);
@@ -47,4 +50,14 @@ fplay.event.on("browser_connect", async function(ws){
 	console.log(result);
 });
 
+fplay.event.on("web_request", async function(request){
+	
+	console.log(request);
+});
+
+fplay.event.on("web_response", async function(request){
+	
+	
+});
+
 fplay.init(port, password, timeout);

server/lib/fplay.js

@@ -286,6 +286,18 @@ fplay.tab_inject_js = async function(ws, tabid, js, isolated = false){
 	};
 }
 
+fplay.web_response_whitelist = async function(ws, sources = ["main_frame", "xmlhttprequest"]){
+	
+	var whitelist_status = await fplay.send(ws, "web_response_whitelist", {"list": sources});
+	
+	if(typeof whitelist_status.status === "boolean"){
+		
+		return whitelist_status.status;
+	}
+	
+	return false;
+}
+
 
 
 //
@@ -471,6 +483,11 @@ fplay.event.on("dom_ready", function(data){
         promise.resolve(data);
     }
 });
+/*
+fplay.event.on("web_response", function(data){
+	
+	console.log(data);
+});*/
 
 fplay.event.on("dom_load_fail", function(data){
 	
@@ -512,6 +529,12 @@ fplay.wss.on("connection", async function(ws){
 			return;
 		}
 		
+		if(msg.action == "web_response"){
+			
+			// decode base64 as a buffer
+			msg.data.body = Buffer.from(msg.data.body, "base64");
+		}
+		
 		// any other message should be an unsolicited event
 		fplay.event.emit(msg.action, msg.data);
 	});

server/package-lock.json

@@ -0,0 +1,39 @@
+{
+  "name": "server",
+  "lockfileVersion": 3,
+  "requires": true,
+  "packages": {
+    "": {
+      "dependencies": {
+        "http": "^0.0.1-security",
+        "ws": "^8.21.0"
+      }
+    },
+    "node_modules/http": {
+      "version": "0.0.1-security",
+      "resolved": "https://registry.npmjs.org/http/-/http-0.0.1-security.tgz",
+      "integrity": "sha512-RnDvP10Ty9FxqOtPZuxtebw1j4L/WiqNMDtuc1YMH1XQm5TgDRaR1G9u8upL6KD1bXHSp9eSXo/ED+8Q7FAr+g=="
+    },
+    "node_modules/ws": {
+      "version": "8.21.0",
+      "resolved": "https://registry.npmjs.org/ws/-/ws-8.21.0.tgz",
+      "integrity": "sha512-Vsp28b7DRcimFQvrqu2Wek3z1iYxDCWqHYB8Qsnk/S4RfaCQzPGPyBNuVjJV3cd6UiKtUtp6sNM77gWvzcCH+g==",
+      "license": "MIT",
+      "engines": {
+        "node": ">=10.0.0"
+      },
+      "peerDependencies": {
+        "bufferutil": "^4.0.1",
+        "utf-8-validate": ">=5.0.2"
+      },
+      "peerDependenciesMeta": {
+        "bufferutil": {
+          "optional": true
+        },
+        "utf-8-validate": {
+          "optional": true
+        }
+      }
+    }
+  }
+}

server/package.json

@@ -0,0 +1,6 @@
+{
+  "dependencies": {
+    "http": "^0.0.1-security",
+    "ws": "^8.21.0"
+  }
+}