From f6bf59a4ed44ec7323a76b11317f715c929a69b4 Mon Sep 17 00:00:00 2001
From: hackademix <giorgio@maone.net>
Date: Sep 01 2018 06:31:51 +0000
Subject: [PATCH 1/2] Moved external licenses check into response pre-processing, in order to avoid filterResponse() as much as possible (fixing issue #11)


---

diff --git a/bg/ResponseProcessor.js b/bg/ResponseProcessor.js
index 7c264a4..e3d6c45 100644
--- a/bg/ResponseProcessor.js
+++ b/bg/ResponseProcessor.js
@@ -33,8 +33,8 @@ class ResponseProcessor {
 
   static install(handler, types = ["main_frame", "sub_frame", "script"]) {
     if (listeners.has(handler)) return false;
-    let listener =
-      request =>  new ResponseTextFilter(request).process(handler);
+    let listener = 
+      async request =>  await new ResponseTextFilter(request).process(handler);
     listeners.set(handler, listener);
     webRequestEvent.addListener(
   		listener,
@@ -70,12 +70,12 @@ class ResponseTextFilter {
       (type === "script" || /\bhtml\b/i.test(md.contentType));
   }
 
-  process(handler) {
+  async process(handler) {
     if (!this.canProcess) return ResponseProcessor.ACCEPT;
     let {metaData, request} = this;
     let response = {request, metaData}; // we keep it around allowing callbacks to store state
     if (typeof handler.pre === "function") {
-      let res = handler.pre(response);
+      let res = await handler.pre(response);
       if (res) return res;
       if (handler.post) handler = handler.post;
       if (typeof handler !== "function") ResponseProcessor.ACCEPT;
@@ -101,7 +101,7 @@ class ResponseTextFilter {
       } catch(e) {
         console.error(e);
       }
-      if (metaData.forcedUTF8 ||
+      if (metaData.forcedUTF8 && request.type !== "script" ||
         editedText !== null && response.text !== editedText) {
         // if we changed the charset, the text or both, let's re-encode
         filter.write(new TextEncoder().encode(editedText));
diff --git a/content/externalLicenseChecker.js b/content/externalLicenseChecker.js
index 89f871f..f150cea 100644
--- a/content/externalLicenseChecker.js
+++ b/content/externalLicenseChecker.js
@@ -79,9 +79,13 @@
   }
 
   browser.runtime.onMessage.addListener(async m => {
-    if (m.action in handlers) {
+    if (m.action in handlers) try {
       debug("Received message", m);
-      return await handlers[m.action](m);
+      let result =  await handlers[m.action](m);
+      console.debug("Returning", result);
+      return result;
+    } catch (e) {
+      console.error(e);
     }
   });
 }
diff --git a/main_background.js b/main_background.js
index 0737926..a521d22 100644
--- a/main_background.js
+++ b/main_background.js
@@ -853,9 +853,9 @@ var ResponseHandler = {
 	/**
 	*	Enforce white/black lists for url/site early (hashes will be handled later)
 	*/
-	pre(response) {
+	async pre(response) {
 		let {request} = response;
-		let {url, type, tabId} = request;
+		let {url, type, tabId, frameId} = request;
 		
 		url = ListStore.urlItem(url);
 		let site = ListStore.siteItem(url);
@@ -878,15 +878,31 @@ var ResponseHandler = {
 			});
 		} else {
 			let whitelistedSite = whitelist.contains(site);
-			if ((response.whitelisted = (whitelistedSite || whitelist.contains(url)))
-					&& type === "script") {
-				// accept the script and stop processing
-				addReportEntry(tabId, url, {url: topUrl, 
-					"whitelisted": [url, whitelistedSite ? `User whitelisted ${site}` : "Whitelisted by user"]});
-				return ResponseProcessor.ACCEPT;
+			let whitelisted = response.whitelisted = whitelistedSite || whitelist.contains(url);
+			if (type === "script") {
+				if (whitelisted) {
+					// accept the script and stop processing
+					addReportEntry(tabId, url, {url: topUrl, 
+						"whitelisted": [url, whitelistedSite ? `User whitelisted ${site}` : "Whitelisted by user"]});
+					return ResponseProcessor.ACCEPT;
+				} else {
+					let scriptInfo = await ExternalLicenses.check({url, tabId, frameId});
+					if (scriptInfo) {
+						let verdict, ret;
+						let msg = scriptInfo.toString();
+						if (scriptInfo.allFree) {
+							verdict = "accepted";
+							ret = ResponseProcessor.ACCEPT;
+						} else {
+							verdict = "blocked";
+							ret = ResponseProcessor.REJECT;
+						}
+						addReportEntry(tabId, url, {url, [verdict]: [url, msg]});
+						return ret;
+					}
+				}
 			}
 		}
-		
 		// it's a page (it's too early to report) or an unknown script:
 		//  let's keep processing
 		return ResponseProcessor.CONTINUE;
@@ -909,21 +925,6 @@ async function handle_script(response, whitelisted){
 	let {text, request} = response;
 	let {url, tabId, frameId} = request;
 	url = ListStore.urlItem(url);
-	if (!whitelisted) {
-		let scriptInfo = await ExternalLicenses.check({url, tabId, frameId});
-		if (scriptInfo) {
-			let verdict;
-			let msg = scriptInfo.toString();
-			if (scriptInfo.allFree) {
-				verdict = "accepted";
-			} else {
-				verdict = "blocked";
-				text = `/* ${msg} */`;
-			}
-			addReportEntry(tabId, url, {url, [verdict]: [url, msg]});
-			return text;
-		}
-	}
   let edited = await get_script(text, url, tabId, whitelisted, -2);
 	return Array.isArray(edited) ? edited[0] : edited;
 }

From 35daafa5e5f7d094698933816e812d666c739982 Mon Sep 17 00:00:00 2001
From: hackademix <giorgio@maone.net>
Date: Sep 01 2018 22:15:48 +0000
Subject: [PATCH 2/2] Fixed weblabels not being checked in whitelisted documents unless they precede scripts.


---

diff --git a/bg/ExternalLicenses.js b/bg/ExternalLicenses.js
index 74692d7..0b2c911 100644
--- a/bg/ExternalLicenses.js
+++ b/bg/ExternalLicenses.js
@@ -35,13 +35,24 @@ let licensesByURL = new Map();
   }
 }
 
+let cachedHrefs = new Map();
+
 var ExternalLicenses = {
+  purgeCache(tabId) {
+    cachedHrefs.delete(tabId);
+  },
+  
   async check(script) {
-    let {url, tabId, frameId} = script;
+    let {url, tabId, frameId, documentUrl} = script;
+    let tabCache = cachedHrefs.get(tabId);
+    let frameCache = tabCache && tabCache.get(frameId);
+    let cache = frameCache && frameCache.get(documentUrl);
     let scriptInfo = await browser.tabs.sendMessage(tabId, {
       action: "checkLicensedScript",
-      url
+      url,
+      cache,
     }, {frameId});
+    
     if (!(scriptInfo && scriptInfo.licenseURLs.length)) {
       return null;
     }
@@ -65,12 +76,24 @@ var ExternalLicenses = {
   
   /**
   * moves / creates external license references before any script in the page
-  * if needed, to have them ready when the first script load is triggered
+  * if needed, to have them ready when the first script load is triggered.
+  * It also caches the external licens href by page URL, to help not actually
+  * modify the rendered HTML but rather feed the content script on demand.
   * Returns true if the document has been actually modified, false otherwise.
   */
-  optimizeDocument(document) {
+  optimizeDocument(document, cachePointer) {
+    let cache = {};
+    let {tabId, frameId, documentUrl} = cachePointer;
+    let frameCache = cachedHrefs.get(tabId);
+    if (!frameCache) {
+      cachedHrefs.set(tabId, frameCache = new Map());
+    }
+    frameCache.set(frameId, new Map([[documentUrl, cache]]));
+    
     let link = document.querySelector(`link[rel="jslicense"], link[data-jslicense="1"], a[rel="jslicense"], a[data-jslicense="1"]`);
     if (link) {
+      let href = link.getAttribute("href");
+      cache.webLabels = {href};
       let move = () => !!document.head.insertBefore(link, document.head.firstChild);
       if (link.parentNode === document.head) {
         for (let node; node = link.previousElementSibling;) {
@@ -82,12 +105,13 @@ var ExternalLicenses = {
         if (link.tagName.toUpperCase() === "A") {
           let newLink = document.createElement("link");
           newLink.rel = "jslicense";
-          newLink.setAttribute("href", link.getAttribute("href"));
+          newLink.setAttribute("href", href);
           link = newLink;
         }
         return move();
       }
     }
+    
     return false;
   }
 };
diff --git a/content/externalLicenseChecker.js b/content/externalLicenseChecker.js
index f150cea..be09ef1 100644
--- a/content/externalLicenseChecker.js
+++ b/content/externalLicenseChecker.js
@@ -22,12 +22,12 @@
 {
   let licensedScripts = null;
     
-  let fetchWebLabels = async (map = new Map()) => {
+  let fetchWebLabels = async args => {
     // see https://www.gnu.org/software/librejs/free-your-javascript.html#step3
-    
+    let {map, cache} = args;
     let link = document.querySelector(`link[rel="jslicense"], link[data-jslicense="1"], a[rel="jslicense"], a[data-jslicense="1"]`);
-    if (link) try {
-      let baseURL = link.href;
+    let baseURL = link ? link.href : cache.webLabels && new URL(cache.webLabels.href, document.baseURI);
+    if (baseURL) try {
       let response = await fetch(baseURL);
       if (!response.ok) throw `${response.status} ${response.statusText}`;
       let doc = new DOMParser().parseFromString(
@@ -55,16 +55,16 @@
     return map;
   }
   
-  let fetchLicenseInfo = async () => {
+  let fetchLicenseInfo = async cache => {
     let map = new Map();
-    
+    let args = {map, cache};
     // in the fetchXxx methods we add to a map whatever license(s)
     // URLs and source code references we can find in various formats 
     // (WebLabels is currently the only implementation), keyed by script URLs.
     await Promise.all([
-    fetchWebLabels(map),
-    // fetchXmlSpdx(),
-    // fetchTxtSpdx(),
+    fetchWebLabels(args),
+    // fetchXmlSpdx(args),
+    // fetchTxtSpdx(args),
     // ...
     ]);
     return map;
@@ -72,8 +72,8 @@
   
   let handlers = {
     async checkLicensedScript(m) {
-      let {url} = m;
-      if (!licensedScripts) licensedScripts = await fetchLicenseInfo();
+      let {url, cache} = m;
+      if (!licensedScripts) licensedScripts = await fetchLicenseInfo(cache);
       return licensedScripts.get(url);
     }
   }
@@ -82,7 +82,6 @@
     if (m.action in handlers) try {
       debug("Received message", m);
       let result =  await handlers[m.action](m);
-      console.debug("Returning", result);
       return result;
     } catch (e) {
       console.error(e);
diff --git a/main_background.js b/main_background.js
index a521d22..e2c3635 100644
--- a/main_background.js
+++ b/main_background.js
@@ -424,6 +424,7 @@ function delete_removed_tab_info(tab_id, remove_info){
 	if(activeMessagePorts[tab_id] !== undefined){
 		delete activeMessagePorts[tab_id];
 	}
+	ExternalLicenses.purgeCache(tab_id);
 }
 
 /**
@@ -855,14 +856,14 @@ var ResponseHandler = {
 	*/
 	async pre(response) {
 		let {request} = response;
-		let {url, type, tabId, frameId} = request;
+		let {url, type, tabId, frameId, documentUrl} = request;
 		
 		url = ListStore.urlItem(url);
 		let site = ListStore.siteItem(url);
 		
 		let blacklistedSite = blacklist.contains(site);
 		let blacklisted = blacklistedSite || blacklist.contains(url);
-		let topUrl = request.frameAncestors && request.frameAncestors.pop() || request.documentUrl;
+		let topUrl = request.frameAncestors && request.frameAncestors.pop() || documentUrl;
 		
 		if (blacklisted) {
 			if (type === "script") {
@@ -886,7 +887,7 @@ var ResponseHandler = {
 						"whitelisted": [url, whitelistedSite ? `User whitelisted ${site}` : "Whitelisted by user"]});
 					return ResponseProcessor.ACCEPT;
 				} else {
-					let scriptInfo = await ExternalLicenses.check({url, tabId, frameId});
+					let scriptInfo = await ExternalLicenses.check({url, tabId, frameId, documentUrl});
 					if (scriptInfo) {
 						let verdict, ret;
 						let msg = scriptInfo.toString();
@@ -991,23 +992,26 @@ function read_metadata(meta_element){
 			return false;
 		}
 }
-
 /**
+
 * 	Reads/changes the HTML of a page and the scripts within it.
 */
-function edit_html(html,url,tabid,wl){
+function edit_html(html, documentUrl, tabId, frameId, whitelisted){
+	
 	
 	return new Promise((resolve, reject) => {
-		if(wl == true){
-			// Don't bother, page is whitelisted
-			resolve(html);	 
-		}
-		
+	
 		var parser = new DOMParser();
 		var html_doc = parser.parseFromString(html, "text/html");
 		
 		// moves external licenses reference, if any, before any <SCRIPT> element
-		ExternalLicenses.optimizeDocument(html_doc); 
+		ExternalLicenses.optimizeDocument(html_doc, {tabId, frameId, documentUrl}); 
+		
+ 		if (whitelisted) { // don't bother rewriting
+			resolve(null);	 
+		}
+		
+		let url = ListStore.urlItem(documentUrl);
 		
 		var amt_scripts = 0;
 		var total_scripts = 0;
@@ -1030,7 +1034,7 @@ function edit_html(html,url,tabid,wl){
 			license = legacy_license_lib.check(first_script_src);
 		if(read_metadata(meta_element) || license != false ){
 			console.log("Valid license for intrinsic events found");
-			addReportEntry(tabid, url, {url, "accepted":[url, `Global license for the page: ${license}`]});
+			addReportEntry(tabId, url, {url, "accepted":[url, `Global license for the page: ${license}`]});
 			// Do not process inline scripts
 			scripts="";
 		}else{
@@ -1072,7 +1076,7 @@ function edit_html(html,url,tabid,wl){
 		for(var i = 0; i < scripts.length; i++){
 			if (scripts[i].src == ""){
 				if (scripts[i].type=="" || scripts[i].type=="text/javascript"){
-					var edit_script = get_script(scripts[i].innerHTML,url,tabid,wl,i);
+					var edit_script = get_script(scripts[i].innerHTML, url, tabId, whitelisted, i);
 					edit_script.then(function(edited){
 						var edited_source = edited[0];
 						var unedited_source = html_doc.scripts[edited[1]].innerHTML.trim();
@@ -1099,12 +1103,12 @@ function edit_html(html,url,tabid,wl){
 */
 async function handle_html(response, whitelisted) {
 	let {text, request} = response;
-	let {url, tabId, type} = request;
+	let {url, tabId, frameId, type} = request;
 	if (type === "main_frame") { 
 		activityReports[tabId] = await createReport({url, tabId});
 		updateBadge(tabId);
 	}
-	return await edit_html(text, ListStore.urlItem(url), tabId, whitelisted);
+	return await edit_html(text, url, tabId, frameId, whitelisted);
 }
 
 var whitelist = new ListStore("pref_whitelist", Storage.CSV);

