From fb94c85df184e7c7f3481804c57cffe426d7585c Mon Sep 17 00:00:00 2001 From: Raymond Hill Date: Tue, 1 Jan 2019 08:34:00 -0500 Subject: [PATCH] Add ability to block early at launch; adapt to changes in Chromium 72+ Related issues: - "Requests bypass uMatrix on Firefox start" Using same approach as with uBO: https://github.com/gorhill/uBloc/commit/41548be6be35 `suspendTabsUntilReady` advanced setting added to "More" pane, useful only for Chromium -- the blocking of early network requests is enforced unconditionally on Firefox (because it supports returning Promises from webRequest handlers). - "Cookies leaking temporarily" Changes in the webRequest API in Chromium 72+ caused uMatrix to fail to process `Cookie` and `Referer` headers on that platform. --- platform/chromium/polyfill.js | 234 --------------------------- platform/chromium/vapi-background.js | 133 ++++++--------- platform/chromium/vapi-common.js | 12 +- platform/chromium/vapi-webrequest.js | 194 ++++++++++++++++++++++ platform/firefox/polyfill.js | 30 ---- platform/firefox/vapi-webrequest.js | 185 +++++++++++++++++++++ src/background.html | 9 +- src/js/background.js | 35 +++- src/js/raw-settings.js | 4 +- src/js/start.js | 69 ++++---- src/js/storage.js | 97 ++++++----- src/js/traffic.js | 78 +++++---- tools/make-firefox.sh | 3 +- 13 files changed, 623 insertions(+), 460 deletions(-) delete mode 100644 platform/chromium/polyfill.js create mode 100644 platform/chromium/vapi-webrequest.js delete mode 100644 platform/firefox/polyfill.js create mode 100644 platform/firefox/vapi-webrequest.js diff --git a/platform/chromium/polyfill.js b/platform/chromium/polyfill.js deleted file mode 100644 index 1ee5a83..0000000 --- a/platform/chromium/polyfill.js +++ /dev/null @@ -1,234 +0,0 @@ -/******************************************************************************* - - uMatrix - a browser extension to block requests. - Copyright (C) 2017 Raymond Hill - - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program. If not, see {http://www.gnu.org/licenses/}. - - Home: https://github.com/gorhill/uMatrix - - This file has been originally imported from: - https://github.com/gorhill/uBlock/tree/master/platform/chromium - -*/ - -// For background page or non-background pages - -/* exported objectAssign */ - -'use strict'; - -/******************************************************************************/ -/******************************************************************************/ - -// https://github.com/gorhill/uBlock/issues/1067 -// https://developer.mozilla.org/en/docs/Web/JavaScript/Reference/Global_Objects/String/startsWith -// Firefox 17/Chromium 41 supports `startsWith`. - -if ( String.prototype.startsWith instanceof Function === false ) { - String.prototype.startsWith = function(needle, pos) { - if ( typeof pos !== 'number' ) { - pos = 0; - } - return this.lastIndexOf(needle, pos) === pos; - }; -} - -/******************************************************************************/ - -// https://github.com/gorhill/uBlock/issues/1067 -// https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/String/endsWith -// Firefox 17/Chromium 41 supports `endsWith`. - -if ( String.prototype.endsWith instanceof Function === false ) { - String.prototype.endsWith = function(needle, pos) { - if ( typeof pos !== 'number' ) { - pos = this.length; - } - pos -= needle.length; - return this.indexOf(needle, pos) === pos; - }; -} - -/******************************************************************************/ - -// As per MDN, Object.assign appeared first in Chromium 45. -// https://developer.mozilla.org/en/docs/Web/JavaScript/Reference/Global_Objects/Object/assign#Browser_compatibility - -var objectAssign = Object.assign || function(target, source) { - var keys = Object.keys(source); - for ( var i = 0, n = keys.length, key; i < n; i++ ) { - key = keys[i]; - target[key] = source[key]; - } - return target; -}; - -/******************************************************************************/ - -// https://github.com/gorhill/uBlock/issues/1070 -// https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Set#Browser_compatibility -// This polyfill is designed to fulfill *only* what uBlock Origin needs -- this -// is not an accurate API of the real Set() type. - -if ( self.Set instanceof Function === false ) { - self.Set = function(iter) { - this.clear(); - if ( Array.isArray(iter) ) { - for ( var i = 0, n = iter.length; i < n; i++ ) { - this.add(iter[i]); - } - return; - } - }; - - self.Set.polyfill = true; - - self.Set.prototype.clear = function() { - this._set = Object.create(null); - this.size = 0; - // Iterator stuff - this._values = undefined; - this._i = undefined; - this.value = undefined; - this.done = true; - }; - - self.Set.prototype.add = function(k) { - if ( this._set[k] === undefined ) { - this._set[k] = true; - this.size += 1; - } - return this; - }; - - self.Set.prototype.delete = function(k) { - if ( this._set[k] !== undefined ) { - delete this._set[k]; - this.size -= 1; - return true; - } - return false; - }; - - self.Set.prototype.has = function(k) { - return this._set[k] !== undefined; - }; - - self.Set.prototype.next = function() { - if ( this._i < this.size ) { - this.value = this._values[this._i++]; - } else { - this._values = undefined; - this.value = undefined; - this.done = true; - } - return this; - }; - - self.Set.prototype.values = function() { - this._values = Object.keys(this._set); - this._i = 0; - this.value = undefined; - this.done = false; - return this; - }; -} - -/******************************************************************************/ - -// https://github.com/gorhill/uBlock/issues/1070 -// https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Set#Browser_compatibility -// This polyfill is designed to fulfill *only* what uBlock Origin needs -- this -// is not an accurate API of the real Map() type. - -if ( self.Map instanceof Function === false ) { - self.Map = function(iter) { - this.clear(); - if ( Array.isArray(iter) ) { - for ( var i = 0, n = iter.length, entry; i < n; i++ ) { - entry = iter[i]; - this.set(entry[0], entry[1]); - } - return; - } - }; - - self.Map.polyfill = true; - - self.Map.prototype.clear = function() { - this._map = Object.create(null); - this.size = 0; - // Iterator stuff - this._keys = undefined; - this._i = undefined; - this.value = undefined; - this.done = true; - }; - - self.Map.prototype.delete = function(k) { - if ( this._map[k] !== undefined ) { - delete this._map[k]; - this.size -= 1; - return true; - } - return false; - }; - - self.Map.prototype.entries = function() { - this._keys = Object.keys(this._map); - this._i = 0; - this.value = [ undefined, undefined ]; - this.done = false; - return this; - }; - - self.Map.prototype.get = function(k) { - return this._map[k]; - }; - - self.Map.prototype.has = function(k) { - return this._map[k] !== undefined; - }; - - self.Map.prototype.next = function() { - if ( this._i < this.size ) { - var key = this._keys[this._i++]; - this.value[0] = key; - this.value[1] = this._map[key]; - } else { - this._keys = undefined; - this.value = undefined; - this.done = true; - } - return this; - }; - - self.Map.prototype.set = function(k, v) { - if ( v !== undefined ) { - if ( this._map[k] === undefined ) { - this.size += 1; - } - this._map[k] = v; - } else { - if ( this._map[k] !== undefined ) { - this.size -= 1; - } - delete this._map[k]; - } - return this; - }; -} - -/******************************************************************************/ diff --git a/platform/chromium/vapi-background.js b/platform/chromium/vapi-background.js index 6c64de0..9d29238 100644 --- a/platform/chromium/vapi-background.js +++ b/platform/chromium/vapi-background.js @@ -504,90 +504,59 @@ vAPI.messaging.broadcast = function(message) { /******************************************************************************/ /******************************************************************************/ -vAPI.net = {}; - -/******************************************************************************/ - -vAPI.net.registerListeners = function() { - var µm = µMatrix; - - // Normalizing request types - // >>>>>>>> - var extToTypeMap = new Map([ - ['eot','font'],['otf','font'],['svg','font'],['ttf','font'],['woff','font'],['woff2','font'], - ['mp3','media'],['mp4','media'],['webm','media'], - ['gif','image'],['ico','image'],['jpeg','image'],['jpg','image'],['png','image'],['webp','image'] - ]); - - var normalizeRequestDetails = function(details) { - if ( - details.tabId === -1 && - details.documentUrl === undefined && - details.initiator !== undefined - ) { - details.documentUrl = details.initiator; - } - - // The rest of the function code is to normalize request type - if ( details.type !== 'other' ) { return; } - - // Try to map known "extension" part of URL to request type. - var path = µm.URI.pathFromURI(details.url), - pos = path.indexOf('.', path.length - 6); - if ( pos !== -1 ) { - var type = extToTypeMap.get(path.slice(pos + 1)); - if ( type !== undefined ) { - details.type = type; +vAPI.net = { + listenerMap: new WeakMap(), + // legacy Chromium understands only these network request types. + validTypes: (function() { + let types = new Set([ + 'main_frame', + 'sub_frame', + 'stylesheet', + 'script', + 'image', + 'object', + 'xmlhttprequest', + 'other' + ]); + let wrrt = browser.webRequest.ResourceType; + if ( wrrt instanceof Object ) { + for ( let typeKey in wrrt ) { + if ( wrrt.hasOwnProperty(typeKey) ) { + types.add(wrrt[typeKey]); + } } } - }; - // <<<<<<<< - // End of: Normalizing request types - - // Network event handlers - // >>>>>>>> - var onBeforeRequestClient = this.onBeforeRequest.callback; - chrome.webRequest.onBeforeRequest.addListener( - function(details) { - normalizeRequestDetails(details); - return onBeforeRequestClient(details); - }, - { - 'urls': this.onBeforeRequest.urls || [ '' ], - 'types': this.onBeforeRequest.types || undefined - }, - this.onBeforeRequest.extra - ); - - var onBeforeSendHeadersClient = this.onBeforeSendHeaders.callback; - var onBeforeSendHeaders = function(details) { - normalizeRequestDetails(details); - return onBeforeSendHeadersClient(details); - }; - chrome.webRequest.onBeforeSendHeaders.addListener( - onBeforeSendHeaders, - { - 'urls': this.onBeforeSendHeaders.urls || [ '' ], - 'types': this.onBeforeSendHeaders.types || undefined - }, - this.onBeforeSendHeaders.extra - ); - - var onHeadersReceivedClient = this.onHeadersReceived.callback; - var onHeadersReceived = function(details) { - normalizeRequestDetails(details); - return onHeadersReceivedClient(details); - }; - chrome.webRequest.onHeadersReceived.addListener( - onHeadersReceived, - { - 'urls': this.onHeadersReceived.urls || [ '' ], - 'types': this.onHeadersReceived.types || undefined - }, - this.onHeadersReceived.extra - ); - // <<<<<<<< - // End of: Network event handlers + return types; + })(), + denormalizeFilters: null, + normalizeDetails: null, + addListener: function(which, clientListener, filters, options) { + if ( typeof this.denormalizeFilters === 'function' ) { + filters = this.denormalizeFilters(filters); + } + let actualListener; + if ( typeof this.normalizeDetails === 'function' ) { + actualListener = function(details) { + vAPI.net.normalizeDetails(details); + return clientListener(details); + }; + this.listenerMap.set(clientListener, actualListener); + } + browser.webRequest[which].addListener( + actualListener || clientListener, + filters, + options + ); + }, + removeListener: function(which, clientListener) { + let actualListener = this.listenerMap.get(clientListener); + if ( actualListener !== undefined ) { + this.listenerMap.delete(clientListener); + } + browser.webRequest[which].removeListener( + actualListener || clientListener + ); + }, }; /******************************************************************************/ diff --git a/platform/chromium/vapi-common.js b/platform/chromium/vapi-common.js index 3a5f947..659a5d4 100644 --- a/platform/chromium/vapi-common.js +++ b/platform/chromium/vapi-common.js @@ -1,7 +1,7 @@ /******************************************************************************* - uMatrix - a browser extension to block requests. - Copyright (C) 2014-2018 The uMatrix/uBlock Origin authors + uMatrix - a browser extension to black/white list requests. + Copyright (C) 2014-present The uMatrix/uBlock Origin authors This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -23,6 +23,14 @@ 'use strict'; +/******************************************************************************/ + +if ( self.browser instanceof Object ) { + self.chrome = self.browser; +} else { + self.browser = self.chrome; +} + /******************************************************************************/ /******************************************************************************/ diff --git a/platform/chromium/vapi-webrequest.js b/platform/chromium/vapi-webrequest.js new file mode 100644 index 0000000..2bb0794 --- /dev/null +++ b/platform/chromium/vapi-webrequest.js @@ -0,0 +1,194 @@ +/******************************************************************************* + + uMatrix - a browser extension to block requests. + Copyright (C) 2017-present Raymond Hill + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see {http://www.gnu.org/licenses/}. + + Home: https://github.com/gorhill/uMatrix +*/ + +// For background page + +'use strict'; + +/******************************************************************************/ + +(function() { + const extToTypeMap = new Map([ + ['eot','font'],['otf','font'],['svg','font'],['ttf','font'],['woff','font'],['woff2','font'], + ['mp3','media'],['mp4','media'],['webm','media'], + ['gif','image'],['ico','image'],['jpeg','image'],['jpg','image'],['png','image'],['webp','image'] + ]); + + // https://www.reddit.com/r/uBlockOrigin/comments/9vcrk3/bug_in_ubo_1173_betas_when_saving_files_hosted_on/ + // Some types can be mapped from 'other', thus include 'other' if and + // only if the caller is interested in at least one of those types. + const denormalizeTypes = function(aa) { + if ( aa.length === 0 ) { + return Array.from(vAPI.net.validTypes); + } + const out = new Set(); + let i = aa.length; + while ( i-- ) { + const type = aa[i]; + if ( vAPI.net.validTypes.has(type) ) { + out.add(type); + } + } + if ( out.has('other') === false ) { + for ( const type of extToTypeMap.values() ) { + if ( out.has(type) ) { + out.add('other'); + break; + } + } + } + return Array.from(out); + }; + + const headerValue = function(headers, name) { + let i = headers.length; + while ( i-- ) { + if ( headers[i].name.toLowerCase() === name ) { + return headers[i].value.trim(); + } + } + return ''; + }; + + const parsedURL = new URL('https://www.example.org/'); + + vAPI.net.normalizeDetails = function(details) { + // Chromium 63+ supports the `initiator` property, which contains + // the URL of the origin from which the network request was made. + if ( + typeof details.initiator === 'string' && + details.initiator !== 'null' + ) { + details.documentUrl = details.initiator; + } + + let type = details.type; + + // https://github.com/gorhill/uBlock/issues/1493 + // Chromium 49+/WebExtensions support a new request type: `ping`, + // which is fired as a result of using `navigator.sendBeacon`. + if ( type === 'ping' ) { + details.type = 'beacon'; + return; + } + + if ( type === 'imageset' ) { + details.type = 'image'; + return; + } + + // The rest of the function code is to normalize type + if ( type !== 'other' ) { return; } + + // Try to map known "extension" part of URL to request type. + parsedURL.href = details.url; + const path = parsedURL.pathname, + pos = path.indexOf('.', path.length - 6); + if ( pos !== -1 && (type = extToTypeMap.get(path.slice(pos + 1))) ) { + details.type = type; + return; + } + + // Try to extract type from response headers if present. + if ( details.responseHeaders ) { + type = headerValue(details.responseHeaders, 'content-type'); + if ( type.startsWith('font/') ) { + details.type = 'font'; + return; + } + if ( type.startsWith('image/') ) { + details.type = 'image'; + return; + } + if ( type.startsWith('audio/') || type.startsWith('video/') ) { + details.type = 'media'; + return; + } + } + }; + + vAPI.net.denormalizeFilters = function(filters) { + const urls = filters.urls || [ '' ]; + let types = filters.types; + if ( Array.isArray(types) ) { + types = denormalizeTypes(types); + } + if ( + (vAPI.net.validTypes.has('websocket')) && + (types === undefined || types.indexOf('websocket') !== -1) && + (urls.indexOf('') === -1) + ) { + if ( urls.indexOf('ws://*/*') === -1 ) { + urls.push('ws://*/*'); + } + if ( urls.indexOf('wss://*/*') === -1 ) { + urls.push('wss://*/*'); + } + } + return { types, urls }; + }; +})(); + +/******************************************************************************/ + +// https://github.com/gorhill/uBlock/issues/2067 +// Experimental: Block everything until uBO is fully ready. + +vAPI.net.onBeforeReady = (function() { + let pendings; + + const handler = function(details) { + if ( pendings === undefined ) { return; } + if ( details.tabId < 0 ) { return; } + + pendings.add(details.tabId); + + //console.log(`Aborting tab ${details.tabId}: ${details.type} ${details.url}`); + + return { cancel: true }; + }; + + return { + experimental: true, + start: function() { + pendings = new Set(); + browser.webRequest.onBeforeRequest.addListener( + handler, + { urls: [ 'http://*/*', 'https://*/*' ] }, + [ 'blocking' ] + ); + }, + // https://github.com/gorhill/uBlock/issues/2067 + // Force-reload tabs for which network requests were blocked + // during launch. This can happen only if tabs were "suspended". + stop: function() { + if ( pendings === undefined ) { return; } + browser.webRequest.onBeforeRequest.removeListener(handler); + for ( const tabId of pendings ) { + //console.log(`Reloading tab ${tabId}`); + vAPI.tabs.reload(tabId); + } + pendings = undefined; + }, + }; +})(); + +/******************************************************************************/ diff --git a/platform/firefox/polyfill.js b/platform/firefox/polyfill.js deleted file mode 100644 index 78e57a9..0000000 --- a/platform/firefox/polyfill.js +++ /dev/null @@ -1,30 +0,0 @@ -/******************************************************************************* - - uBlock Origin - a browser extension to block requests. - Copyright (C) 2016 The uBlock Origin authors - - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program. If not, see {http://www.gnu.org/licenses/}. - - Home: https://github.com/gorhill/uBlock -*/ - -// For background page or non-background pages - -'use strict'; - -/******************************************************************************/ - -var objectAssign = Object.assign; - -/******************************************************************************/ diff --git a/platform/firefox/vapi-webrequest.js b/platform/firefox/vapi-webrequest.js new file mode 100644 index 0000000..45b819c --- /dev/null +++ b/platform/firefox/vapi-webrequest.js @@ -0,0 +1,185 @@ +/******************************************************************************* + + uMatrix - a browser extension to block requests. + Copyright (C) 2017-present Raymond Hill + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see {http://www.gnu.org/licenses/}. + + Home: https://github.com/gorhill/uMatrix +*/ + +// For background page + +'use strict'; + +/******************************************************************************/ + +(function() { + + // https://github.com/gorhill/uBlock/issues/2950 + // Firefox 56 does not normalize URLs to ASCII, uBO must do this itself. + // https://bugzilla.mozilla.org/show_bug.cgi?id=945240 + const evalMustPunycode = function() { + return vAPI.webextFlavor.soup.has('firefox') && + vAPI.webextFlavor.major < 57; + }; + + let mustPunycode = evalMustPunycode(); + + // The real actual webextFlavor value may not be set in stone, so listen + // for possible future changes. + window.addEventListener('webextFlavor', ( ) => { + mustPunycode = evalMustPunycode(); + }, { once: true }); + + const denormalizeTypes = function(aa) { + if ( aa.length === 0 ) { + return Array.from(vAPI.net.validTypes); + } + const out = new Set(); + let i = aa.length; + while ( i-- ) { + let type = aa[i]; + if ( vAPI.net.validTypes.has(type) ) { + out.add(type); + } + if ( type === 'image' && vAPI.net.validTypes.has('imageset') ) { + out.add('imageset'); + } + if ( type === 'sub_frame' ) { + out.add('object'); + } + } + return Array.from(out); + }; + + const punycode = self.punycode; + const reAsciiHostname = /^https?:\/\/[0-9a-z_.:@-]+[/?#]/; + const parsedURL = new URL('about:blank'); + + vAPI.net.normalizeDetails = function(details) { + if ( mustPunycode && !reAsciiHostname.test(details.url) ) { + parsedURL.href = details.url; + details.url = details.url.replace( + parsedURL.hostname, + punycode.toASCII(parsedURL.hostname) + ); + } + + const type = details.type; + + // https://github.com/gorhill/uBlock/issues/1493 + // Chromium 49+/WebExtensions support a new request type: `ping`, + // which is fired as a result of using `navigator.sendBeacon`. + if ( type === 'ping' ) { + details.type = 'beacon'; + return; + } + + if ( type === 'imageset' ) { + details.type = 'image'; + return; + } + + // https://github.com/uBlockOrigin/uBlock-issues/issues/345 + // Re-categorize an embedded object as a `sub_frame` if its + // content type is that of a HTML document. + if ( type === 'object' && Array.isArray(details.responseHeaders) ) { + for ( const header of details.responseHeaders ) { + if ( header.name.toLowerCase() === 'content-type' ) { + if ( header.value.startsWith('text/html') ) { + details.type = 'sub_frame'; + } + break; + } + } + } + }; + + vAPI.net.denormalizeFilters = function(filters) { + const urls = filters.urls || [ '' ]; + let types = filters.types; + if ( Array.isArray(types) ) { + types = denormalizeTypes(types); + } + if ( + (vAPI.net.validTypes.has('websocket')) && + (types === undefined || types.indexOf('websocket') !== -1) && + (urls.indexOf('') === -1) + ) { + if ( urls.indexOf('ws://*/*') === -1 ) { + urls.push('ws://*/*'); + } + if ( urls.indexOf('wss://*/*') === -1 ) { + urls.push('wss://*/*'); + } + } + return { types, urls }; + }; +})(); + +/******************************************************************************/ + +// Related issues: +// - https://github.com/gorhill/uBlock/issues/1327 +// - https://github.com/uBlockOrigin/uBlock-issues/issues/128 +// - https://bugzilla.mozilla.org/show_bug.cgi?id=1503721 + +vAPI.net.onBeforeReady = (function() { + let pendings; + + const handler = function(details) { + if ( pendings === undefined ) { return; } + if ( details.tabId < 0 ) { return; } + + //console.log(`Deferring tab ${details.tabId}: ${details.type} ${details.url}`); + + const pending = { + details: Object.assign({}, details), + resolve: undefined, + promise: undefined + }; + + pending.promise = new Promise(function(resolve) { + pending.resolve = resolve; + }); + + pendings.push(pending); + + return pending.promise; + }; + + return { + start: function() { + pendings = []; + browser.webRequest.onBeforeRequest.addListener( + handler, + { urls: [ 'http://*/*', 'https://*/*' ] }, + [ 'blocking' ] + ); + }, + stop: function(resolver) { + if ( pendings === undefined ) { return; } + for ( const pending of pendings ) { + const details = pending.details; + vAPI.net.normalizeDetails(details); + //console.log(`Processing tab ${details.tabId}: ${details.type} ${details.url}`); + pending.resolve(resolver(details)); + } + pendings = undefined; + }, + }; +})(); + +/******************************************************************************/ diff --git a/src/background.html b/src/background.html index a5abee8..0a6a791 100644 --- a/src/background.html +++ b/src/background.html @@ -5,12 +5,17 @@ uMatrix - - + + + + + + + diff --git a/src/js/background.js b/src/js/background.js index 3c81606..6c1f8ba 100644 --- a/src/js/background.js +++ b/src/js/background.js @@ -1,7 +1,7 @@ /******************************************************************************* uMatrix - a browser extension to black/white list requests. - Copyright (C) 2014-2018 Raymond Hill + Copyright (C) 2014-present Raymond Hill This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -23,14 +23,14 @@ /******************************************************************************/ -var µMatrix = (function() { // jshint ignore:line +const µMatrix = (function() { // jshint ignore:line /******************************************************************************/ -var oneSecond = 1000; -var oneMinute = 60 * oneSecond; -var oneHour = 60 * oneMinute; -var oneDay = 24 * oneHour; +const oneSecond = 1000; +const oneMinute = 60 * oneSecond; +const oneHour = 60 * oneMinute; +const oneDay = 24 * oneHour; /******************************************************************************* @@ -53,7 +53,7 @@ var oneDay = 24 * oneHour; */ -var rawSettingsDefault = { +const rawSettingsDefault = { contributorMode: false, disableCSPReportInjection: false, enforceEscapedFragment: true, @@ -111,6 +111,7 @@ var rawSettingsDefault = { '' ].join(''), framePlaceholderBackground: 'default', + suspendTabsUntilReady: false }; /******************************************************************************/ @@ -154,7 +155,25 @@ return { }, rawSettingsDefault: rawSettingsDefault, - rawSettings: Object.assign({}, rawSettingsDefault), + rawSettings: (function() { + let out = Object.assign({}, rawSettingsDefault), + json = vAPI.localStorage.getItem('immediateRawSettings'); + if ( typeof json === 'string' ) { + try { + let o = JSON.parse(json); + if ( o instanceof Object ) { + for ( const k in o ) { + if ( out.hasOwnProperty(k) ) { + out[k] = o[k]; + } + } + } + } + catch(ex) { + } + } + return out; + })(), rawSettingsWriteTime: 0, clearBrowserCacheCycle: 0, diff --git a/src/js/raw-settings.js b/src/js/raw-settings.js index ae99d57..c1fb3f1 100644 --- a/src/js/raw-settings.js +++ b/src/js/raw-settings.js @@ -1,7 +1,7 @@ /******************************************************************************* - uMatrix - a browser extension to block requests. - Copyright (C) 2018 Raymond Hill + uMatrix - a browser extension to black/white list requests. + Copyright (C) 2018-present Raymond Hill This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by diff --git a/src/js/start.js b/src/js/start.js index b46038a..41f0bed 100644 --- a/src/js/start.js +++ b/src/js/start.js @@ -52,24 +52,6 @@ var processCallbackQueue = function(queue, callback) { var onAllDone = function() { µm.webRequest.start(); - vAPI.cloud.start([ 'myRulesPane' ]); -}; - -/******************************************************************************/ - -var onTabsReady = function(tabs) { - let i = tabs.length; - while ( i-- ) { - let tab = tabs[i]; - µm.tabContextManager.push(tab.id, tab.url, 'newURL'); - } - onAllDone(); -}; - -/******************************************************************************/ - -var onUserSettingsLoaded = function() { - µm.loadHostsFiles(); µm.loadRecipes(); // https://github.com/uBlockOrigin/uMatrix-issues/issues/63 @@ -77,30 +59,53 @@ var onUserSettingsLoaded = function() { // asset updater. µm.assets.addObserver(µm.assetObserver.bind(µm)); µm.scheduleAssetUpdater(µm.userSettings.autoUpdate ? 7 * 60 * 1000 : 0); + + vAPI.cloud.start([ 'myRulesPane' ]); }; /******************************************************************************/ var onPSLReady = function() { - µm.loadUserSettings(onUserSettingsLoaded); - µm.loadRawSettings(); - µm.loadMatrix(); - - // rhill 2013-11-24: bind behind-the-scene virtual tab/url manually, since the - // normal way forbid binding behind the scene tab. - // https://github.com/gorhill/httpswitchboard/issues/67 - let pageStore = - µm.pageStoreFactory(µm.tabContextManager.mustLookup(vAPI.noTabId)); - pageStore.title = vAPI.i18n('statsPageDetailedBehindTheScenePage'); - µm.pageStores.set(vAPI.noTabId, pageStore); - - vAPI.tabs.getAll(onTabsReady); + // TODO: Promisify + let count = 4; + const countdown = ( ) => { + count -= 1; + if ( count !== 0 ) { return; } + onAllDone(); + }; + + µm.loadRawSettings(countdown); + µm.loadMatrix(countdown); + µm.loadHostsFiles(countdown); + + vAPI.tabs.getAll(tabs => { + const pageStore = + µm.pageStoreFactory(µm.tabContextManager.mustLookup(vAPI.noTabId)); + pageStore.title = vAPI.i18n('statsPageDetailedBehindTheScenePage'); + µm.pageStores.set(vAPI.noTabId, pageStore); + + if ( Array.isArray(tabs) ) { + for ( const tab of tabs ) { + µm.tabContextManager.push(tab.id, tab.url, 'newURL'); + } + } + countdown(); + }); }; /******************************************************************************/ processCallbackQueue(µm.onBeforeStartQueue, function() { - µm.publicSuffixList.load(onPSLReady); + // TODO: Promisify + let count = 2; + const countdown = ( ) => { + count -= 1; + if ( count !== 0 ) { return; } + onPSLReady(); + }; + + µm.publicSuffixList.load(countdown); + µm.loadUserSettings(countdown); }); /******************************************************************************/ diff --git a/src/js/storage.js b/src/js/storage.js index 20ae6e1..002f9ab 100644 --- a/src/js/storage.js +++ b/src/js/storage.js @@ -1,7 +1,7 @@ /******************************************************************************* uMatrix - a Chromium browser extension to black/white list requests. - Copyright (C) 2014-2018 Raymond Hill + Copyright (C) 2014-present Raymond Hill This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -19,7 +19,7 @@ Home: https://github.com/gorhill/uMatrix */ -/* global objectAssign, punycode, publicSuffixList */ +/* global punycode, publicSuffixList */ 'use strict'; @@ -147,35 +147,43 @@ /******************************************************************************/ -µMatrix.loadRawSettings = function() { - var µm = this; +µMatrix.loadRawSettings = function(callback) { + if ( typeof callback !== 'function' ) { + callback = this.noopFunc; + } - var onLoaded = function(bin) { - if ( !bin || bin.rawSettings instanceof Object === false ) { return; } - for ( var key of Object.keys(bin.rawSettings) ) { + const onLoaded = bin => { + if ( + bin instanceof Object === false || + bin.rawSettings instanceof Object === false + ) { + return callback(); + } + for ( const key of Object.keys(bin.rawSettings) ) { if ( - µm.rawSettings.hasOwnProperty(key) === false || - typeof bin.rawSettings[key] !== typeof µm.rawSettings[key] + this.rawSettings.hasOwnProperty(key) === false || + typeof bin.rawSettings[key] !== typeof this.rawSettings[key] ) { continue; } - µm.rawSettings[key] = bin.rawSettings[key]; + this.rawSettings[key] = bin.rawSettings[key]; } - µm.rawSettingsWriteTime = Date.now(); + this.rawSettingsWriteTime = Date.now(); + callback(); }; vAPI.storage.get('rawSettings', onLoaded); }; µMatrix.saveRawSettings = function(rawSettings, callback) { - var keys = Object.keys(rawSettings); + const keys = Object.keys(rawSettings); if ( keys.length === 0 ) { if ( typeof callback === 'function' ) { callback(); } return; } - for ( var key of keys ) { + for ( const key of keys ) { if ( this.rawSettingsDefault.hasOwnProperty(key) && typeof rawSettings[key] === typeof this.rawSettingsDefault[key] @@ -184,6 +192,7 @@ } } vAPI.storage.set({ rawSettings: this.rawSettings }, callback); + this.saveImmediateHiddenSettings(); this.rawSettingsWriteTime = Date.now(); }; @@ -232,13 +241,25 @@ }; µMatrix.stringFromRawSettings = function() { - var out = []; - for ( var key of Object.keys(this.rawSettings).sort() ) { + const out = []; + for ( const key of Object.keys(this.rawSettings).sort() ) { out.push(key + ' ' + this.rawSettings[key]); } return out.join('\n'); }; +// These settings must be available immediately on startup, without delay +// through the vAPI.localStorage. Add/remove settings as needed. + +µMatrix.saveImmediateHiddenSettings = function() { + vAPI.localStorage.setItem( + 'immediateRawSettings', + JSON.stringify({ + suspendTabsUntilReady: this.rawSettings.suspendTabsUntilReady, + }) + ); +}; + /******************************************************************************/ µMatrix.saveMatrix = function() { @@ -249,17 +270,16 @@ if ( typeof callback !== 'function' ) { callback = this.noopFunc; } - let µm = this; - let onLoaded = function(bin) { + const onLoaded = bin => { if ( bin instanceof Object === false ) { return callback(); } if ( typeof bin.userMatrix === 'string' ) { - µm.pMatrix.fromString(bin.userMatrix); + this.pMatrix.fromString(bin.userMatrix); } else if ( Array.isArray(bin.userMatrix) ) { - µm.pMatrix.fromArray(bin.userMatrix); + this.pMatrix.fromArray(bin.userMatrix); } - µm.tMatrix.assign(µm.pMatrix); + this.tMatrix.assign(this.pMatrix); callback(); }; vAPI.storage.get('userMatrix', onLoaded); @@ -413,7 +433,7 @@ µm.assets.remove(assetKey); continue; } - availableHostsFiles.set(assetKey, objectAssign({}, entry)); + availableHostsFiles.set(assetKey, Object.assign({}, entry)); } vAPI.storage.get('liveHostsFiles', onHostsFilesDataReady); @@ -455,7 +475,7 @@ µm.assets.remove(assetKey); continue; } - availableRecipeFiles.set(assetKey, objectAssign({}, entry)); + availableRecipeFiles.set(assetKey, Object.assign({}, entry)); } for ( let asseyKey of µm.userSettings.selectedRecipeFiles ) { @@ -474,40 +494,39 @@ /******************************************************************************/ µMatrix.loadHostsFiles = function(callback) { - var µm = µMatrix; - var hostsFileLoadCount; + let hostsFileLoadCount; if ( typeof callback !== 'function' ) { callback = this.noopFunc; } - var loadHostsFilesEnd = function(fromSelfie) { + const loadHostsFilesEnd = fromSelfie => { if ( fromSelfie !== true ) { - µm.ubiquitousBlacklist.freeze(); - vAPI.storage.set({ liveHostsFiles: Array.from(µm.liveHostsFiles) }); - µm.hostsFilesSelfie.create(); + this.ubiquitousBlacklist.freeze(); + vAPI.storage.set({ liveHostsFiles: Array.from(this.liveHostsFiles) }); + this.hostsFilesSelfie.create(); } vAPI.messaging.broadcast({ what: 'loadHostsFilesCompleted' }); - µm.getBytesInUse(); + this.getBytesInUse(); callback(); }; - var mergeHostsFile = function(details) { - µm.mergeHostsFile(details); + const mergeHostsFile = details => { + this.mergeHostsFile(details); hostsFileLoadCount -= 1; if ( hostsFileLoadCount === 0 ) { loadHostsFilesEnd(); } }; - var loadHostsFilesStart = function(hostsFiles) { - µm.liveHostsFiles = hostsFiles; - µm.ubiquitousBlacklist.reset(); - hostsFileLoadCount = µm.userSettings.selectedHostsFiles.length; + const loadHostsFilesStart = hostsFiles => { + this.liveHostsFiles = hostsFiles; + this.ubiquitousBlacklist.reset(); + hostsFileLoadCount = this.userSettings.selectedHostsFiles.length; // Load all hosts file which are not disabled. - for ( let assetKey of µm.userSettings.selectedHostsFiles ) { - µm.assets.get(assetKey, mergeHostsFile); + for ( const assetKey of this.userSettings.selectedHostsFiles ) { + this.assets.get(assetKey, mergeHostsFile); } // https://github.com/gorhill/uMatrix/issues/2 @@ -517,11 +536,11 @@ } }; - var onSelfieReady = function(status) { + const onSelfieReady = status => { if ( status === true ) { return loadHostsFilesEnd(true); } - µm.getAvailableHostsFiles(loadHostsFilesStart); + this.getAvailableHostsFiles(loadHostsFilesStart); }; this.hostsFilesSelfie.load(onSelfieReady); diff --git a/src/js/traffic.js b/src/js/traffic.js index e98506f..3b5da78 100644 --- a/src/js/traffic.js +++ b/src/js/traffic.js @@ -341,7 +341,7 @@ var onBeforeSendHeadersHandler = function(details) { // This fixes: // https://github.com/gorhill/httpswitchboard/issues/35 -var onHeadersReceived = function(details) { +var onHeadersReceivedHandler = function(details) { // Ignore schemes other than 'http...' let µm = µMatrix, tabId = details.tabId, @@ -502,25 +502,6 @@ var requestTypeNormalizer = { 'xmlhttprequest': 'xhr' }; -/******************************************************************************/ - -vAPI.net.onBeforeRequest = { - extra: [ 'blocking' ], - callback: onBeforeRequestHandler -}; - -vAPI.net.onBeforeSendHeaders = { - extra: [ 'blocking', 'requestHeaders' ], - callback: onBeforeSendHeadersHandler -}; - -vAPI.net.onHeadersReceived = { - urls: [ 'http://*/*', 'https://*/*' ], - types: [ 'main_frame', 'sub_frame' ], - extra: [ 'blocking', 'responseHeaders' ], - callback: onHeadersReceived -}; - /******************************************************************************* Use a `http-equiv` `meta` tag to enforce CSP directives for documents @@ -614,15 +595,58 @@ vAPI.net.onHeadersReceived = { /******************************************************************************/ -var start = function() { - vAPI.net.registerListeners(); -}; +const start = (function() { + if ( + vAPI.net.onBeforeReady instanceof Object && + ( + vAPI.net.onBeforeReady.experimental !== true || + µMatrix.rawSettings.suspendTabsUntilReady + ) + ) { + vAPI.net.onBeforeReady.start(); + } -/******************************************************************************/ + return function() { + vAPI.net.addListener( + 'onBeforeRequest', + onBeforeRequestHandler, + { }, + [ 'blocking' ] + ); -return { - start: start -}; + // https://github.com/uBlockOrigin/uMatrix-issues/issues/74#issuecomment-450687707 + // https://groups.google.com/a/chromium.org/forum/#!topic/chromium-extensions/vYIaeezZwfQ + // Chromium 72+: use `extraHeaders` to keep the ability to access + // the `Cookie`, `Referer` headers. + const beforeSendHeadersExtra = [ 'blocking', 'requestHeaders' ]; + const wrObsho = browser.webRequest.OnBeforeSendHeadersOptions; + if ( + wrObsho instanceof Object && + wrObsho.hasOwnProperty('EXTRA_HEADERS') + ) { + beforeSendHeadersExtra.push(wrObsho.EXTRA_HEADERS); + } + vAPI.net.addListener( + 'onBeforeSendHeaders', + onBeforeSendHeadersHandler, + { }, + beforeSendHeadersExtra + ); + + vAPI.net.addListener( + 'onHeadersReceived', + onHeadersReceivedHandler, + { types: [ 'main_frame', 'sub_frame' ] }, + [ 'blocking', 'responseHeaders' ] + ); + + if ( vAPI.net.onBeforeReady instanceof Object ) { + vAPI.net.onBeforeReady.stop(onBeforeRequestHandler); + } + }; +})(); + +return { start }; /******************************************************************************/ diff --git a/tools/make-firefox.sh b/tools/make-firefox.sh index aa2366b..618dd3b 100755 --- a/tools/make-firefox.sh +++ b/tools/make-firefox.sh @@ -16,8 +16,7 @@ cp platform/chromium/*.js $DES/js/ cp -R platform/chromium/img/* $DES/img/ cp LICENSE.txt $DES/ -cp platform/firefox/polyfill.js $DES/js/ -cp platform/firefox/vapi-cachestorage.js $DES/js/ +cp platform/firefox/*.js $DES/js/ cp platform/firefox/manifest.json $DES/ echo "*** uMatrix.firefox: Generating meta..."