diff --git a/src/js/liquid-dict.js b/src/js/liquid-dict.js index 257e3b3..4e0b335 100644 --- a/src/js/liquid-dict.js +++ b/src/js/liquid-dict.js @@ -1,7 +1,7 @@ /******************************************************************************* - µMatrix - a Chromium browser extension to black/white list requests. - Copyright (C) 2014 Raymond Hill + uMatrix - a Chromium browser extension to black/white list requests. + Copyright (C) 2014-2018 Raymond Hill This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -19,6 +19,8 @@ Home: https://github.com/gorhill/uMatrix */ +'use strict'; + /******************************************************************************/ µMatrix.LiquidDict = (function() { @@ -26,55 +28,37 @@ /******************************************************************************/ var LiquidDict = function() { - this.dict = {}; - this.count = 0; - this.duplicateCount = 0; - this.bucketCount = 0; - this.frozenBucketCount = 0; - - // Somewhat arbitrary: I need to come up with hard data to know at which - // point binary search is better than indexOf. - this.cutoff = 500; + this.dict = new Map(); + this.reset(); }; /******************************************************************************/ -var meltBucket = function(ldict, len, bucket) { - ldict.frozenBucketCount -= 1; - var map = {}; - if ( bucket.charAt(0) === ' ' ) { - bucket.trim().split(' ').map(function(k) { - map[k] = true; - }); - } else { - var offset = 0; - while ( offset < bucket.length ) { - map[bucket.substring(offset, len)] = true; - offset += len; - } - } - return map; -}; +// Somewhat arbitrary: I need to come up with hard data to know at which +// point binary search is better than indexOf. + +LiquidDict.prototype.cutoff = 500; /******************************************************************************/ -var melt = function(ldict) { - var buckets = ldict.dict; - var bucket; - for ( var key in buckets ) { - bucket = buckets[key]; - if ( typeof bucket === 'string' ) { - buckets[key] = meltBucket(ldict, key.charCodeAt(0) & 0xFF, bucket); - } +var meltBucket = function(ldict, len, bucket) { + ldict.frozenBucketCount -= 1; + if ( bucket.charCodeAt(0) === 0x20 /* ' ' */ ) { + return new Set(bucket.trim().split(' ')); } + let dict = new Set(); + let offset = 0; + while ( offset < bucket.length ) { + dict.add(bucket.substring(offset, len)); + offset += len; + } + return dict; }; -/******************************************************************************/ - var freezeBucket = function(ldict, bucket) { ldict.frozenBucketCount += 1; - var words = Object.keys(bucket); - var wordLen = words[0].length; + let words = Array.from(bucket); + let wordLen = words[0].length; if ( wordLen * words.length < ldict.cutoff ) { return ' ' + words.join(' ') + ' '; } @@ -91,43 +75,38 @@ var freezeBucket = function(ldict, bucket) { // helper function? LiquidDict.prototype.makeKey = function(word) { - var len = word.length; - if ( len > 255 ) { - len = 255; - } - var i = len >> 2; - return String.fromCharCode( - (word.charCodeAt( 0) & 0x03) << 14 | - (word.charCodeAt( i) & 0x03) << 12 | - (word.charCodeAt( i+i) & 0x03) << 10 | - (word.charCodeAt(i+i+i) & 0x03) << 8 | - len - ); + let len = word.length; + if ( len > 255 ) { len = 255; } + let i = len >> 2; + return (word.charCodeAt( 0) & 0x03) << 14 | + (word.charCodeAt( i) & 0x03) << 12 | + (word.charCodeAt( i+i) & 0x03) << 10 | + (word.charCodeAt(i+i+i) & 0x03) << 8 | + len; }; /******************************************************************************/ LiquidDict.prototype.test = function(word) { - var key = this.makeKey(word); - var bucket = this.dict[key]; + let key = this.makeKey(word); + let bucket = this.dict.get(key); if ( bucket === undefined ) { return false; } if ( typeof bucket === 'object' ) { - return bucket[word] !== undefined; + return bucket.has(word); } - if ( bucket.charAt(0) === ' ' ) { - return bucket.indexOf(' ' + word + ' ') >= 0; + if ( bucket.charCodeAt(0) === 0x20 /* ' ' */ ) { + return bucket.indexOf(' ' + word + ' ') !== -1; } // binary search - var len = word.length; - var left = 0; + let len = word.length; + let left = 0; // http://jsperf.com/or-vs-floor/3 - var right = ~~(bucket.length / len + 0.5); - var i, needle; + let right = ~~(bucket.length / len + 0.5); while ( left < right ) { - i = left + right >> 1; - needle = bucket.substr( len * i, len ); + let i = left + right >> 1; + let needle = bucket.substr( len * i, len ); if ( word < needle ) { right = i; } else if ( word > needle ) { @@ -142,22 +121,21 @@ LiquidDict.prototype.test = function(word) { /******************************************************************************/ LiquidDict.prototype.add = function(word) { - var key = this.makeKey(word); - if ( key === undefined ) { - return false; - } - var bucket = this.dict[key]; + let key = this.makeKey(word); + let bucket = this.dict.get(key); if ( bucket === undefined ) { - this.dict[key] = bucket = {}; - this.bucketCount += 1; - bucket[word] = true; + bucket = new Set(); + this.dict.set(key, bucket); + bucket.add(word); this.count += 1; return true; - } else if ( typeof bucket === 'string' ) { - this.dict[key] = bucket = meltBucket(this, word.len, bucket); } - if ( bucket[word] === undefined ) { - bucket[word] = true; + if ( typeof bucket === 'string' ) { + bucket = meltBucket(this, word.len, bucket); + this.dict.set(key, bucket); + } + if ( bucket.has(word) === false ) { + bucket.add(word); this.count += 1; return true; } @@ -168,12 +146,9 @@ LiquidDict.prototype.add = function(word) { /******************************************************************************/ LiquidDict.prototype.freeze = function() { - var buckets = this.dict; - var bucket; - for ( var key in buckets ) { - bucket = buckets[key]; - if ( typeof bucket === 'object' ) { - buckets[key] = freezeBucket(this, bucket); + for ( let entry of this.dict ) { + if ( typeof entry[1] === 'object' ) { + this.dict.set(entry[0], freezeBucket(this, entry[1])); } } }; @@ -181,15 +156,38 @@ LiquidDict.prototype.freeze = function() { /******************************************************************************/ LiquidDict.prototype.reset = function() { - this.dict = {}; + this.dict.clear(); this.count = 0; this.duplicateCount = 0; - this.bucketCount = 0; this.frozenBucketCount = 0; }; /******************************************************************************/ +let selfieVersion = 1; + +LiquidDict.prototype.toSelfie = function() { + this.freeze(); + return { + version: selfieVersion, + count: this.count, + duplicateCount: this.duplicateCount, + frozenBucketCount: this.frozenBucketCount, + dict: Array.from(this.dict) + }; +}; + +LiquidDict.prototype.fromSelfie = function(selfie) { + if ( selfie.version !== selfieVersion ) { return false; } + this.count = selfie.count; + this.duplicateCount = selfie.duplicateCount; + this.frozenBucketCount = selfie.frozenBucketCount; + this.dict = new Map(selfie.dict); + return true; +}; + +/******************************************************************************/ + return LiquidDict; /******************************************************************************/ @@ -199,4 +197,3 @@ return LiquidDict; /******************************************************************************/ µMatrix.ubiquitousBlacklist = new µMatrix.LiquidDict(); -µMatrix.ubiquitousWhitelist = new µMatrix.LiquidDict(); diff --git a/src/js/storage.js b/src/js/storage.js index 052ae6b..5e2134a 100644 --- a/src/js/storage.js +++ b/src/js/storage.js @@ -439,9 +439,12 @@ callback = this.noopFunc; } - var loadHostsFilesEnd = function() { - µm.ubiquitousBlacklist.freeze(); - vAPI.storage.set({ liveHostsFiles: Array.from(µm.liveHostsFiles) }); + var loadHostsFilesEnd = function(fromSelfie) { + if ( fromSelfie !== true ) { + µm.ubiquitousBlacklist.freeze(); + vAPI.storage.set({ liveHostsFiles: Array.from(µm.liveHostsFiles) }); + µm.hostsFilesSelfie.create(); + } vAPI.messaging.broadcast({ what: 'loadHostsFilesCompleted' }); µm.getBytesInUse(); callback(); @@ -472,7 +475,14 @@ } }; - this.getAvailableHostsFiles(loadHostsFilesStart); + var onSelfieReady = function(status) { + if ( status === true ) { + return loadHostsFilesEnd(true); + } + µm.getAvailableHostsFiles(loadHostsFilesStart); + }; + + this.hostsFilesSelfie.load(onSelfieReady); }; /******************************************************************************/ @@ -647,6 +657,9 @@ 'selectedHostsFiles', 'externalHostsFiles' ); + if ( hostsChanged ) { + µm.hostsFilesSelfie.destroy(); + } let recipesChanged = applyAssetSelection( metadata, details.recipes, @@ -657,7 +670,6 @@ µm.recipeManager.reset(); µm.loadRecipes(true); } - if ( typeof callback === 'function' ) { callback({ hostsChanged: hostsChanged, @@ -680,7 +692,50 @@ /******************************************************************************/ +µMatrix.hostsFilesSelfie = (function() { + let timer; + + return { + create: function() { + this.cancel(); + timer = vAPI.setTimeout( + function() { + timer = undefined; + vAPI.cacheStorage.set({ + hostsFilesSelfie: µMatrix.ubiquitousBlacklist.toSelfie() + }); + }, + 120000 + ); + }, + destroy: function() { + this.cancel(); + vAPI.cacheStorage.remove('hostsFilesSelfie'); + }, + load: function(callback) { + this.cancel(); + vAPI.cacheStorage.get('hostsFilesSelfie', function(bin) { + callback( + bin instanceof Object && + bin.hostsFilesSelfie instanceof Object && + µMatrix.ubiquitousBlacklist.fromSelfie(bin.hostsFilesSelfie) + ); + }); + }, + cancel: function() { + if ( timer !== undefined ) { + clearTimeout(timer); + } + timer = undefined; + } + }; +})(); + +/******************************************************************************/ + µMatrix.loadPublicSuffixList = function(callback) { + let µm = this; + if ( typeof callback !== 'function' ) { callback = this.noopFunc; } @@ -688,15 +743,64 @@ var applyPublicSuffixList = function(details) { if ( !details.error ) { publicSuffixList.parse(details.content, punycode.toASCII); + µm.publicSuffixListSelfie.create(); } callback(); }; - this.assets.get(this.pslAssetKey, applyPublicSuffixList); + let onSelfieReady = function(status) { + if ( status === true ) { + return callback(); + } + µm.assets.get(µm.pslAssetKey, applyPublicSuffixList); + }; + + this.publicSuffixListSelfie.load(onSelfieReady); }; /******************************************************************************/ +µMatrix.publicSuffixListSelfie = (function() { + let timer; + + return { + create: function() { + this.cancel(); + timer = vAPI.setTimeout( + function() { + timer = undefined; + vAPI.cacheStorage.set({ + publicSuffixListSelfie: publicSuffixList.toSelfie() + }); + }, + 60000 + ); + }, + destroy: function() { + this.cancel(); + vAPI.cacheStorage.remove('publicSuffixListSelfie'); + }, + load: function(callback) { + this.cancel(); + vAPI.cacheStorage.get('publicSuffixListSelfie', function(bin) { + callback( + bin instanceof Object && + bin.publicSuffixListSelfie instanceof Object && + publicSuffixList.fromSelfie(bin.publicSuffixListSelfie) + ); + }); + }, + cancel: function() { + if ( timer !== undefined ) { + clearTimeout(timer); + } + timer = undefined; + } + }; +})(); + +/******************************************************************************/ + µMatrix.scheduleAssetUpdater = (function() { var timer, next = 0; return function(updateDelay) { @@ -727,9 +831,10 @@ /******************************************************************************/ µMatrix.assetObserver = function(topic, details) { + let µmus = this.userSettings; + // Do not update filter list if not in use. if ( topic === 'before-asset-updated' ) { - let µmus = this.userSettings; if ( details.type === 'internal' || details.type === 'filters' && @@ -743,6 +848,14 @@ } if ( topic === 'after-asset-updated' ) { + if ( + details.type === 'filters' && + µmus.selectedHostsFiles.indexOf(details.assetKey) !== -1 + ) { + this.hostsFilesSelfie.destroy(); + } else if ( details.assetKey === this.pslAssetKey ) { + this.publicSuffixListSelfie.destroy(); + } vAPI.messaging.broadcast({ what: 'assetUpdated', key: details.assetKey, @@ -766,7 +879,7 @@ if ( this.arraysIntersect( details.assetKeys, - this.userSettings.selectedRecipeFiles + µmus.selectedRecipeFiles ) ) { this.loadRecipes(true); @@ -774,12 +887,12 @@ if ( this.arraysIntersect( details.assetKeys, - this.userSettings.selectedHostsFiles + µmus.selectedHostsFiles ) ) { this.loadHostsFiles(); } - if ( this.userSettings.autoUpdate ) { + if ( µmus.autoUpdate ) { this.scheduleAssetUpdater(25200000); } else { this.scheduleAssetUpdater(0); diff --git a/src/lib/publicsuffixlist.js b/src/lib/publicsuffixlist.js index a823188..79605a1 100644 --- a/src/lib/publicsuffixlist.js +++ b/src/lib/publicsuffixlist.js @@ -37,9 +37,8 @@ /******************************************************************************/ -var exceptions = {}; -var rules = {}; -var selfieMagic = 'iscjsfsaolnm'; +var exceptions = new Map(); +var rules = new Map(); // This value dictate how the search will be performed: // < this.cutoffLength = indexOf() @@ -92,9 +91,8 @@ function getPublicSuffix(hostname) { } // Since we slice down the hostname with each pass, the first match // is the longest, so no need to find all the matching rules. - var pos; while ( true ) { - pos = hostname.indexOf('.'); + let pos = hostname.indexOf('.'); if ( pos < 0 ) { return hostname; } @@ -118,17 +116,17 @@ function getPublicSuffix(hostname) { function search(store, hostname) { // Extract TLD - var pos = hostname.lastIndexOf('.'); - var tld, remainder; - if ( pos < 0 ) { + let tld, remainder; + let pos = hostname.lastIndexOf('.'); + if ( pos === -1 ) { tld = hostname; remainder = hostname; } else { tld = hostname.slice(pos + 1); remainder = hostname.slice(0, pos); } - var substore = store[tld]; - if ( !substore ) { + let substore = store.get(tld); + if ( substore === undefined ) { return false; } // If substore is a string, use indexOf() @@ -136,17 +134,19 @@ function search(store, hostname) { return substore.indexOf(' ' + remainder + ' ') >= 0; } // It is an array: use binary search. - var l = remainder.length; - var haystack = substore[l]; - if ( !haystack ) { + let l = remainder.length; + if ( l >= substore.length ) { return false; } - var left = 0; - var right = Math.floor(haystack.length / l + 0.5); - var i, needle; + let haystack = substore[l]; + if ( haystack === null ) { + return false; + } + let left = 0; + let right = Math.floor(haystack.length / l + 0.5); while ( left < right ) { - i = left + right >> 1; - needle = haystack.substr( l * i, l ); + let i = left + right >> 1; + let needle = haystack.substr(l*i, l); if ( remainder < needle ) { right = i; } else if ( remainder > needle ) { @@ -168,22 +168,21 @@ function search(store, hostname) { // Suggestion: use it's quite good. function parse(text, toAscii) { - exceptions = {}; - rules = {}; + exceptions = new Map(); + rules = new Map(); - var lineBeg = 0, lineEnd; - var textEnd = text.length; - var line, store, pos, tld; + let lineBeg = 0; + let textEnd = text.length; while ( lineBeg < textEnd ) { - lineEnd = text.indexOf('\n', lineBeg); + let lineEnd = text.indexOf('\n', lineBeg); if ( lineEnd < 0 ) { lineEnd = text.indexOf('\r', lineBeg); if ( lineEnd < 0 ) { lineEnd = textEnd; } } - line = text.slice(lineBeg, lineEnd).trim(); + let line = text.slice(lineBeg, lineEnd).trim(); lineBeg = lineEnd + 1; if ( line.length === 0 ) { @@ -191,18 +190,19 @@ function parse(text, toAscii) { } // Ignore comments - pos = line.indexOf('//'); - if ( pos >= 0 ) { + let pos = line.indexOf('//'); + if ( pos !== -1 ) { line = line.slice(0, pos); } // Ignore surrounding whitespaces line = line.trim(); - if ( !line ) { + if ( line.length === 0 ) { continue; } // Is this an exception rule? + let store; if ( line.charAt(0) === '!' ) { store = exceptions; line = line.slice(1); @@ -220,8 +220,9 @@ function parse(text, toAscii) { line = line.toLowerCase(); // Extract TLD + let tld; pos = line.lastIndexOf('.'); - if ( pos < 0 ) { + if ( pos === -1 ) { tld = line; } else { tld = line.slice(pos + 1); @@ -229,13 +230,15 @@ function parse(text, toAscii) { } // Store suffix using tld as key - if ( !store.hasOwnProperty(tld) ) { - store[tld] = []; + let substore = store.get(tld); + if ( substore === undefined ) { + store.set(tld, substore = []); } if ( line ) { - store[tld].push(line); + substore.push(line); } } + crystallize(exceptions); crystallize(rules); @@ -248,69 +251,81 @@ function parse(text, toAscii) { // for future look up. function crystallize(store) { - var suffixes, suffix, i, l; + for ( let entry of store ) { + let tld = entry[0]; + let suffixes = entry[1]; - for ( var tld in store ) { - if ( !store.hasOwnProperty(tld) ) { - continue; - } - suffixes = store[tld].join(' '); // No suffix - if ( !suffixes ) { - store[tld] = ''; + if ( suffixes.length === 0 ) { + store.set(tld, ''); continue; } + // Concatenated list of suffixes less than cutoff length: // Store as string, lookup using indexOf() - if ( suffixes.length < cutoffLength ) { - store[tld] = ' ' + suffixes + ' '; + let s = suffixes.join(' '); + if ( s.length < cutoffLength ) { + store.set(tld, ' ' + s + ' '); continue; } + // Concatenated list of suffixes greater or equal to cutoff length // Store as array keyed on suffix length, lookup using binary search. // I borrowed the idea to key on string length here: // http://ejohn.org/blog/dictionary-lookups-in-javascript/#comment-392072 - - i = store[tld].length; - suffixes = []; - while ( i-- ) { - suffix = store[tld][i]; - l = suffix.length; - if ( !suffixes[l] ) { - suffixes[l] = []; + let buckets = []; + for ( let suffix of suffixes ) { + let l = suffix.length; + if ( buckets.length <= l ) { + extendArray(buckets, l); } - suffixes[l].push(suffix); + if ( buckets[l] === null ) { + buckets[l] = []; + } + buckets[l].push(suffix); } - l = suffixes.length; - while ( l-- ) { - if ( suffixes[l] ) { - suffixes[l] = suffixes[l].sort().join(''); + for ( let i = 0; i < buckets.length; i++ ) { + let bucket = buckets[i]; + if ( bucket !== null ) { + buckets[i] = bucket.sort().join(''); } } - store[tld] = suffixes; + store.set(tld, buckets); } + return store; } +let extendArray = function(aa, rb) { + for ( let i = aa.length; i <= rb; i++ ) { + aa.push(null); + } +}; + /******************************************************************************/ -function toSelfie() { +let selfieMagic = 3; + +let toSelfie = function() { return { magic: selfieMagic, - rules: rules, - exceptions: exceptions + rules: Array.from(rules), + exceptions: Array.from(exceptions) }; -} +}; -function fromSelfie(selfie) { - if ( typeof selfie !== 'object' || typeof selfie.magic !== 'string' || selfie.magic !== selfieMagic ) { +let fromSelfie = function(selfie) { + if ( + selfie instanceof Object === false || + selfie.magic !== selfieMagic + ) { return false; } - rules = selfie.rules; - exceptions = selfie.exceptions; + rules = new Map(selfie.rules); + exceptions = new Map(selfie.exceptions); callListeners(onChangedListeners); return true; -} +}; /******************************************************************************/