Mirror
/
nuTensor
mirror of https://github.com/geekprojects/nuTensor.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity
Browse Source

fix #935

pull/2/head
Raymond Hill 7 years ago
parent
b8c4eadc7a
commit
e9e5aa295c
No known key found for this signature in database GPG Key ID: 25E1490B761470C2
3 changed files with 283 additions and 158 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 155
      src/js/liquid-dict.js
  2. 129
      src/js/storage.js
  3. 145
      src/lib/publicsuffixlist.js

155
src/js/liquid-dict.js
View File

@ -1,7 +1,7 @@
/*******************************************************************************
µMatrix - a Chromium browser extension to black/white list requests.
Copyright (C) 2014 Raymond Hill
uMatrix - a Chromium browser extension to black/white list requests.
Copyright (C) 2014-2018 Raymond Hill
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
@ -19,6 +19,8 @@
Home: https://github.com/gorhill/uMatrix
*/
'use strict';
/******************************************************************************/
µMatrix.LiquidDict = (function() {
@ -26,55 +28,37 @@
/******************************************************************************/
var LiquidDict = function() {
this.dict = {};
this.count = 0;
this.duplicateCount = 0;
this.bucketCount = 0;
this.frozenBucketCount = 0;
// Somewhat arbitrary: I need to come up with hard data to know at which
// point binary search is better than indexOf.
this.cutoff = 500;
this.dict = new Map();
this.reset();
};
/******************************************************************************/
var meltBucket = function(ldict, len, bucket) {
ldict.frozenBucketCount -= 1;
var map = {};
if ( bucket.charAt(0) === ' ' ) {
bucket.trim().split(' ').map(function(k) {
map[k] = true;
});
} else {
var offset = 0;
while ( offset < bucket.length ) {
map[bucket.substring(offset, len)] = true;
offset += len;
}
}
return map;
};
// Somewhat arbitrary: I need to come up with hard data to know at which
// point binary search is better than indexOf.
LiquidDict.prototype.cutoff = 500;
/******************************************************************************/
var melt = function(ldict) {
var buckets = ldict.dict;
var bucket;
for ( var key in buckets ) {
bucket = buckets[key];
if ( typeof bucket === 'string' ) {
buckets[key] = meltBucket(ldict, key.charCodeAt(0) & 0xFF, bucket);
var meltBucket = function(ldict, len, bucket) {
ldict.frozenBucketCount -= 1;
if ( bucket.charCodeAt(0) === 0x20 /* ' ' */ ) {
return new Set(bucket.trim().split(' '));
}
let dict = new Set();
let offset = 0;
while ( offset < bucket.length ) {
dict.add(bucket.substring(offset, len));
offset += len;
}
return dict;
};
/******************************************************************************/
var freezeBucket = function(ldict, bucket) {
ldict.frozenBucketCount += 1;
var words = Object.keys(bucket);
var wordLen = words[0].length;
let words = Array.from(bucket);
let wordLen = words[0].length;
if ( wordLen * words.length < ldict.cutoff ) {
return ' ' + words.join(' ') + ' ';
}
@ -91,43 +75,38 @@ var freezeBucket = function(ldict, bucket) {
// helper function?
LiquidDict.prototype.makeKey = function(word) {
var len = word.length;
if ( len > 255 ) {
len = 255;
}
var i = len >> 2;
return String.fromCharCode(
(word.charCodeAt( 0) & 0x03) << 14 |
let len = word.length;
if ( len > 255 ) { len = 255; }
let i = len >> 2;
return (word.charCodeAt( 0) & 0x03) << 14 |
(word.charCodeAt( i) & 0x03) << 12 |
(word.charCodeAt( i+i) & 0x03) << 10 |
(word.charCodeAt(i+i+i) & 0x03) << 8 |
len
);
len;
};
/******************************************************************************/
LiquidDict.prototype.test = function(word) {
var key = this.makeKey(word);
var bucket = this.dict[key];
let key = this.makeKey(word);
let bucket = this.dict.get(key);
if ( bucket === undefined ) {
return false;
}
if ( typeof bucket === 'object' ) {
return bucket[word] !== undefined;
return bucket.has(word);
}
if ( bucket.charAt(0) === ' ' ) {
return bucket.indexOf(' ' + word + ' ') >= 0;
if ( bucket.charCodeAt(0) === 0x20 /* ' ' */ ) {
return bucket.indexOf(' ' + word + ' ') !== -1;
}
// binary search
var len = word.length;
var left = 0;
let len = word.length;
let left = 0;
// http://jsperf.com/or-vs-floor/3
var right = ~~(bucket.length / len + 0.5);
var i, needle;
let right = ~~(bucket.length / len + 0.5);
while ( left < right ) {
i = left + right >> 1;
needle = bucket.substr( len * i, len );
let i = left + right >> 1;
let needle = bucket.substr( len * i, len );
if ( word < needle ) {
right = i;
} else if ( word > needle ) {
@ -142,22 +121,21 @@ LiquidDict.prototype.test = function(word) {
/******************************************************************************/
LiquidDict.prototype.add = function(word) {
var key = this.makeKey(word);
if ( key === undefined ) {
return false;
}
var bucket = this.dict[key];
let key = this.makeKey(word);
let bucket = this.dict.get(key);
if ( bucket === undefined ) {
this.dict[key] = bucket = {};
this.bucketCount += 1;
bucket[word] = true;
bucket = new Set();
this.dict.set(key, bucket);
bucket.add(word);
this.count += 1;
return true;
} else if ( typeof bucket === 'string' ) {
this.dict[key] = bucket = meltBucket(this, word.len, bucket);
}
if ( bucket[word] === undefined ) {
bucket[word] = true;
if ( typeof bucket === 'string' ) {
bucket = meltBucket(this, word.len, bucket);
this.dict.set(key, bucket);
}
if ( bucket.has(word) === false ) {
bucket.add(word);
this.count += 1;
return true;
}
@ -168,12 +146,9 @@ LiquidDict.prototype.add = function(word) {
/******************************************************************************/
LiquidDict.prototype.freeze = function() {
var buckets = this.dict;
var bucket;
for ( var key in buckets ) {
bucket = buckets[key];
if ( typeof bucket === 'object' ) {
buckets[key] = freezeBucket(this, bucket);
for ( let entry of this.dict ) {
if ( typeof entry[1] === 'object' ) {
this.dict.set(entry[0], freezeBucket(this, entry[1]));
}
}
};
@ -181,15 +156,38 @@ LiquidDict.prototype.freeze = function() {
/******************************************************************************/
LiquidDict.prototype.reset = function() {
this.dict = {};
this.dict.clear();
this.count = 0;
this.duplicateCount = 0;
this.bucketCount = 0;
this.frozenBucketCount = 0;
};
/******************************************************************************/
let selfieVersion = 1;
LiquidDict.prototype.toSelfie = function() {
this.freeze();
return {
version: selfieVersion,
count: this.count,
duplicateCount: this.duplicateCount,
frozenBucketCount: this.frozenBucketCount,
dict: Array.from(this.dict)
};
};
LiquidDict.prototype.fromSelfie = function(selfie) {
if ( selfie.version !== selfieVersion ) { return false; }
this.count = selfie.count;
this.duplicateCount = selfie.duplicateCount;
this.frozenBucketCount = selfie.frozenBucketCount;
this.dict = new Map(selfie.dict);
return true;
};
/******************************************************************************/
return LiquidDict;
/******************************************************************************/
@ -199,4 +197,3 @@ return LiquidDict;
/******************************************************************************/
µMatrix.ubiquitousBlacklist = new µMatrix.LiquidDict();
µMatrix.ubiquitousWhitelist = new µMatrix.LiquidDict();

129
src/js/storage.js
View File

@ -439,9 +439,12 @@
callback = this.noopFunc;
}
var loadHostsFilesEnd = function() {
var loadHostsFilesEnd = function(fromSelfie) {
if ( fromSelfie !== true ) {
µm.ubiquitousBlacklist.freeze();
vAPI.storage.set({ liveHostsFiles: Array.from(µm.liveHostsFiles) });
µm.hostsFilesSelfie.create();
}
vAPI.messaging.broadcast({ what: 'loadHostsFilesCompleted' });
µm.getBytesInUse();
callback();
@ -472,7 +475,14 @@
}
};
this.getAvailableHostsFiles(loadHostsFilesStart);
var onSelfieReady = function(status) {
if ( status === true ) {
return loadHostsFilesEnd(true);
}
µm.getAvailableHostsFiles(loadHostsFilesStart);
};
this.hostsFilesSelfie.load(onSelfieReady);
};
/******************************************************************************/
@ -647,6 +657,9 @@
'selectedHostsFiles',
'externalHostsFiles'
);
if ( hostsChanged ) {
µm.hostsFilesSelfie.destroy();
}
let recipesChanged = applyAssetSelection(
metadata,
details.recipes,
@ -657,7 +670,6 @@
µm.recipeManager.reset();
µm.loadRecipes(true);
}
if ( typeof callback === 'function' ) {
callback({
hostsChanged: hostsChanged,
@ -680,7 +692,50 @@
/******************************************************************************/
µMatrix.hostsFilesSelfie = (function() {
let timer;
return {
create: function() {
this.cancel();
timer = vAPI.setTimeout(
function() {
timer = undefined;
vAPI.cacheStorage.set({
hostsFilesSelfie: µMatrix.ubiquitousBlacklist.toSelfie()
});
},
120000
);
},
destroy: function() {
this.cancel();
vAPI.cacheStorage.remove('hostsFilesSelfie');
},
load: function(callback) {
this.cancel();
vAPI.cacheStorage.get('hostsFilesSelfie', function(bin) {
callback(
bin instanceof Object &&
bin.hostsFilesSelfie instanceof Object &&
µMatrix.ubiquitousBlacklist.fromSelfie(bin.hostsFilesSelfie)
);
});
},
cancel: function() {
if ( timer !== undefined ) {
clearTimeout(timer);
}
timer = undefined;
}
};
})();
/******************************************************************************/
µMatrix.loadPublicSuffixList = function(callback) {
let µm = this;
if ( typeof callback !== 'function' ) {
callback = this.noopFunc;
}
@ -688,15 +743,64 @@
var applyPublicSuffixList = function(details) {
if ( !details.error ) {
publicSuffixList.parse(details.content, punycode.toASCII);
µm.publicSuffixListSelfie.create();
}
callback();
};
this.assets.get(this.pslAssetKey, applyPublicSuffixList);
let onSelfieReady = function(status) {
if ( status === true ) {
return callback();
}
µm.assets.get(µm.pslAssetKey, applyPublicSuffixList);
};
this.publicSuffixListSelfie.load(onSelfieReady);
};
/******************************************************************************/
µMatrix.publicSuffixListSelfie = (function() {
let timer;
return {
create: function() {
this.cancel();
timer = vAPI.setTimeout(
function() {
timer = undefined;
vAPI.cacheStorage.set({
publicSuffixListSelfie: publicSuffixList.toSelfie()
});
},
60000
);
},
destroy: function() {
this.cancel();
vAPI.cacheStorage.remove('publicSuffixListSelfie');
},
load: function(callback) {
this.cancel();
vAPI.cacheStorage.get('publicSuffixListSelfie', function(bin) {
callback(
bin instanceof Object &&
bin.publicSuffixListSelfie instanceof Object &&
publicSuffixList.fromSelfie(bin.publicSuffixListSelfie)
);
});
},
cancel: function() {
if ( timer !== undefined ) {
clearTimeout(timer);
}
timer = undefined;
}
};
})();
/******************************************************************************/
µMatrix.scheduleAssetUpdater = (function() {
var timer, next = 0;
return function(updateDelay) {
@ -727,9 +831,10 @@
/******************************************************************************/
µMatrix.assetObserver = function(topic, details) {
let µmus = this.userSettings;
// Do not update filter list if not in use.
if ( topic === 'before-asset-updated' ) {
let µmus = this.userSettings;
if (
details.type === 'internal' ||
details.type === 'filters' &&
@ -743,6 +848,14 @@
}
if ( topic === 'after-asset-updated' ) {
if (
details.type === 'filters' &&
µmus.selectedHostsFiles.indexOf(details.assetKey) !== -1
) {
this.hostsFilesSelfie.destroy();
} else if ( details.assetKey === this.pslAssetKey ) {
this.publicSuffixListSelfie.destroy();
}
vAPI.messaging.broadcast({
what: 'assetUpdated',
key: details.assetKey,
@ -766,7 +879,7 @@
if (
this.arraysIntersect(
details.assetKeys,
this.userSettings.selectedRecipeFiles
µmus.selectedRecipeFiles
)
) {
this.loadRecipes(true);
@ -774,12 +887,12 @@
if (
this.arraysIntersect(
details.assetKeys,
this.userSettings.selectedHostsFiles
µmus.selectedHostsFiles
)
) {
this.loadHostsFiles();
}
if ( this.userSettings.autoUpdate ) {
if ( µmus.autoUpdate ) {
this.scheduleAssetUpdater(25200000);
} else {
this.scheduleAssetUpdater(0);

145
src/lib/publicsuffixlist.js
View File

@ -37,9 +37,8 @@
/******************************************************************************/
var exceptions = {};
var rules = {};
var selfieMagic = 'iscjsfsaolnm';
var exceptions = new Map();
var rules = new Map();
// This value dictate how the search will be performed:
// < this.cutoffLength = indexOf()
@ -92,9 +91,8 @@ function getPublicSuffix(hostname) {
}
// Since we slice down the hostname with each pass, the first match
// is the longest, so no need to find all the matching rules.
var pos;
while ( true ) {
pos = hostname.indexOf('.');
let pos = hostname.indexOf('.');
if ( pos < 0 ) {
return hostname;
}
@ -118,17 +116,17 @@ function getPublicSuffix(hostname) {
function search(store, hostname) {
// Extract TLD
var pos = hostname.lastIndexOf('.');
var tld, remainder;
if ( pos < 0 ) {
let tld, remainder;
let pos = hostname.lastIndexOf('.');
if ( pos === -1 ) {
tld = hostname;
remainder = hostname;
} else {
tld = hostname.slice(pos + 1);
remainder = hostname.slice(0, pos);
}
var substore = store[tld];
if ( !substore ) {
let substore = store.get(tld);
if ( substore === undefined ) {
return false;
}
// If substore is a string, use indexOf()
@ -136,17 +134,19 @@ function search(store, hostname) {
return substore.indexOf(' ' + remainder + ' ') >= 0;
}
// It is an array: use binary search.
var l = remainder.length;
var haystack = substore[l];
if ( !haystack ) {
let l = remainder.length;
if ( l >= substore.length ) {
return false;
}
let haystack = substore[l];
if ( haystack === null ) {
return false;
}
var left = 0;
var right = Math.floor(haystack.length / l + 0.5);
var i, needle;
let left = 0;
let right = Math.floor(haystack.length / l + 0.5);
while ( left < right ) {
i = left + right >> 1;
needle = haystack.substr( l * i, l );
let i = left + right >> 1;
let needle = haystack.substr(l*i, l);
if ( remainder < needle ) {
right = i;
} else if ( remainder > needle ) {
@ -168,22 +168,21 @@ function search(store, hostname) {
// Suggestion: use <https://github.com/bestiejs/punycode.js> it's quite good.
function parse(text, toAscii) {
exceptions = {};
rules = {};
exceptions = new Map();
rules = new Map();
var lineBeg = 0, lineEnd;
var textEnd = text.length;
var line, store, pos, tld;
let lineBeg = 0;
let textEnd = text.length;
while ( lineBeg < textEnd ) {
lineEnd = text.indexOf('\n', lineBeg);
let lineEnd = text.indexOf('\n', lineBeg);
if ( lineEnd < 0 ) {
lineEnd = text.indexOf('\r', lineBeg);
if ( lineEnd < 0 ) {
lineEnd = textEnd;
}
}
line = text.slice(lineBeg, lineEnd).trim();
let line = text.slice(lineBeg, lineEnd).trim();
lineBeg = lineEnd + 1;
if ( line.length === 0 ) {
@ -191,18 +190,19 @@ function parse(text, toAscii) {
}
// Ignore comments
pos = line.indexOf('//');
if ( pos >= 0 ) {
let pos = line.indexOf('//');
if ( pos !== -1 ) {
line = line.slice(0, pos);
}
// Ignore surrounding whitespaces
line = line.trim();
if ( !line ) {
if ( line.length === 0 ) {
continue;
}
// Is this an exception rule?
let store;
if ( line.charAt(0) === '!' ) {
store = exceptions;
line = line.slice(1);
@ -220,8 +220,9 @@ function parse(text, toAscii) {
line = line.toLowerCase();
// Extract TLD
let tld;
pos = line.lastIndexOf('.');
if ( pos < 0 ) {
if ( pos === -1 ) {
tld = line;
} else {
tld = line.slice(pos + 1);
@ -229,13 +230,15 @@ function parse(text, toAscii) {
}
// Store suffix using tld as key
if ( !store.hasOwnProperty(tld) ) {
store[tld] = [];
let substore = store.get(tld);
if ( substore === undefined ) {
store.set(tld, substore = []);
}
if ( line ) {
store[tld].push(line);
substore.push(line);
}
}
crystallize(exceptions);
crystallize(rules);
@ -248,69 +251,81 @@ function parse(text, toAscii) {
// for future look up.
function crystallize(store) {
var suffixes, suffix, i, l;
for ( let entry of store ) {
let tld = entry[0];
let suffixes = entry[1];
for ( var tld in store ) {
if ( !store.hasOwnProperty(tld) ) {
continue;
}
suffixes = store[tld].join(' ');
// No suffix
if ( !suffixes ) {
store[tld] = '';
if ( suffixes.length === 0 ) {
store.set(tld, '');
continue;
}
// Concatenated list of suffixes less than cutoff length:
// Store as string, lookup using indexOf()
if ( suffixes.length < cutoffLength ) {
store[tld] = ' ' + suffixes + ' ';
let s = suffixes.join(' ');
if ( s.length < cutoffLength ) {
store.set(tld, ' ' + s + ' ');
continue;
}
// Concatenated list of suffixes greater or equal to cutoff length
// Store as array keyed on suffix length, lookup using binary search.
// I borrowed the idea to key on string length here:
// http://ejohn.org/blog/dictionary-lookups-in-javascript/#comment-392072
i = store[tld].length;
suffixes = [];
while ( i-- ) {
suffix = store[tld][i];
l = suffix.length;
if ( !suffixes[l] ) {
suffixes[l] = [];
let buckets = [];
for ( let suffix of suffixes ) {
let l = suffix.length;
if ( buckets.length <= l ) {
extendArray(buckets, l);
}
if ( buckets[l] === null ) {
buckets[l] = [];
}
suffixes[l].push(suffix);
buckets[l].push(suffix);
}
l = suffixes.length;
while ( l-- ) {
if ( suffixes[l] ) {
suffixes[l] = suffixes[l].sort().join('');
for ( let i = 0; i < buckets.length; i++ ) {
let bucket = buckets[i];
if ( bucket !== null ) {
buckets[i] = bucket.sort().join('');
}
}
store[tld] = suffixes;
store.set(tld, buckets);
}
return store;
}
let extendArray = function(aa, rb) {
for ( let i = aa.length; i <= rb; i++ ) {
aa.push(null);
}
};
/******************************************************************************/
function toSelfie() {
let selfieMagic = 3;
let toSelfie = function() {
return {
magic: selfieMagic,
rules: rules,
exceptions: exceptions
rules: Array.from(rules),
exceptions: Array.from(exceptions)
};
}
};
function fromSelfie(selfie) {
if ( typeof selfie !== 'object' || typeof selfie.magic !== 'string' || selfie.magic !== selfieMagic ) {
let fromSelfie = function(selfie) {
if (
selfie instanceof Object === false ||
selfie.magic !== selfieMagic
) {
return false;
}
rules = selfie.rules;
exceptions = selfie.exceptions;
rules = new Map(selfie.rules);
exceptions = new Map(selfie.exceptions);
callListeners(onChangedListeners);
return true;
}
};
/******************************************************************************/

Write Preview
Loading…
Cancel
Save