Mirror
/
nuTensor
mirror of https://github.com/geekprojects/nuTensor.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity
Browse Source

code review: simplify code for counting distinct URLs

pull/2/head
gorhill 7 years ago
parent
67e7c8a5bf
commit
ebac131247
No known key found for this signature in database GPG Key ID: 25E1490B761470C2
6 changed files with 159 additions and 403 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 46
      src/js/messaging.js
  2. 483
      src/js/pagestats.js
  3. 28
      src/js/popup.js
  4. 2
      src/js/start.js
  5. 2
      src/js/tab.js
  6. 1
      src/js/traffic.js

46
src/js/messaging.js
View File

@ -211,16 +211,12 @@ var matrixSnapshot = function(pageStore, details) {
var desHostname;
var row, typeIndex;
var anyIndex = headerIndices.get('*');
var pos, count;
var pageRequests = pageStore.requests;
var reqKeys = pageRequests.getRequestKeys();
var iReqKey = reqKeys.length;
var pos;
while ( iReqKey-- ) {
reqKey = reqKeys[iReqKey];
reqType = pageRequests.typeFromRequestKey(reqKey);
reqHostname = pageRequests.hostnameFromRequestKey(reqKey);
for ( var entry of pageStore.requests.hostnameTypeCells ) {
pos = entry[0].indexOf(' ');
reqHostname = entry[0].slice(0, pos);
reqType = entry[0].slice(pos + 1);
// rhill 2013-10-23: hostname can be empty if the request is a data url
// https://github.com/gorhill/httpswitchboard/issues/26
if ( reqHostname === '' ) {
@ -230,39 +226,31 @@ var matrixSnapshot = function(pageStore, details) {
// We want rows of self and ancestors
desHostname = reqHostname;
for ( ;; ) {
for (;;) {
// If row exists, ancestors exist
if ( r.rows.hasOwnProperty(desHostname) !== false ) {
break;
}
if ( r.rows.hasOwnProperty(desHostname) !== false ) { break; }
r.rows[desHostname] = new RowSnapshot(r.scope, desHostname, reqDomain);
r.rowCount += 1;
if ( desHostname === reqDomain ) {
break;
}
if ( desHostname === reqDomain ) { break; }
pos = desHostname.indexOf('.');
if ( pos === -1 ) {
break;
}
if ( pos === -1 ) { break; }
desHostname = desHostname.slice(pos + 1);
}
count = entry[1].size;
typeIndex = headerIndices.get(reqType);
row = r.rows[reqHostname];
row.counts[typeIndex] += 1;
row.counts[anyIndex] += 1;
row.counts[typeIndex] += count;
row.counts[anyIndex] += count;
row = r.rows[reqDomain];
row.totals[typeIndex] += 1;
row.totals[anyIndex] += 1;
row.totals[typeIndex] += count;
row.totals[anyIndex] += count;
row = r.rows['*'];
row.totals[typeIndex] += 1;
row.totals[anyIndex] += 1;
row.totals[typeIndex] += count;
row.totals[anyIndex] += count;
}
r.diff = µm.tMatrix.diff(µm.pMatrix, r.hostname, Object.keys(r.rows));
r.diff = µm.tMatrix.diff(µm.pMatrix, r.hostname, r.rowCount + 1);
return r;
};

483
src/js/pagestats.js
View File

@ -25,393 +25,162 @@
/*******************************************************************************
A PageRequestStore object is used to store net requests in two ways:
A PageRequestStats object is used to store distinct network requests.
This is used to:
To record distinct net requests
- remember which hostname/type were seen
- count the number of distinct URLs for any given hostname-type pair
**/
µMatrix.PageRequestStats = (function() {
µMatrix.pageRequestStatsFactory = (function() {
/******************************************************************************/
// Caching useful global vars
var µm = µMatrix;
var µmuri = null;
/******************************************************************************/
// Hidden vars
var typeToCode = {
'doc' : 'a',
'frame' : 'b',
'css' : 'c',
'script': 'd',
'image' : 'e',
'media' : 'f',
'xhr' : 'g',
'other' : 'h',
'cookie': 'i'
};
var codeToType = {
'a': 'doc',
'b': 'frame',
'c': 'css',
'd': 'script',
'e': 'image',
'f': 'media',
'g': 'xhr',
'h': 'other',
'i': 'cookie'
};
var µm = µMatrix;
var µmuri;
var pageRequestStoreJunkyard = [];
/******************************************************************************/
// It's just a dict-based "packer"
var stringPacker = {
codeGenerator: 1,
codeJunkyard: [],
mapStringToEntry: {},
mapCodeToString: {},
Entry: function(code) {
this.count = 0;
this.code = code;
},
remember: function(code) {
if ( code === '' ) {
return;
}
var s = this.mapCodeToString[code];
if ( s ) {
var entry = this.mapStringToEntry[s];
entry.count++;
}
},
// Ref: Given a URL, returns a (somewhat) unique 32-bit value
// Based on: FNV32a
// http://www.isthe.com/chongo/tech/comp/fnv/index.html#FNV-reference-source
// The rest is custom, suited for µMatrix.
forget: function(code) {
if ( code === '' ) {
return;
var uidFromURL = function(uri) {
var hint = 0x811c9dc5;
var i = uri.length;
while ( i-- ) {
hint ^= uri.charCodeAt(i) | 0;
hint += (hint<<1) + (hint<<4) + (hint<<7) + (hint<<8) + (hint<<24) | 0;
hint >>>= 0;
}
var s = this.mapCodeToString[code];
if ( s ) {
var entry = this.mapStringToEntry[s];
entry.count--;
if ( !entry.count ) {
// console.debug('stringPacker > releasing code "%s" (aka "%s")', code, s);
this.codeJunkyard.push(entry);
delete this.mapCodeToString[code];
delete this.mapStringToEntry[s];
return hint;
};
var PageRequestStats = function() {
this.hostnameTypeCells = new Map();
};
PageRequestStats.prototype = {
dispose: function() {
this.hostnameTypeCells.clear();
if ( pageRequestStoreJunkyard.length < 8 ) {
pageRequestStoreJunkyard.push(this);
}
}
},
pack: function(s) {
var entry = this.entryFromString(s);
if ( !entry ) {
return '';
}
return entry.code;
},
unpack: function(packed) {
return this.mapCodeToString[packed] || '';
},
stringify: function(code) {
if ( code <= 0xFFFF ) {
return String.fromCharCode(code);
}
return String.fromCharCode(code >>> 16) + String.fromCharCode(code & 0xFFFF);
},
entryFromString: function(s) {
if ( s === '' ) {
return null;
}
var entry = this.mapStringToEntry[s];
if ( !entry ) {
entry = this.codeJunkyard.pop();
if ( !entry ) {
entry = new this.Entry(this.stringify(this.codeGenerator++));
},
createEntryIfNotExists: function(url, type) {
var hn = µmuri.hostnameFromURI(url),
key = hn + ' ' + type,
uids = this.hostnameTypeCells.get(key);
if ( uids === undefined ) {
this.hostnameTypeCells.set(key, (uids = new Set()));
} else {
// console.debug('stringPacker > recycling code "%s" (aka "%s")', entry.code, s);
entry.count = 0;
if ( uids.size > 99 ) { return false; }
}
this.mapStringToEntry[s] = entry;
this.mapCodeToString[entry.code] = s;
var uid = uidFromURL(url);
if ( uids.has(uid) ) { return false; }
uids.add(uid);
return true;
}
return entry;
}
};
/******************************************************************************/
var PageRequestStats = function() {
this.requests = {};
if ( !µmuri ) {
µmuri = µm.URI;
}
};
/******************************************************************************/
PageRequestStats.prototype.init = function() {
return this;
};
/******************************************************************************/
var pageRequestStoreJunkyard = [];
var pageRequestStoreFactory = function() {
var pageRequestStore = pageRequestStoreJunkyard.pop();
if ( pageRequestStore ) {
pageRequestStore.init();
} else {
pageRequestStore = new PageRequestStats();
}
return pageRequestStore;
};
/******************************************************************************/
PageRequestStats.prototype.disposeOne = function(reqKey) {
if ( this.requests[reqKey] ) {
delete this.requests[reqKey];
forgetRequestKey(reqKey);
}
};
/******************************************************************************/
};
PageRequestStats.prototype.dispose = function() {
var requests = this.requests;
for ( var reqKey in requests ) {
if ( requests.hasOwnProperty(reqKey) === false ) {
continue;
return function pageRequestStatsFactory() {
if ( pageRequestStoreJunkyard.length !== 0 ) {
return pageRequestStoreJunkyard.pop();
}
stringPacker.forget(reqKey.slice(3));
}
this.requests = {};
if ( pageRequestStoreJunkyard.length < 8 ) {
pageRequestStoreJunkyard.push(this);
}
};
/******************************************************************************/
// Request key:
// index: 0123
// THHN
// ^^ ^
// || |
// || +--- short string code for hostname (dict-based)
// |+--- FNV32a hash of whole URI (irreversible)
// +--- single char code for type of request
var makeRequestKey = function(uri, reqType) {
// Ref: Given a URL, returns a unique 4-character long hash string
// Based on: FNV32a
// http://www.isthe.com/chongo/tech/comp/fnv/index.html#FNV-reference-source
// The rest is custom, suited for µMatrix.
var hint = 0x811c9dc5;
var i = uri.length;
while ( i-- ) {
hint ^= uri.charCodeAt(i) | 0;
hint += (hint<<1) + (hint<<4) + (hint<<7) + (hint<<8) + (hint<<24) | 0;
hint >>>= 0;
}
var key = typeToCode[reqType] || 'z';
return key +
String.fromCharCode(hint >>> 22, hint >>> 12 & 0x3FF, hint & 0xFFF) +
stringPacker.pack(µmuri.hostnameFromURI(uri));
};
/******************************************************************************/
var rememberRequestKey = function(reqKey) {
stringPacker.remember(reqKey.slice(4));
};
var forgetRequestKey = function(reqKey) {
stringPacker.forget(reqKey.slice(4));
};
/******************************************************************************/
// Exported
var hostnameFromRequestKey = function(reqKey) {
return stringPacker.unpack(reqKey.slice(4));
};
PageRequestStats.prototype.hostnameFromRequestKey = hostnameFromRequestKey;
var typeFromRequestKey = function(reqKey) {
return codeToType[reqKey.charAt(0)];
};
PageRequestStats.prototype.typeFromRequestKey = typeFromRequestKey;
/******************************************************************************/
PageRequestStats.prototype.createEntryIfNotExists = function(url, type) {
var reqKey = makeRequestKey(url, type);
if ( this.requests[reqKey] ) {
return false;
}
rememberRequestKey(reqKey);
this.requests[reqKey] = Date.now();
return true;
};
/******************************************************************************/
PageRequestStats.prototype.getRequestKeys = function() {
return Object.keys(this.requests);
};
/******************************************************************************/
PageRequestStats.prototype.getRequestDict = function() {
return this.requests;
};
/******************************************************************************/
// Export
return {
factory: pageRequestStoreFactory,
hostnameFromRequestKey: hostnameFromRequestKey,
typeFromRequestKey: typeFromRequestKey
};
/******************************************************************************/
if ( µmuri === undefined ) { µmuri = µm.URI; }
return new PageRequestStats();
};
})();
/******************************************************************************/
/******************************************************************************/
µMatrix.PageStore = (function() {
µMatrix.pageStoreFactory = (function() {
/******************************************************************************/
var µm = µMatrix;
var pageStoreJunkyard = [];
/******************************************************************************/
var µm = µMatrix;
var pageStoreJunkyard = [];
var pageStoreFactory = function(tabContext) {
var entry = pageStoreJunkyard.pop();
if ( entry ) {
return entry.init(tabContext);
function PageStore(tabContext) {
this.requestStats = µm.requestStatsFactory();
this.off = false;
this.init(tabContext);
}
return new PageStore(tabContext);
};
/******************************************************************************/
function PageStore(tabContext) {
this.requestStats = µm.requestStatsFactory();
this.off = false;
this.init(tabContext);
}
/******************************************************************************/
PageStore.prototype.init = function(tabContext) {
this.tabId = tabContext.tabId;
this.rawUrl = tabContext.rawURL;
this.pageUrl = tabContext.normalURL;
this.pageHostname = tabContext.rootHostname;
this.pageDomain = tabContext.rootDomain;
this.title = '';
this.requests = µm.PageRequestStats.factory();
this.domains = {};
this.allHostnamesString = ' ';
this.requestStats.reset();
this.distinctRequestCount = 0;
this.perLoadAllowedRequestCount = 0;
this.perLoadBlockedRequestCount = 0;
this.incinerationTimer = null;
this.mtxContentModifiedTime = 0;
this.mtxCountModifiedTime = 0;
return this;
};
/******************************************************************************/
PageStore.prototype.dispose = function() {
this.requests.dispose();
this.rawUrl = '';
this.pageUrl = '';
this.pageHostname = '';
this.pageDomain = '';
this.title = '';
this.domains = {};
this.allHostnamesString = ' ';
if ( this.incinerationTimer !== null ) {
clearTimeout(this.incinerationTimer);
this.incinerationTimer = null;
}
if ( pageStoreJunkyard.length < 8 ) {
pageStoreJunkyard.push(this);
}
};
/******************************************************************************/
PageStore.prototype.recordRequest = function(type, url, block) {
if ( !this.requests.createEntryIfNotExists(url, type, block) ) {
return;
}
// Count blocked/allowed requests
this.requestStats.record(type, block);
// https://github.com/gorhill/httpswitchboard/issues/306
// If it is recorded locally, record globally
µm.requestStats.record(type, block);
µm.updateBadgeAsync(this.tabId);
if ( block !== false ) {
this.perLoadBlockedRequestCount++;
} else {
this.perLoadAllowedRequestCount++;
}
var hostname = µm.URI.hostnameFromURI(url);
PageStore.prototype = {
init: function(tabContext) {
this.tabId = tabContext.tabId;
this.rawUrl = tabContext.rawURL;
this.pageUrl = tabContext.normalURL;
this.pageHostname = tabContext.rootHostname;
this.pageDomain = tabContext.rootDomain;
this.title = '';
this.requests = µm.pageRequestStatsFactory();
this.domains = {};
this.allHostnamesString = ' ';
this.requestStats.reset();
this.distinctRequestCount = 0;
this.perLoadAllowedRequestCount = 0;
this.perLoadBlockedRequestCount = 0;
this.incinerationTimer = null;
this.mtxContentModifiedTime = 0;
this.mtxCountModifiedTime = 0;
return this;
},
dispose: function() {
this.requests.dispose();
this.rawUrl = '';
this.pageUrl = '';
this.pageHostname = '';
this.pageDomain = '';
this.title = '';
this.domains = {};
this.allHostnamesString = ' ';
if ( this.incinerationTimer !== null ) {
clearTimeout(this.incinerationTimer);
this.incinerationTimer = null;
}
if ( pageStoreJunkyard.length < 8 ) {
pageStoreJunkyard.push(this);
}
},
recordRequest: function(type, url, block) {
if ( this.requests.createEntryIfNotExists(url, type) === false ) {
return;
}
this.distinctRequestCount++;
this.mtxCountModifiedTime = Date.now();
// Count blocked/allowed requests
this.requestStats.record(type, block);
if ( this.domains.hasOwnProperty(hostname) === false ) {
this.domains[hostname] = true;
this.allHostnamesString += hostname + ' ';
this.mtxContentModifiedTime = Date.now();
}
// https://github.com/gorhill/httpswitchboard/issues/306
// If it is recorded locally, record globally
µm.requestStats.record(type, block);
µm.updateBadgeAsync(this.tabId);
// console.debug("pagestats.js > PageStore.recordRequest(): %o: %s @ %s", this, type, url);
};
if ( block !== false ) {
this.perLoadBlockedRequestCount++;
} else {
this.perLoadAllowedRequestCount++;
}
/******************************************************************************/
var hostname = µm.URI.hostnameFromURI(url);
return {
factory: pageStoreFactory
};
this.distinctRequestCount++;
this.mtxCountModifiedTime = Date.now();
/******************************************************************************/
if ( this.domains.hasOwnProperty(hostname) === false ) {
this.domains[hostname] = true;
this.allHostnamesString += hostname + ' ';
this.mtxContentModifiedTime = Date.now();
}
}
};
return function pageStoreFactory(tabContext) {
var entry = pageStoreJunkyard.pop();
if ( entry ) {
return entry.init(tabContext);
}
return new PageStore(tabContext);
};
})();
/******************************************************************************/

28
src/js/popup.js
View File

@ -437,10 +437,16 @@ function updateMatrixCounts() {
count = rows[expandos.hostname][counts][headerIndices.get(expandos.reqType)];
if ( count === expandos.count ) { continue; }
expandos.count = count;
matCell.textContent = count ? count : '\u00A0';
matCell.textContent = cellTextFromCount(count);
}
}
function cellTextFromCount(count) {
if ( count === 0 ) { return '\u00A0'; }
if ( count < 100 ) { return count; }
return '99+';
}
/******************************************************************************/
// Update color of matrix cells(s)
@ -686,16 +692,13 @@ function renderMatrixMetaCellDomain(cell, domain) {
}
function renderMatrixCellType(cell, hostname, type, count) {
var expandos = expandosFromNode(cell);
var node = cell.nodeAt(0),
expandos = expandosFromNode(node);
expandos.hostname = hostname;
expandos.reqType = type;
expandos.count = count;
addCellClass(cell.nodeAt(0), hostname, type);
if ( count ) {
cell.text(count);
} else {
cell.text('\u00A0');
}
addCellClass(node, hostname, type);
node.textContent = cellTextFromCount(count);
}
function renderMatrixCellTypes(cells, hostname, countName) {
@ -742,16 +745,13 @@ function makeMatrixMetaRowDomain(domain) {
function renderMatrixMetaCellType(cell, count) {
// https://github.com/gorhill/uMatrix/issues/24
// Don't forget to reset cell properties
var expandos = expandosFromNode(cell);
var node = cell.nodeAt(0),
expandos = expandosFromNode(node);
expandos.hostname = '';
expandos.reqType = '';
expandos.count = count;
cell.addClass('t1');
if ( count ) {
cell.text(count);
} else {
cell.text('\u00A0');
}
node.textContent = cellTextFromCount(count);
}
function makeMatrixMetaRow(totals) {

2
src/js/start.js
View File

@ -187,7 +187,7 @@ var onPSLReady = function() {
// rhill 2013-11-24: bind behind-the-scene virtual tab/url manually, since the
// normal way forbid binding behind the scene tab.
// https://github.com/gorhill/httpswitchboard/issues/67
µm.pageStores[vAPI.noTabId] = µm.PageStore.factory(µm.tabContextManager.mustLookup(vAPI.noTabId));
µm.pageStores[vAPI.noTabId] = µm.pageStoreFactory(µm.tabContextManager.mustLookup(vAPI.noTabId));
µm.pageStores[vAPI.noTabId].title = vAPI.i18n('statsPageDetailedBehindTheScenePage');
vAPI.tabs.getAll(onTabsReady);

2
src/js/tab.js
View File

@ -445,7 +445,7 @@ vAPI.tabs.registerListeners();
// Try to resurrect first.
pageStore = this.resurrectPageStore(tabId, normalURL);
if ( pageStore === null ) {
pageStore = this.PageStore.factory(tabContext);
pageStore = this.pageStoreFactory(tabContext);
}
this.pageStores[tabId] = pageStore;
this.updateTitle(tabId);

1
src/js/traffic.js
View File

@ -374,7 +374,6 @@ var requestTypeNormalizer = {
'media' : 'media',
'object' : 'media',
'other' : 'other',
'ping' : 'ping',
'script' : 'script',
'stylesheet' : 'css',
'sub_frame' : 'frame',

Write Preview
Loading…
Cancel
Save