Mirror
/
nuTensor
mirror of https://github.com/geekprojects/nuTensor.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity
Browse Source

code review: simplify code for counting distinct URLs

pull/2/head
gorhill 7 years ago
parent
67e7c8a5bf
commit
ebac131247
No known key found for this signature in database GPG Key ID: 25E1490B761470C2
6 changed files with 159 additions and 403 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Unified View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 46
      src/js/messaging.js
  2. 361
      src/js/pagestats.js
  3. 28
      src/js/popup.js
  4. 2
      src/js/start.js
  5. 2
      src/js/tab.js
  6. 1
      src/js/traffic.js

46
src/js/messaging.js
View File

@ -211,16 +211,12 @@ var matrixSnapshot = function(pageStore, details) {
var desHostname; var desHostname;
var row, typeIndex; var row, typeIndex;
var anyIndex = headerIndices.get('*'); var anyIndex = headerIndices.get('*');
var pos, count;
var pageRequests = pageStore.requests;
var reqKeys = pageRequests.getRequestKeys();
var iReqKey = reqKeys.length;
var pos;
while ( iReqKey-- ) {
reqKey = reqKeys[iReqKey];
reqType = pageRequests.typeFromRequestKey(reqKey);
reqHostname = pageRequests.hostnameFromRequestKey(reqKey);
for ( var entry of pageStore.requests.hostnameTypeCells ) {
pos = entry[0].indexOf(' ');
reqHostname = entry[0].slice(0, pos);
reqType = entry[0].slice(pos + 1);
// rhill 2013-10-23: hostname can be empty if the request is a data url // rhill 2013-10-23: hostname can be empty if the request is a data url
// https://github.com/gorhill/httpswitchboard/issues/26 // https://github.com/gorhill/httpswitchboard/issues/26
if ( reqHostname === '' ) { if ( reqHostname === '' ) {
@ -230,39 +226,31 @@ var matrixSnapshot = function(pageStore, details) {
// We want rows of self and ancestors // We want rows of self and ancestors
desHostname = reqHostname; desHostname = reqHostname;
for ( ;; ) {
for (;;) {
// If row exists, ancestors exist // If row exists, ancestors exist
if ( r.rows.hasOwnProperty(desHostname) !== false ) {
break;
}
if ( r.rows.hasOwnProperty(desHostname) !== false ) { break; }
r.rows[desHostname] = new RowSnapshot(r.scope, desHostname, reqDomain); r.rows[desHostname] = new RowSnapshot(r.scope, desHostname, reqDomain);
r.rowCount += 1; r.rowCount += 1;
if ( desHostname === reqDomain ) {
break;
}
if ( desHostname === reqDomain ) { break; }
pos = desHostname.indexOf('.'); pos = desHostname.indexOf('.');
if ( pos === -1 ) {
break;
}
if ( pos === -1 ) { break; }
desHostname = desHostname.slice(pos + 1); desHostname = desHostname.slice(pos + 1);
} }
count = entry[1].size;
typeIndex = headerIndices.get(reqType); typeIndex = headerIndices.get(reqType);
row = r.rows[reqHostname]; row = r.rows[reqHostname];
row.counts[typeIndex] += 1;
row.counts[anyIndex] += 1;
row.counts[typeIndex] += count;
row.counts[anyIndex] += count;
row = r.rows[reqDomain]; row = r.rows[reqDomain];
row.totals[typeIndex] += 1;
row.totals[anyIndex] += 1;
row.totals[typeIndex] += count;
row.totals[anyIndex] += count;
row = r.rows['*']; row = r.rows['*'];
row.totals[typeIndex] += 1;
row.totals[anyIndex] += 1;
row.totals[typeIndex] += count;
row.totals[anyIndex] += count;
} }
r.diff = µm.tMatrix.diff(µm.pMatrix, r.hostname, Object.keys(r.rows));
r.diff = µm.tMatrix.diff(µm.pMatrix, r.hostname, r.rowCount + 1);
return r; return r;
}; };

361
src/js/pagestats.js
View File

@ -25,201 +25,26 @@
/******************************************************************************* /*******************************************************************************
A PageRequestStore object is used to store net requests in two ways:
A PageRequestStats object is used to store distinct network requests.
This is used to:
To record distinct net requests
- remember which hostname/type were seen
- count the number of distinct URLs for any given hostname-type pair
**/ **/
µMatrix.PageRequestStats = (function() {
µMatrix.pageRequestStatsFactory = (function() {
/******************************************************************************/
// Caching useful global vars
var µm = µMatrix;
var µmuri = null;
/******************************************************************************/
// Hidden vars
var typeToCode = {
'doc' : 'a',
'frame' : 'b',
'css' : 'c',
'script': 'd',
'image' : 'e',
'media' : 'f',
'xhr' : 'g',
'other' : 'h',
'cookie': 'i'
};
var codeToType = {
'a': 'doc',
'b': 'frame',
'c': 'css',
'd': 'script',
'e': 'image',
'f': 'media',
'g': 'xhr',
'h': 'other',
'i': 'cookie'
};
/******************************************************************************/
// It's just a dict-based "packer"
var stringPacker = {
codeGenerator: 1,
codeJunkyard: [],
mapStringToEntry: {},
mapCodeToString: {},
Entry: function(code) {
this.count = 0;
this.code = code;
},
var µm = µMatrix;
var µmuri;
var pageRequestStoreJunkyard = [];
remember: function(code) {
if ( code === '' ) {
return;
}
var s = this.mapCodeToString[code];
if ( s ) {
var entry = this.mapStringToEntry[s];
entry.count++;
}
},
forget: function(code) {
if ( code === '' ) {
return;
}
var s = this.mapCodeToString[code];
if ( s ) {
var entry = this.mapStringToEntry[s];
entry.count--;
if ( !entry.count ) {
// console.debug('stringPacker > releasing code "%s" (aka "%s")', code, s);
this.codeJunkyard.push(entry);
delete this.mapCodeToString[code];
delete this.mapStringToEntry[s];
}
}
},
pack: function(s) {
var entry = this.entryFromString(s);
if ( !entry ) {
return '';
}
return entry.code;
},
unpack: function(packed) {
return this.mapCodeToString[packed] || '';
},
stringify: function(code) {
if ( code <= 0xFFFF ) {
return String.fromCharCode(code);
}
return String.fromCharCode(code >>> 16) + String.fromCharCode(code & 0xFFFF);
},
entryFromString: function(s) {
if ( s === '' ) {
return null;
}
var entry = this.mapStringToEntry[s];
if ( !entry ) {
entry = this.codeJunkyard.pop();
if ( !entry ) {
entry = new this.Entry(this.stringify(this.codeGenerator++));
} else {
// console.debug('stringPacker > recycling code "%s" (aka "%s")', entry.code, s);
entry.count = 0;
}
this.mapStringToEntry[s] = entry;
this.mapCodeToString[entry.code] = s;
}
return entry;
}
};
/******************************************************************************/
var PageRequestStats = function() {
this.requests = {};
if ( !µmuri ) {
µmuri = µm.URI;
}
};
/******************************************************************************/
PageRequestStats.prototype.init = function() {
return this;
};
/******************************************************************************/
var pageRequestStoreJunkyard = [];
var pageRequestStoreFactory = function() {
var pageRequestStore = pageRequestStoreJunkyard.pop();
if ( pageRequestStore ) {
pageRequestStore.init();
} else {
pageRequestStore = new PageRequestStats();
}
return pageRequestStore;
};
/******************************************************************************/
PageRequestStats.prototype.disposeOne = function(reqKey) {
if ( this.requests[reqKey] ) {
delete this.requests[reqKey];
forgetRequestKey(reqKey);
}
};
/******************************************************************************/
PageRequestStats.prototype.dispose = function() {
var requests = this.requests;
for ( var reqKey in requests ) {
if ( requests.hasOwnProperty(reqKey) === false ) {
continue;
}
stringPacker.forget(reqKey.slice(3));
}
this.requests = {};
if ( pageRequestStoreJunkyard.length < 8 ) {
pageRequestStoreJunkyard.push(this);
}
};
/******************************************************************************/
// Request key:
// index: 0123
// THHN
// ^^ ^
// || |
// || +--- short string code for hostname (dict-based)
// |+--- FNV32a hash of whole URI (irreversible)
// +--- single char code for type of request
var makeRequestKey = function(uri, reqType) {
// Ref: Given a URL, returns a unique 4-character long hash string
// Ref: Given a URL, returns a (somewhat) unique 32-bit value
// Based on: FNV32a // Based on: FNV32a
// http://www.isthe.com/chongo/tech/comp/fnv/index.html#FNV-reference-source // http://www.isthe.com/chongo/tech/comp/fnv/index.html#FNV-reference-source
// The rest is custom, suited for µMatrix. // The rest is custom, suited for µMatrix.
var uidFromURL = function(uri) {
var hint = 0x811c9dc5; var hint = 0x811c9dc5;
var i = uri.length; var i = uri.length;
while ( i-- ) { while ( i-- ) {
@ -227,114 +52,68 @@ var makeRequestKey = function(uri, reqType) {
hint += (hint<<1) + (hint<<4) + (hint<<7) + (hint<<8) + (hint<<24) | 0; hint += (hint<<1) + (hint<<4) + (hint<<7) + (hint<<8) + (hint<<24) | 0;
hint >>>= 0; hint >>>= 0;
} }
var key = typeToCode[reqType] || 'z';
return key +
String.fromCharCode(hint >>> 22, hint >>> 12 & 0x3FF, hint & 0xFFF) +
stringPacker.pack(µmuri.hostnameFromURI(uri));
};
return hint;
};
/******************************************************************************/
var rememberRequestKey = function(reqKey) {
stringPacker.remember(reqKey.slice(4));
};
var forgetRequestKey = function(reqKey) {
stringPacker.forget(reqKey.slice(4));
};
/******************************************************************************/
// Exported
var hostnameFromRequestKey = function(reqKey) {
return stringPacker.unpack(reqKey.slice(4));
};
PageRequestStats.prototype.hostnameFromRequestKey = hostnameFromRequestKey;
var typeFromRequestKey = function(reqKey) {
return codeToType[reqKey.charAt(0)];
};
var PageRequestStats = function() {
this.hostnameTypeCells = new Map();
};
PageRequestStats.prototype.typeFromRequestKey = typeFromRequestKey;
/******************************************************************************/
PageRequestStats.prototype.createEntryIfNotExists = function(url, type) {
var reqKey = makeRequestKey(url, type);
if ( this.requests[reqKey] ) {
return false;
PageRequestStats.prototype = {
dispose: function() {
this.hostnameTypeCells.clear();
if ( pageRequestStoreJunkyard.length < 8 ) {
pageRequestStoreJunkyard.push(this);
}
},
createEntryIfNotExists: function(url, type) {
var hn = µmuri.hostnameFromURI(url),
key = hn + ' ' + type,
uids = this.hostnameTypeCells.get(key);
if ( uids === undefined ) {
this.hostnameTypeCells.set(key, (uids = new Set()));
} else {
if ( uids.size > 99 ) { return false; }
} }
rememberRequestKey(reqKey);
this.requests[reqKey] = Date.now();
var uid = uidFromURL(url);
if ( uids.has(uid) ) { return false; }
uids.add(uid);
return true; return true;
};
/******************************************************************************/
PageRequestStats.prototype.getRequestKeys = function() {
return Object.keys(this.requests);
};
/******************************************************************************/
PageRequestStats.prototype.getRequestDict = function() {
return this.requests;
};
/******************************************************************************/
// Export
return {
factory: pageRequestStoreFactory,
hostnameFromRequestKey: hostnameFromRequestKey,
typeFromRequestKey: typeFromRequestKey
};
/******************************************************************************/
}
};
return function pageRequestStatsFactory() {
if ( pageRequestStoreJunkyard.length !== 0 ) {
return pageRequestStoreJunkyard.pop();
}
if ( µmuri === undefined ) { µmuri = µm.URI; }
return new PageRequestStats();
};
})(); })();
/******************************************************************************/ /******************************************************************************/
/******************************************************************************/ /******************************************************************************/
µMatrix.PageStore = (function() {
/******************************************************************************/
var µm = µMatrix;
var pageStoreJunkyard = [];
/******************************************************************************/
var pageStoreFactory = function(tabContext) {
var entry = pageStoreJunkyard.pop();
if ( entry ) {
return entry.init(tabContext);
}
return new PageStore(tabContext);
};
µMatrix.pageStoreFactory = (function() {
/******************************************************************************/
var µm = µMatrix;
var pageStoreJunkyard = [];
function PageStore(tabContext) {
function PageStore(tabContext) {
this.requestStats = µm.requestStatsFactory(); this.requestStats = µm.requestStatsFactory();
this.off = false; this.off = false;
this.init(tabContext); this.init(tabContext);
}
/******************************************************************************/
}
PageStore.prototype.init = function(tabContext) {
PageStore.prototype = {
init: function(tabContext) {
this.tabId = tabContext.tabId; this.tabId = tabContext.tabId;
this.rawUrl = tabContext.rawURL; this.rawUrl = tabContext.rawURL;
this.pageUrl = tabContext.normalURL; this.pageUrl = tabContext.normalURL;
this.pageHostname = tabContext.rootHostname; this.pageHostname = tabContext.rootHostname;
this.pageDomain = tabContext.rootDomain; this.pageDomain = tabContext.rootDomain;
this.title = ''; this.title = '';
this.requests = µm.PageRequestStats.factory();
this.requests = µm.pageRequestStatsFactory();
this.domains = {}; this.domains = {};
this.allHostnamesString = ' '; this.allHostnamesString = ' ';
this.requestStats.reset(); this.requestStats.reset();
@ -345,11 +124,8 @@ PageStore.prototype.init = function(tabContext) {
this.mtxContentModifiedTime = 0; this.mtxContentModifiedTime = 0;
this.mtxCountModifiedTime = 0; this.mtxCountModifiedTime = 0;
return this; return this;
};
/******************************************************************************/
PageStore.prototype.dispose = function() {
},
dispose: function() {
this.requests.dispose(); this.requests.dispose();
this.rawUrl = ''; this.rawUrl = '';
this.pageUrl = ''; this.pageUrl = '';
@ -358,21 +134,16 @@ PageStore.prototype.dispose = function() {
this.title = ''; this.title = '';
this.domains = {}; this.domains = {};
this.allHostnamesString = ' '; this.allHostnamesString = ' ';
if ( this.incinerationTimer !== null ) { if ( this.incinerationTimer !== null ) {
clearTimeout(this.incinerationTimer); clearTimeout(this.incinerationTimer);
this.incinerationTimer = null; this.incinerationTimer = null;
} }
if ( pageStoreJunkyard.length < 8 ) { if ( pageStoreJunkyard.length < 8 ) {
pageStoreJunkyard.push(this); pageStoreJunkyard.push(this);
} }
};
/******************************************************************************/
PageStore.prototype.recordRequest = function(type, url, block) {
if ( !this.requests.createEntryIfNotExists(url, type, block) ) {
},
recordRequest: function(type, url, block) {
if ( this.requests.createEntryIfNotExists(url, type) === false ) {
return; return;
} }
@ -400,18 +171,16 @@ PageStore.prototype.recordRequest = function(type, url, block) {
this.allHostnamesString += hostname + ' '; this.allHostnamesString += hostname + ' ';
this.mtxContentModifiedTime = Date.now(); this.mtxContentModifiedTime = Date.now();
} }
}
};
// console.debug("pagestats.js > PageStore.recordRequest(): %o: %s @ %s", this, type, url);
};
/******************************************************************************/
return {
factory: pageStoreFactory
};
/******************************************************************************/
return function pageStoreFactory(tabContext) {
var entry = pageStoreJunkyard.pop();
if ( entry ) {
return entry.init(tabContext);
}
return new PageStore(tabContext);
};
})(); })();
/******************************************************************************/ /******************************************************************************/

28
src/js/popup.js
View File

@ -437,10 +437,16 @@ function updateMatrixCounts() {
count = rows[expandos.hostname][counts][headerIndices.get(expandos.reqType)]; count = rows[expandos.hostname][counts][headerIndices.get(expandos.reqType)];
if ( count === expandos.count ) { continue; } if ( count === expandos.count ) { continue; }
expandos.count = count; expandos.count = count;
matCell.textContent = count ? count : '\u00A0';
matCell.textContent = cellTextFromCount(count);
} }
} }
function cellTextFromCount(count) {
if ( count === 0 ) { return '\u00A0'; }
if ( count < 100 ) { return count; }
return '99+';
}
/******************************************************************************/ /******************************************************************************/
// Update color of matrix cells(s) // Update color of matrix cells(s)
@ -686,16 +692,13 @@ function renderMatrixMetaCellDomain(cell, domain) {
} }
function renderMatrixCellType(cell, hostname, type, count) { function renderMatrixCellType(cell, hostname, type, count) {
var expandos = expandosFromNode(cell);
var node = cell.nodeAt(0),
expandos = expandosFromNode(node);
expandos.hostname = hostname; expandos.hostname = hostname;
expandos.reqType = type; expandos.reqType = type;
expandos.count = count; expandos.count = count;
addCellClass(cell.nodeAt(0), hostname, type);
if ( count ) {
cell.text(count);
} else {
cell.text('\u00A0');
}
addCellClass(node, hostname, type);
node.textContent = cellTextFromCount(count);
} }
function renderMatrixCellTypes(cells, hostname, countName) { function renderMatrixCellTypes(cells, hostname, countName) {
@ -742,16 +745,13 @@ function makeMatrixMetaRowDomain(domain) {
function renderMatrixMetaCellType(cell, count) { function renderMatrixMetaCellType(cell, count) {
// https://github.com/gorhill/uMatrix/issues/24 // https://github.com/gorhill/uMatrix/issues/24
// Don't forget to reset cell properties // Don't forget to reset cell properties
var expandos = expandosFromNode(cell);
var node = cell.nodeAt(0),
expandos = expandosFromNode(node);
expandos.hostname = ''; expandos.hostname = '';
expandos.reqType = ''; expandos.reqType = '';
expandos.count = count; expandos.count = count;
cell.addClass('t1'); cell.addClass('t1');
if ( count ) {
cell.text(count);
} else {
cell.text('\u00A0');
}
node.textContent = cellTextFromCount(count);
} }
function makeMatrixMetaRow(totals) { function makeMatrixMetaRow(totals) {

2
src/js/start.js
View File

@ -187,7 +187,7 @@ var onPSLReady = function() {
// rhill 2013-11-24: bind behind-the-scene virtual tab/url manually, since the // rhill 2013-11-24: bind behind-the-scene virtual tab/url manually, since the
// normal way forbid binding behind the scene tab. // normal way forbid binding behind the scene tab.
// https://github.com/gorhill/httpswitchboard/issues/67 // https://github.com/gorhill/httpswitchboard/issues/67
µm.pageStores[vAPI.noTabId] = µm.PageStore.factory(µm.tabContextManager.mustLookup(vAPI.noTabId));
µm.pageStores[vAPI.noTabId] = µm.pageStoreFactory(µm.tabContextManager.mustLookup(vAPI.noTabId));
µm.pageStores[vAPI.noTabId].title = vAPI.i18n('statsPageDetailedBehindTheScenePage'); µm.pageStores[vAPI.noTabId].title = vAPI.i18n('statsPageDetailedBehindTheScenePage');
vAPI.tabs.getAll(onTabsReady); vAPI.tabs.getAll(onTabsReady);

2
src/js/tab.js
View File

@ -445,7 +445,7 @@ vAPI.tabs.registerListeners();
// Try to resurrect first. // Try to resurrect first.
pageStore = this.resurrectPageStore(tabId, normalURL); pageStore = this.resurrectPageStore(tabId, normalURL);
if ( pageStore === null ) { if ( pageStore === null ) {
pageStore = this.PageStore.factory(tabContext);
pageStore = this.pageStoreFactory(tabContext);
} }
this.pageStores[tabId] = pageStore; this.pageStores[tabId] = pageStore;
this.updateTitle(tabId); this.updateTitle(tabId);

1
src/js/traffic.js
View File

@ -374,7 +374,6 @@ var requestTypeNormalizer = {
'media' : 'media', 'media' : 'media',
'object' : 'media', 'object' : 'media',
'other' : 'other', 'other' : 'other',
'ping' : 'ping',
'script' : 'script', 'script' : 'script',
'stylesheet' : 'css', 'stylesheet' : 'css',
'sub_frame' : 'frame', 'sub_frame' : 'frame',

Write Preview
Loading…
Cancel
Save