General restructure for performance improvements (bug 441907, r=thunder)

This commit is contained in:
Anant Narayanan 2008-06-27 20:16:43 -07:00
parent d709a5c770
commit 089c1c0f42
9 changed files with 193 additions and 348 deletions

View File

@ -130,13 +130,6 @@ Engine.prototype = {
},
// _core, _store and _tracker need to be overridden in subclasses
__core: null,
get _core() {
if (!this.__core)
this.__core = new SyncCore();
return this.__core;
},
__store: null,
get _store() {
if (!this.__store)
@ -144,6 +137,13 @@ Engine.prototype = {
return this.__store;
},
__core: null,
get _core() {
if (!this.__core)
this.__core = new SyncCore(this._store);
return this.__core;
},
__tracker: null,
get _tracker() {
if (!this.__tracker)
@ -264,6 +264,7 @@ Engine.prototype = {
yield this._remote.keys.getKeyAndIV(self.cb, this.engineId);
// 1) Fetch server deltas
let server = {};
let serverSnap = yield this._remote.wrap(self.cb, this._snapshot);
server.snapshot = serverSnap.data;

View File

@ -628,13 +628,6 @@ BookmarksEngine.prototype = {
get logName() { return "BmkEngine"; },
get serverPrefix() { return "user-data/bookmarks/"; },
__core: null,
get _core() {
if (!this.__core)
this.__core = new BookmarksSyncCore();
return this.__core;
},
__store: null,
get _store() {
if (!this.__store)
@ -642,6 +635,13 @@ BookmarksEngine.prototype = {
return this.__store;
},
__core: null,
get _core() {
if (!this.__core)
this.__core = new BookmarksSyncCore(this._store);
return this.__core;
},
__tracker: null,
get _tracker() {
if (!this.__tracker)
@ -685,23 +685,13 @@ BookmarksEngine.prototype = {
};
BookmarksEngine.prototype.__proto__ = new Engine();
function BookmarksSyncCore() {
function BookmarksSyncCore(store) {
this._store = store;
this._init();
}
BookmarksSyncCore.prototype = {
_logName: "BMSync",
__bms: null,
get _bms() {
if (!this.__bms)
this.__bms = Cc["@mozilla.org/browser/nav-bookmarks-service;1"].
getService(Ci.nsINavBookmarksService);
return this.__bms;
},
_itemExists: function BSC__itemExists(GUID) {
return this._bms.getItemIdForGUID(GUID) >= 0;
},
_store: null,
_getEdits: function BSC__getEdits(a, b) {
// NOTE: we do not increment ret.numProps, as that would cause
@ -788,6 +778,7 @@ function BookmarksStore() {
}
BookmarksStore.prototype = {
_logName: "BStore",
_lookup: null,
__bms: null,
get _bms() {
@ -850,7 +841,7 @@ BookmarksStore.prototype = {
}
return null;
},
_createCommand: function BStore__createCommand(command) {
let newId;
let parentId = this._getItemIdForGUID(command.data.parentGUID);
@ -1252,6 +1243,7 @@ BookmarksStore.prototype = {
this._wrap(this._getNode(this._bms.bookmarksMenuFolder), items, "menu");
this._wrap(this._getNode(this._bms.toolbarFolder), items, "toolbar");
this._wrap(this._getNode(this._bms.unfiledBookmarksFolder), items, "unfiled");
this._lookup = items;
return items;
},

View File

@ -54,13 +54,6 @@ CookieEngine.prototype = {
get logName() { return "CookieEngine"; },
get serverPrefix() { return "user-data/cookies/"; },
__core: null,
get _core() {
if (!this.__core)
this.__core = new CookieSyncCore();
return this.__core;
},
__store: null,
get _store() {
if (!this.__store)
@ -68,6 +61,13 @@ CookieEngine.prototype = {
return this.__store;
},
__core: null,
get _core() {
if (!this.__core)
this.__core = new CookieSyncCore(this._store);
return this.__core;
},
__tracker: null,
get _tracker() {
if (!this.__tracker)
@ -77,61 +77,13 @@ CookieEngine.prototype = {
};
CookieEngine.prototype.__proto__ = new Engine();
function CookieSyncCore() {
function CookieSyncCore(store) {
this._store = store;
this._init();
}
CookieSyncCore.prototype = {
_logName: "CookieSync",
__cookieManager: null,
get _cookieManager() {
if (!this.__cookieManager)
this.__cookieManager = Cc["@mozilla.org/cookiemanager;1"].
getService(Ci.nsICookieManager2);
/* need the 2nd revision of the ICookieManager interface
because it supports add() and the 1st one doesn't. */
return this.__cookieManager;
},
_itemExists: function CSC__itemExists(GUID) {
/* true if a cookie with the given GUID exists.
The GUID that we are passed should correspond to the keys
that we define in the JSON returned by CookieStore.wrap()
That is, it will be a string of the form
"host:path:name". */
/* TODO verify that colons can't normally appear in any of
the fields -- if they did it then we can't rely on .split(":")
to parse correctly.*/
let cookieArray = GUID.split( ":" );
let cookieHost = cookieArray[0];
let cookiePath = cookieArray[1];
let cookieName = cookieArray[2];
/* alternate implementation would be to instantiate a cookie from
cookieHost, cookiePath, and cookieName, then call
cookieManager.cookieExists(). Maybe that would have better
performance? This implementation seems pretty slow.*/
let enumerator = this._cookieManager.enumerator;
while (enumerator.hasMoreElements())
{
let aCookie = enumerator.getNext();
if (aCookie.host == cookieHost &&
aCookie.path == cookiePath &&
aCookie.name == cookieName ) {
return true;
}
}
return false;
/* Note: We can't just call cookieManager.cookieExists() with a generic
javascript object with .host, .path, and .name attributes attatched.
cookieExists is implemented in C and does a hard static_cast to an
nsCookie object, so duck typing doesn't work (and in fact makes
Firefox hard-crash as the static_cast returns null and is not checked.)
*/
},
_store: null,
_commandLike: function CSC_commandLike(a, b) {
/* Method required to be overridden.
@ -155,7 +107,7 @@ function CookieStore( cookieManagerStub ) {
}
CookieStore.prototype = {
_logName: "CookieStore",
_lookup: null,
// Documentation of the nsICookie interface says:
// name ACString The name of the cookie. Read only.
@ -186,7 +138,7 @@ CookieStore.prototype = {
// because it supports add() and the 1st one doesn't.
return this.__cookieManager;
},
_createCommand: function CookieStore__createCommand(command) {
/* we got a command to create a cookie in the local browser
in order to sync with the server. */
@ -275,39 +227,39 @@ CookieStore.prototype = {
/* Return contents of this store, as JSON.
A dictionary of cookies where the keys are GUIDs and the
values are sub-dictionaries containing all cookie fields. */
let items = {};
var iter = this._cookieManager.enumerator;
while (iter.hasMoreElements()){
while (iter.hasMoreElements()) {
var cookie = iter.getNext();
if (cookie.QueryInterface( Ci.nsICookie )){
// String used to identify cookies is
// host:path:name
if ( cookie.isSession ) {
/* Skip session-only cookies, sync only persistent cookies. */
continue;
}
if (cookie.QueryInterface( Ci.nsICookie )) {
// String used to identify cookies is
// host:path:name
if ( cookie.isSession ) {
/* Skip session-only cookies, sync only persistent cookies. */
continue;
}
let key = cookie.host + ":" + cookie.path + ":" + cookie.name;
items[ key ] = { parentGUID: '',
name: cookie.name,
value: cookie.value,
isDomain: cookie.isDomain,
host: cookie.host,
path: cookie.path,
isSecure: cookie.isSecure,
// nsICookie2 values:
rawHost: cookie.rawHost,
isSession: cookie.isSession,
expiry: cookie.expiry,
isHttpOnly: cookie.isHttpOnly };
let key = cookie.host + ":" + cookie.path + ":" + cookie.name;
items[ key ] = { parentGUID: '',
name: cookie.name,
value: cookie.value,
isDomain: cookie.isDomain,
host: cookie.host,
path: cookie.path,
isSecure: cookie.isSecure,
// nsICookie2 values:
rawHost: cookie.rawHost,
isSession: cookie.isSession,
expiry: cookie.expiry,
isHttpOnly: cookie.isHttpOnly };
/* See http://developer.mozilla.org/en/docs/nsICookie
Note: not syncing "expires", "status", or "policy"
since they're deprecated. */
/* See http://developer.mozilla.org/en/docs/nsICookie
Note: not syncing "expires", "status", or "policy"
since they're deprecated. */
}
}
this._lookup = items;
return items;
},

View File

@ -47,58 +47,6 @@ Cu.import("resource://weave/syncCores.js");
Cu.import("resource://weave/stores.js");
Cu.import("resource://weave/trackers.js");
/*
* Generate GUID from a name,value pair.
* If the concatenated length is less than 40, we just Base64 the JSON.
* Otherwise, we Base64 the JSON of the name and SHA1 of the value.
* The first character of the key determines which method we used:
* '0' for full Base64, '1' for SHA-1'ed val.
*/
function _generateFormGUID(nam, val) {
var key;
var con = nam + val;
var jso = Cc["@mozilla.org/dom/json;1"].
createInstance(Ci.nsIJSON);
if (con.length <= 40) {
key = '0' + btoa(jso.encode([nam, val]));
} else {
val = Utils.sha1(val);
key = '1' + btoa(jso.encode([nam, val]));
}
return key;
}
/*
* Unwrap a name,value pair from a GUID.
* Return an array [sha1ed, name, value]
* sha1ed is a boolean determining if the value is SHA-1'ed or not.
*/
function _unwrapFormGUID(guid) {
var jso = Cc["@mozilla.org/dom/json;1"].
createInstance(Ci.nsIJSON);
var ret;
var dec = atob(guid.slice(1));
var obj = jso.decode(dec);
switch (guid[0]) {
case '0':
ret = [false, obj[0], obj[1]];
break;
case '1':
ret = [true, obj[0], obj[1]];
break;
default:
this._log.warn("Unexpected GUID header: " + guid[0] + ", aborting!");
return false;
}
return ret;
}
function FormEngine(pbeId) {
this._init(pbeId);
}
@ -107,13 +55,6 @@ FormEngine.prototype = {
get logName() { return "FormEngine"; },
get serverPrefix() { return "user-data/forms/"; },
__core: null,
get _core() {
if (!this.__core)
this.__core = new FormSyncCore();
return this.__core;
},
__store: null,
get _store() {
if (!this.__store)
@ -121,6 +62,13 @@ FormEngine.prototype = {
return this.__store;
},
__core: null,
get _core() {
if (!this.__core)
this.__core = new FormSyncCore(this._store);
return this.__core;
},
__tracker: null,
get _tracker() {
if (!this.__tracker)
@ -130,43 +78,13 @@ FormEngine.prototype = {
};
FormEngine.prototype.__proto__ = new Engine();
function FormSyncCore() {
function FormSyncCore(store) {
this._store = store;
this._init();
}
FormSyncCore.prototype = {
_logName: "FormSync",
__formDB: null,
get _formDB() {
if (!this.__formDB) {
var file = Cc["@mozilla.org/file/directory_service;1"].
getService(Ci.nsIProperties).
get("ProfD", Ci.nsIFile);
file.append("formhistory.sqlite");
var stor = Cc["@mozilla.org/storage/service;1"].
getService(Ci.mozIStorageService);
this.__formDB = stor.openDatabase(file);
}
return this.__formDB;
},
_itemExists: function FSC__itemExists(GUID) {
var found = false;
var stmnt = this._formDB.createStatement("SELECT * FROM moz_formhistory");
/* Same performance restrictions as PasswordSyncCore apply here:
caching required */
while (stmnt.executeStep()) {
var nam = stmnt.getUTF8String(1);
var val = stmnt.getUTF8String(2);
var key = _generateFormGUID(nam, val);
if (key == GUID)
found = true;
}
return found;
},
_store: null,
_commandLike: function FSC_commandLike(a, b) {
/* Not required as GUIDs for similar data sets will be the same */
@ -180,7 +98,8 @@ function FormStore() {
}
FormStore.prototype = {
_logName: "FormStore",
_lookup: null,
__formDB: null,
get _formDB() {
if (!this.__formDB) {
@ -203,21 +122,6 @@ FormStore.prototype = {
return this.__formHistory;
},
_getValueFromSHA1: function FormStore__getValueFromSHA1(name, sha) {
var query = "SELECT value FROM moz_formhistory WHERE fieldname = '" + name + "'";
var stmnt = this._formDB.createStatement(query);
var found = false;
while (stmnt.executeStep()) {
var val = stmnt.getUTF8String(0);
if (Utils.sha1(val) == sha) {
found = val;
break;
}
}
return found;
},
_createCommand: function FormStore__createCommand(command) {
this._log.info("FormStore got createCommand: " + command);
this._formHistory.addEntry(command.data.name, command.data.value);
@ -226,23 +130,19 @@ FormStore.prototype = {
_removeCommand: function FormStore__removeCommand(command) {
this._log.info("FormStore got removeCommand: " + command);
var data = _unwrapFormGUID(command.GUID);
if (!data) {
var data;
if (command.GUID in this._lookup) {
data = this._lookup[command.GUID];
} else {
this._log.warn("Invalid GUID found, ignoring remove request.");
return;
}
var nam = data[1];
var val = data[2];
if (data[0]) {
val = this._getValueFromSHA1(nam, val);
}
var nam = data.name;
var val = data.value;
this._formHistory.removeEntry(nam, val);
if (val) {
this._formHistory.removeEntry(nam, val);
} else {
this._log.warn("Form value not found from GUID, ignoring remove request.");
}
delete this._lookup[command.GUID];
},
_editCommand: function FormStore__editCommand(command) {
@ -251,18 +151,18 @@ FormStore.prototype = {
},
wrap: function FormStore_wrap() {
var items = [];
this._lookup = {};
var stmnt = this._formDB.createStatement("SELECT * FROM moz_formhistory");
while (stmnt.executeStep()) {
var nam = stmnt.getUTF8String(1);
var val = stmnt.getUTF8String(2);
var key = _generateFormGUID(nam, val);
var key = Utils.sha1(nam + val);
items[key] = { name: nam, value: val };
this._lookup[key] = { name: nam, value: val };
}
return items;
return this._lookup;
},
wipe: function FormStore_wipe() {

View File

@ -58,13 +58,6 @@ HistoryEngine.prototype = {
get logName() { return "HistEngine"; },
get serverPrefix() { return "user-data/history/"; },
__core: null,
get _core() {
if (!this.__core)
this.__core = new HistorySyncCore();
return this.__core;
},
__store: null,
get _store() {
if (!this.__store)
@ -72,6 +65,13 @@ HistoryEngine.prototype = {
return this.__store;
},
__core: null,
get _core() {
if (!this.__core)
this.__core = new HistorySyncCore(this._store);
return this.__core;
},
__tracker: null,
get _tracker() {
if (!this.__tracker)
@ -81,16 +81,13 @@ HistoryEngine.prototype = {
};
HistoryEngine.prototype.__proto__ = new Engine();
function HistorySyncCore() {
function HistorySyncCore(store) {
this._store = store;
this._init();
}
HistorySyncCore.prototype = {
_logName: "HistSync",
_itemExists: function HSC__itemExists(GUID) {
// we don't care about already-existing items; just try to re-add them
return false;
},
_store: null,
_commandLike: function HSC_commandLike(a, b) {
// History commands never qualify for likeness. We will always
@ -134,6 +131,11 @@ HistoryStore.prototype = {
return this.__hsvc;
},
_itemExists: function HistStore__itemExists(GUID) {
// we don't care about already-existing items; just try to re-add them
return false;
},
_createCommand: function HistStore__createCommand(command) {
this._log.debug(" -> creating history entry: " + command.GUID);
try {

View File

@ -43,27 +43,6 @@ Cu.import("resource://weave/engines.js");
Cu.import("resource://weave/syncCores.js");
Cu.import("resource://weave/stores.js");
/*
* _hashLoginInfo
*
* nsILoginInfo objects don't have a unique GUID, so we need to generate one
* on the fly. This is done by taking a hash of every field in the object.
* Note that the resulting GUID could potentiually reveal passwords via
* dictionary attacks or brute force. But GUIDs shouldn't be obtainable by
* anyone, so this should generally be safe.
*/
function _hashLoginInfo(aLogin) {
var loginKey = aLogin.hostname + ":" +
aLogin.formSubmitURL + ":" +
aLogin.httpRealm + ":" +
aLogin.username + ":" +
aLogin.password + ":" +
aLogin.usernameField + ":" +
aLogin.passwordField;
return Utils.sha1(loginKey);
}
function PasswordEngine() {
this._init();
}
@ -72,50 +51,29 @@ PasswordEngine.prototype = {
get logName() { return "PasswordEngine"; },
get serverPrefix() { return "user-data/passwords/"; },
__core: null,
get _core() {
if (!this.__core)
this.__core = new PasswordSyncCore();
return this.__core;
},
__store: null,
get _store() {
if (!this.__store)
this.__store = new PasswordStore();
return this.__store;
},
__core: null,
get _core() {
if (!this.__core)
this.__core = new PasswordSyncCore(this._store);
return this.__core;
}
};
PasswordEngine.prototype.__proto__ = new Engine();
function PasswordSyncCore() {
function PasswordSyncCore(store) {
this._store = store;
this._init();
}
PasswordSyncCore.prototype = {
_logName: "PasswordSync",
__loginManager : null,
get _loginManager() {
if (!this.__loginManager)
this.__loginManager = Utils.getLoginManager();
return this.__loginManager;
},
_itemExists: function PSC__itemExists(GUID) {
var found = false;
var logins = this._loginManager.getAllLogins({});
// XXX It would be more efficient to compute all the hashes in one shot,
// cache the results, and check the cache here. That would need to happen
// once per sync -- not sure how to invalidate cache after current sync?
for (var i = 0; i < logins.length && !found; i++) {
var hash = _hashLoginInfo(logins[i]);
if (hash == GUID)
found = true;;
}
return found;
},
_store: null,
_commandLike: function PSC_commandLike(a, b) {
// Not used.
@ -129,20 +87,42 @@ function PasswordStore() {
}
PasswordStore.prototype = {
_logName: "PasswordStore",
_lookup: null,
__loginManager : null,
__loginManager: null,
get _loginManager() {
if (!this.__loginManager)
this.__loginManager = Utils.getLoginManager();
return this.__loginManager;
},
__nsLoginInfo : null,
__nsLoginInfo: null,
get _nsLoginInfo() {
if (!this.__nsLoginInfo)
this.__nsLoginInfo = Utils.makeNewLoginInfo();
return this.__nsLoginInfo;
},
/*
* _hashLoginInfo
*
* nsILoginInfo objects don't have a unique GUID, so we need to generate one
* on the fly. This is done by taking a hash of every field in the object.
* Note that the resulting GUID could potentiually reveal passwords via
* dictionary attacks or brute force. But GUIDs shouldn't be obtainable by
* anyone, so this should generally be safe.
*/
_hashLoginInfo: function PasswordStore__hashLoginInfo(aLogin) {
var loginKey = aLogin.hostname + ":" +
aLogin.formSubmitURL + ":" +
aLogin.httpRealm + ":" +
aLogin.username + ":" +
aLogin.password + ":" +
aLogin.usernameField + ":" +
aLogin.passwordField;
return Utils.sha1(loginKey);
},
_createCommand: function PasswordStore__createCommand(command) {
this._log.info("PasswordStore got createCommand: " + command );
@ -180,13 +160,12 @@ PasswordStore.prototype = {
wrap: function PasswordStore_wrap() {
/* Return contents of this store, as JSON. */
var items = {};
var logins = this._loginManager.getAllLogins({});
for (var i = 0; i < logins.length; i++) {
var login = logins[i];
var key = _hashLoginInfo(login);
var key = this._hashLoginInfo(login);
items[key] = { hostname : login.hostname,
formSubmitURL : login.formSubmitURL,
@ -197,6 +176,7 @@ PasswordStore.prototype = {
passwordField : login.passwordField };
}
this._lookup = items;
return items;
},

View File

@ -62,17 +62,17 @@ TabEngine.prototype = {
get serverPrefix() "user-data/tabs/",
get store() this._store,
get _core() {
let core = new TabSyncCore(this);
this.__defineGetter__("_core", function() core);
return this._core;
},
get _store() {
let store = new TabStore();
this.__defineGetter__("_store", function() store);
return this._store;
},
get _core() {
let core = new TabSyncCore(this._store);
this.__defineGetter__("_core", function() core);
return this._core;
},
get _tracker() {
let tracker = new TabTracker(this);
@ -82,16 +82,15 @@ TabEngine.prototype = {
};
function TabSyncCore(engine) {
this._engine = engine;
function TabSyncCore(store) {
this._store = store;
this._init();
}
TabSyncCore.prototype = {
__proto__: new SyncCore(),
_logName: "TabSync",
_engine: null,
_store: null,
get _sessionStore() {
let sessionStore = Cc["@mozilla.org/browser/sessionstore;1"].
@ -100,27 +99,6 @@ TabSyncCore.prototype = {
return this._sessionStore;
},
_itemExists: function TSC__itemExists(GUID) {
// Note: this method returns true if the tab exists in any window, not just
// the window from which the tab came. In the future, if we care about
// windows, we might need to make this more specific, although in that case
// we'll have to identify tabs by something other than URL, since even
// window-specific tabs look the same when identified by URL.
// Get the set of all real and virtual tabs.
let tabs = this._engine.store.wrap();
// XXX Should we convert both to nsIURIs and then use nsIURI::equals
// to compare them?
if (GUID in tabs) {
this._log.trace("_itemExists: " + GUID + " exists");
return true;
}
this._log.trace("_itemExists: " + GUID + " doesn't exist");
return false;
},
_commandLike: function TSC_commandLike(a, b) {
// Not implemented.
return false;
@ -263,6 +241,27 @@ TabStore.prototype = {
self.done();
},
_itemExists: function TabStore__itemExists(GUID) {
// Note: this method returns true if the tab exists in any window, not just
// the window from which the tab came. In the future, if we care about
// windows, we might need to make this more specific, although in that case
// we'll have to identify tabs by something other than URL, since even
// window-specific tabs look the same when identified by URL.
// Get the set of all real and virtual tabs.
let tabs = this.wrap();
// XXX Should we convert both to nsIURIs and then use nsIURI::equals
// to compare them?
if (GUID in tabs) {
this._log.trace("_itemExists: " + GUID + " exists");
return true;
}
this._log.trace("_itemExists: " + GUID + " doesn't exist");
return false;
},
_createCommand: function TabStore__createCommand(command) {
this._log.debug("_createCommand: " + command.GUID);

View File

@ -62,6 +62,9 @@ Store.prototype = {
_logName: "Store",
_yieldDuringApply: true,
// set this property in child object's wrap()!
_lookup: null,
__json: null,
get _json() {
if (!this.__json)
@ -102,10 +105,28 @@ Store.prototype = {
self.done();
},
// override only if neccessary
_itemExists: function Store__itemExists(GUID) {
if (GUID in this._lookup)
return true;
else
return false;
},
// override these in derived objects
wrap: function Store_wrap() {},
wipe: function Store_wipe() {},
resetGUIDs: function Store_resetGUIDs() {}
// wrap MUST save the wrapped store in the _lookup property!
wrap: function Store_wrap() {
throw "wrap needs to be subclassed";
},
wipe: function Store_wipe() {
throw "wipe needs to be subclassed";
},
resetGUIDs: function Store_resetGUIDs() {
throw "resetGUIDs needs to be subclassed";
}
};
function SnapshotStore(name) {

View File

@ -62,7 +62,10 @@ function SyncCore() {
}
SyncCore.prototype = {
_logName: "Sync",
// Set this property in child objects!
_store: null,
_init: function SC__init() {
this._log = Log4Moz.Service.getLogger("Service." + this._logName);
},
@ -209,11 +212,6 @@ SyncCore.prototype = {
}
},
_itemExists: function SC__itemExists(GUID) {
this._log.error("itemExists needs to be subclassed");
return false;
},
_reconcile: function SC__reconcile(listA, listB) {
let self = yield;
@ -253,7 +251,7 @@ SyncCore.prototype = {
}
// watch out for create commands with GUIDs that already exist
if (b.action == "create" && this._itemExists(b.GUID)) {
if (b.action == "create" && this._store._itemExists(b.GUID)) {
this._log.error("Remote command has GUID that already exists " +
"locally. Dropping command.");
return false; // delete b