From e2fc25dbfac59e36571311ae6e2c9b3417136c42 Mon Sep 17 00:00:00 2001 From: Dan Mills Date: Tue, 24 Jun 2008 12:33:27 -0700 Subject: [PATCH] Store each delta as a separate file on the server --- services/sync/modules/engines.js | 12 ++-- services/sync/modules/remote.js | 113 ++++++++++++++++++++----------- 2 files changed, 79 insertions(+), 46 deletions(-) diff --git a/services/sync/modules/engines.js b/services/sync/modules/engines.js index 08ee457ca73..ee37ba01c80 100644 --- a/services/sync/modules/engines.js +++ b/services/sync/modules/engines.js @@ -379,20 +379,18 @@ Engine.prototype = { this._snapshot.data = newSnapshot; this._snapshot.version = ++this._remote.status.data.maxVersion; + // XXX don't append delta if we do a full upload? if (this._remote.status.data.formatVersion != ENGINE_STORAGE_FORMAT_VERSION) yield this._remote.initialize(self.cb, this._snapshot); - this._remote.appendDelta(self.cb, serverDelta); - yield; - let c = 0; for (GUID in this._snapshot.data) c++; - this._remote.status.data.maxVersion = this._snapshot.version; - this._remote.status.data.snapEncryption = Crypto.defaultAlgorithm; - this._remote.status.data.itemCount = c; - this._remote.status.put(self.cb, this._remote.status.data); + this._remote.appendDelta(self.cb, serverDelta, + {maxVersion: this._snapshot.version, + deltasEncryption: Crypto.defaultAlgorithm, + itemCount: c}); yield; this._log.info("Successfully updated deltas and status on server"); diff --git a/services/sync/modules/remote.js b/services/sync/modules/remote.js index 2a54af932ca..49dfa172102 100644 --- a/services/sync/modules/remote.js +++ b/services/sync/modules/remote.js @@ -230,6 +230,49 @@ Resource.prototype = { } }; +function ResourceSet(basePath) { + this._init(basePath); +} +ResourceSet.prototype = { + __proto__: new Resource(), + _init: function ResSet__init(basePath) { + this.__proto__.__proto__._init.call(this); + this._basePath = basePath; + this._log = Log4Moz.Service.getLogger("Service.ResourceSet"); + }, + + _hack: function ResSet__hack(action, id, data) { + let self = yield; + let savedData = this._data; + + if ("PUT" == action) + this._data = data; + + this._path = this._basePath + id; + yield this._request.async(this, self.cb, action, data); + + let newData = this._data; + this._data = savedData; + if (this._data == null) + this._data = {}; + this._data[id] = newData; + + self.done(this._data[id]); + }, + + get: function ResSet_get(onComplete, id) { + this._hack.async(this, onComplete, "GET", id); + }, + + put: function ResSet_put(onComplete, id, data) { + this._hack.async(this, onComplete, "PUT", id, data); + }, + + delete: function ResSet_delete(onComplete, id) { + this._hack.async(this, onComplete, "DELETE", id); + } +}; + function ResourceFilter() { this._log = Log4Moz.Service.getLogger("Service.ResourceFilter"); } @@ -298,18 +341,6 @@ CryptoFilter.prototype = { } }; -function Status(remoteStore) { - this._init(remoteStore); -} -Status.prototype = { - __proto__: new Resource(), - _init: function Status__init(remoteStore) { - this._remote = remoteStore; - this.__proto__.__proto__._init.call(this, this._remote.serverPrefix + "status.json"); - this.pushFilter(new JsonFilter()); - } -}; - function Keychain(remoteStore) { this._init(remoteStore); } @@ -352,7 +383,8 @@ RemoteStore.prototype = { get engineId() this._engine.engineId, get status() { - let status = new Status(this); + let status = new Resource(this.serverPrefix + "status.json"); + status.pushFilter(new JsonFilter()); this.__defineGetter__("status", function() status); return status; }, @@ -372,7 +404,7 @@ RemoteStore.prototype = { }, get _deltas() { - let deltas = new Resource(this.serverPrefix + "deltas.json"); + let deltas = new ResourceSet(this.serverPrefix + "deltas/"); deltas.pushFilter(new JsonFilter()); deltas.pushFilter(new CryptoFilter(this, "deltasEncryption")); this.__defineGetter__("_deltas", function() deltas); @@ -390,8 +422,7 @@ RemoteStore.prototype = { this._snapshot.data = null; this._deltas.data = null; - DAV.MKCOL(this.serverPrefix, self.cb); - let ret = yield; + let ret = yield DAV.MKCOL(this.serverPrefix + "deltas", self.cb); if (!ret) throw "Could not create remote folder"; @@ -421,6 +452,7 @@ RemoteStore.prototype = { }, // Does a fresh upload of the given snapshot to a new store + // FIXME: add 'metadata' arg here like appendDelta's _initialize: function RStore__initialize(snapshot) { let self = yield; let wrappedSymkey; @@ -440,7 +472,7 @@ RemoteStore.prototype = { yield this.keys.put(self.cb, keys); yield this._snapshot.put(self.cb, snapshot.data); - yield this._deltas.put(self.cb, []); + //yield this._deltas.put(self.cb, []); let c = 0; for (GUID in snapshot.data) @@ -467,7 +499,7 @@ RemoteStore.prototype = { yield this.status.delete(self.cb); yield this.keys.delete(self.cb); yield this._snapshot.delete(self.cb); - yield this._deltas.delete(self.cb); + //yield this._deltas.delete(self.cb); this._log.debug("Server files deleted"); }, wipe: function Engine_wipe(onComplete) { @@ -478,20 +510,17 @@ RemoteStore.prototype = { // (snapshot + deltas) _getLatestFromScratch: function RStore__getLatestFromScratch() { let self = yield; + let status = this.status.data; this._log.info("Downloading all server data from scratch"); - this._log.debug("Downloading server snapshot"); - let data = yield this._snapshot.get(self.cb); - this._log.debug("Downloading server deltas"); - let deltas = yield this._deltas.get(self.cb); - let snap = new SnapshotStore(); - snap.version = this.status.data.maxVersion; - snap.data = data; - for (let i = 0; i < deltas.length; i++) { - snap.applyCommands.async(snap, self.cb, deltas[i]); - yield; + snap.data = yield this._snapshot.get(self.cb); + snap.version = status.maxVersion; + + for (let id = status.snapVersion + 1; id <= status.maxVersion; id++) { + let delta = yield this._deltas.get(self.cb, id); + yield snap.applyCommands.async(snap, self.cb, delta); } self.done(snap); @@ -514,8 +543,12 @@ RemoteStore.prototype = { this._log.debug("Using last sync snapshot as starting point for server snapshot"); snap.data = Utils.deepCopy(lastSyncSnap.data); this._log.info("Downloading server deltas"); - let allDeltas = yield this._deltas.get(self.cb); - deltas = allDeltas.slice(lastSyncSnap.version - this.status.data.snapVersion); + deltas = []; + let min = lastSyncSnap.version + 1; + let max = this.status.data.maxVersion; + for (let id = min; id <= max; id++) { + deltas.push(yield this._deltas.get(self.cb, id)); + } } else if (lastSyncSnap.version == this.status.data.maxVersion) { this._log.debug("Using last sync snapshot as server snapshot (snap version == max version)"); @@ -557,17 +590,19 @@ RemoteStore.prototype = { }, // Adds a new set of changes (a delta) to this store - _appendDelta: function RStore__appendDelta(delta) { + _appendDelta: function RStore__appendDelta(delta, metadata) { let self = yield; - if (this._deltas.data == null) { - yield this._deltas.get(self.cb); - if (this._deltas.data == null) - this._deltas.data = []; + + if (metadata) { + for (let key in metadata) + this.status.data[key] = metadata[key]; } - this._deltas.data.push(delta); - yield this._deltas.put(self.cb, this._deltas.data); + + let id = this.status.data.maxVersion; // FIXME: we increment maxVersion in Engine + yield this._deltas.put(self.cb, id, delta); + yield this.status.put(self.cb, this.status.data); }, - appendDelta: function RStore_appendDelta(onComplete, delta) { - this._appendDelta.async(this, onComplete, delta); + appendDelta: function RStore_appendDelta(onComplete, delta, metadata) { + this._appendDelta.async(this, onComplete, delta, metadata); } };