Bug 673548 - Part 2: implement JS SyncServer. r=philikon

services/sync/tests/unit/head_http_server.js

@@ -1,6 +1,7 @@
 // Shared logging for all HTTP server functions.
 Cu.import("resource://services-sync/log4moz.js");
 const SYNC_HTTP_LOGGER = "Sync.Test.Server";
+const SYNC_API_VERSION = "1.1";
 
 // Use the same method that record.js does, which mirrors the server.
 // The server returns timestamps with 1/100 sec granularity. Note that this is
@@ -496,3 +497,420 @@ function track_collections_helper() {
           "with_updated_collection": with_updated_collection,
           "update_collection": update_collection};
 }
+
+//===========================================================================//
+// httpd.js-based Sync server.                                               //
+//===========================================================================//
+
+/**
+ * In general, the preferred way of using SyncServer is to directly introspect
+ * it. Callbacks are available for operations which are hard to verify through
+ * introspection, such as deletions.
+ *
+ * One of the goals of this server is to provide enough hooks for test code to
+ * find out what it needs without monkeypatching. Use this object as your
+ * prototype, and override as appropriate.
+ */
+let SyncServerCallback = {
+  onCollectionDeleted: function onCollectionDeleted(user, collection) {},
+  onItemDeleted: function onItemDeleted(user, collection, wboID) {}
+};
+
+/**
+ * Construct a new test Sync server. Takes a callback object (e.g.,
+ * SyncServerCallback) as input.
+ */
+function SyncServer(callback) {
+  this.callback = callback || {__proto__: SyncServerCallback};
+  this.server   = new nsHttpServer();
+  this.started  = false;
+  this.users    = {};
+  this._log     = Log4Moz.repository.getLogger(SYNC_HTTP_LOGGER);
+
+  // Install our own default handler. This allows us to mess around with the
+  // whole URL space.
+  let handler = this.server._handler;
+  handler._handleDefault = this.handleDefault.bind(this, handler);
+}
+SyncServer.prototype = {
+  port:   8080,
+  server: null,    // nsHttpServer.
+  users:  null,    // Map of username => {collections, password}.
+
+  /**
+   * Start the SyncServer's underlying HTTP server.
+   *
+   * @param port
+   *        The numeric port on which to start. A falsy value implies the
+   *        default (8080).
+   * @param cb
+   *        A callback function (of no arguments) which is invoked after
+   *        startup.
+   */
+  start: function start(port, cb) {
+    if (this.started) {
+      this._log.warn("Warning: server already started on " + this.port);
+      return;
+    }
+    if (port) {
+      this.port = port;
+    }
+    try {
+      this.server.start(this.port);
+      this.started = true;
+      if (cb) {
+        cb();
+      }
+    } catch (ex) {
+      _("==========================================");
+      _("Got exception starting Sync HTTP server on port " + this.port);
+      _("Error: " + Utils.exceptionStr(ex));
+      _("Is there a process already listening on port " + this.port + "?");
+      _("==========================================");
+      do_throw(ex);
+    }
+  },
+
+  /**
+   * Stop the SyncServer's HTTP server.
+   *
+   * @param cb
+   *        A callback function. Invoked after the server has been stopped.
+   *
+   */
+  stop: function stop(cb) {
+    if (!this.started) {
+      this._log.warn("SyncServer: Warning: server not running. Can't stop me now!");
+      return;
+    }
+
+    this.server.stop(cb);
+    this.started = false;
+  },
+
+  /**
+   * Return a server timestamp for a record.
+   * The server returns timestamps with 1/100 sec granularity. Note that this is
+   * subject to change: see Bug 650435.
+   */
+  timestamp: function timestamp() {
+    return Math.round(Date.now() / 10) / 100;
+  },
+
+  /**
+   * Create a new user, complete with an empty set of collections.
+   */
+  registerUser: function registerUser(username, password) {
+    if (username in this.users) {
+      throw new Error("User already exists.");
+    }
+    this.users[username] = {
+      password: password,
+      collections: {}
+    };
+  },
+
+  userExists: function userExists(username) {
+    return username in this.users;
+  },
+
+  getCollection: function getCollection(username, collection) {
+    return this.users[username].collections[collection];
+  },
+
+  _insertCollection: function _insertCollection(collections, collection, wbos) {
+    let coll = new ServerCollection(wbos, true);
+    coll.collectionHandler = coll.handler();
+    collections[collection] = coll;
+    return coll;
+  },
+
+  createCollection: function createCollection(username, collection, wbos) {
+    if (!(username in this.users)) {
+      throw new Error("Unknown user.");
+    }
+    let collections = this.users[username].collections;
+    if (collection in collections) {
+      throw new Error("Collection already exists.");
+    }
+    return this._insertCollection(collections, collection, wbos);
+  },
+
+  /**
+   * Accept a map like the following:
+   * {
+   *   meta: {global: {version: 1, ...}},
+   *   crypto: {"keys": {}, foo: {bar: 2}},
+   *   bookmarks: {}
+   * }
+   * to cause collections and WBOs to be created.
+   * If a collection already exists, no error is raised.
+   * If a WBO already exists, it will be updated to the new contents.
+   */
+  createContents: function createContents(username, collections) {
+    if (!(username in this.users)) {
+      throw new Error("Unknown user.");
+    }
+    let userCollections = this.users[username].collections;
+    for (let [id, contents] in Iterator(collections)) {
+      let coll = userCollections[id] ||
+                 this._insertCollection(userCollections, id);
+      for (let [wboID, payload] in Iterator(contents)) {
+        coll.insert(wboID, payload);
+      }
+    }
+  },
+
+  /**
+   * Insert a WBO in an existing collection.
+   */
+  insertWBO: function insertWBO(username, collection, wbo) {
+    if (!(username in this.users)) {
+      throw new Error("Unknown user.");
+    }
+    let userCollections = this.users[username].collections;
+    if (!(collection in userCollections)) {
+      throw new Error("Unknown collection.");
+    }
+    userCollections[collection].insertWBO(wbo);
+    return wbo;
+  },
+
+  /**
+   * Simple accessor to allow collective binding and abbreviation of a bunch of
+   * methods. Yay!
+   * Use like this:
+   *
+   *   let u = server.user("john");
+   *   u.collection("bookmarks").wbo("abcdefg").payload;  // Etc.
+   *
+   * @return a proxy for the user data stored in this server.
+   */
+  user: function user(username) {
+    let collection       = this.getCollection.bind(this, username);
+    let createCollection = this.createCollection.bind(this, username);
+    return {
+      collection:       collection,
+      createCollection: createCollection
+    };
+  },
+
+  /*
+   * Regular expressions for splitting up Sync request paths.
+   * Sync URLs are of the form:
+   *   /$apipath/$version/$user/$further
+   * where $further is usually:
+   *   storage/$collection/$wbo
+   * or
+   *   storage/$collection
+   * or
+   *   info/$op
+   * We assume for the sake of simplicity that $apipath is empty.
+   *
+   * N.B., we don't follow any kind of username spec here, because as far as I
+   * can tell there isn't one. See Bug 689671. Instead we follow the Python
+   * server code.
+   *
+   * Path: [all, version, username, first, rest]
+   * Storage: [all, collection, id?]
+   */
+  pathRE: /^\/([0-9]+(?:\.[0-9]+)?)\/([-._a-zA-Z0-9]+)\/([^\/]+)\/(.*)$/,
+  storageRE: /^([-_a-zA-Z0-9]+)(?:\/([-_a-zA-Z0-9]+)\/?)?$/,
+
+  defaultHeaders: {},
+
+  /**
+   * HTTP response utility.
+   */
+  respond: function respond(req, resp, code, status, body, headers) {
+    resp.setStatusLine(req.httpVersion, code, status);
+    for each (let [header, value] in Iterator(headers || this.defaultHeaders)) {
+      resp.setHeader(header, value);
+    }
+    resp.setHeader("X-Weave-Timestamp", "" + this.timestamp(), false);
+    resp.bodyOutputStream.write(body, body.length);
+  },
+
+  /**
+   * This is invoked by the nsHttpServer. `this` is bound to the SyncServer;
+   * `handler` is the nsHttpServer's handler.
+   *
+   * TODO: need to use the correct Sync API response codes and errors here.
+   * TODO: Basic Auth.
+   * TODO: check username in path against username in BasicAuth. 
+   */
+  handleDefault: function handleDefault(handler, req, resp) {
+    this._log.debug("SyncServer: Handling request: " + req.method + " " + req.path);
+    let parts = this.pathRE.exec(req.path);
+    if (!parts) {
+      this._log.debug("SyncServer: Unexpected request: bad URL " + req.path);
+      throw HTTP_404;
+    }
+
+    let [all, version, username, first, rest] = parts;
+    if (version != SYNC_API_VERSION) {
+      this._log.debug("SyncServer: Unknown version.");
+      throw HTTP_404;
+    }
+
+    if (!this.userExists(username)) {
+      this._log.debug("SyncServer: Unknown user.");
+      throw HTTP_401;
+    }
+
+    // Hand off to the appropriate handler for this path component.
+    if (first in this.toplevelHandlers) {
+      let handler = this.toplevelHandlers[first];
+      return handler.call(this, handler, req, resp, version, username, rest);
+    }
+    this._log.debug("SyncServer: Unknown top-level " + first);
+    throw HTTP_404;
+  },
+
+  /**
+   * Compute the object that is returned for an info/collections request.
+   */
+  infoCollections: function infoCollections(username) {
+    let responseObject = {};
+    let colls = this.users[username].collections;
+    for (let coll in colls) {
+      responseObject[coll] = colls[coll].timestamp;
+    }
+    this._log.trace("SyncServer: info/collections returning " +
+                    JSON.stringify(responseObject));
+    return responseObject;
+  },
+
+  /**
+   * Collection of the handler methods we use for top-level path components.
+   */
+  toplevelHandlers: {
+    "storage": function handleStorage(handler, req, resp, version, username, rest) {
+      let match = this.storageRE.exec(rest);
+      if (!match) {
+        this._log.warn("SyncServer: Unknown storage operation " + rest);
+        throw HTTP_404;
+      }
+      let [all, collection, wboID] = match;
+      let coll = this.getCollection(username, collection);
+      let respond = this.respond.bind(this, req, resp);
+      switch (req.method) {
+        case "GET":
+          if (!coll) {
+            // *cries inside*: Bug 687299.
+            respond(200, "OK", "[]");
+            return;
+          }
+          if (!wboID) {
+            return coll.collectionHandler(req, resp);
+          }
+          let wbo = coll.wbo(wboID);
+          if (!wbo) {
+            respond(404, "Not found", "Not found");
+            return;
+          }
+          return wbo.handler()(req, resp);
+
+        // TODO: implement handling of X-If-Unmodified-Since for write verbs.
+        case "DELETE":
+          if (!coll) {
+            respond(200, "OK", "{}");
+            return;
+          }
+          if (wboID) {
+            let wbo = coll.wbo(wboID);
+            if (wbo) {
+              wbo.delete();
+            }
+            respond(200, "OK", "{}");
+            this.callback.onItemDeleted(username, collectin, wboID);
+            return;
+          }
+          coll.collectionHandler(req, resp);
+
+          // Spot if this is a DELETE for some IDs, and don't blow away the
+          // whole collection!
+          //
+          // We already handled deleting the WBOs by invoking the deleted
+          // collection's handler. However, in the case of 
+          //
+          //   DELETE storage/foobar
+          //
+          // we also need to remove foobar from the collections map. This
+          // clause tries to differentiate the above request from
+          //
+          //  DELETE storage/foobar?ids=foo,baz
+          //
+          // and do the right thing.
+          // TODO: less hacky method.
+          if (-1 == req.queryString.indexOf("ids=")) {
+            // When you delete the entire collection, we drop it.
+            this._log.debug("Deleting entire collection.");
+            delete this.users[username].collections[collection];
+            this.callback.onCollectionDeleted(username, collection);
+          }
+
+          // Notify of item deletion.
+          let deleted = resp.deleted || [];
+          for (let i = 0; i < deleted.length; ++i) {
+            this.callback.onItemDeleted(username, collection, deleted[i]);
+          }
+          return;
+        case "POST":
+        case "PUT":
+          if (!coll) {
+            coll = this.createCollection(username, collection);
+          }
+          if (wboID) {
+            let wbo = coll.wbo(wboID);
+            if (!wbo) {
+              this._log.trace("SyncServer: creating WBO " + collection + "/" + wboID);
+              wbo = coll.insert(wboID);
+            }
+            // Rather than instantiate each WBO's handler function, do it once
+            // per request. They get hit far less often than do collections.
+            wbo.handler()(req, resp);
+            coll.timestamp = resp.newModified;
+            return resp;
+          }
+          return coll.collectionHandler(req, resp);
+        default:
+          throw "Request method " + req.method + " not implemented.";
+      }
+    },
+
+    "info": function handleInfo(handler, req, resp, version, username, rest) {
+      switch (rest) {
+        case "collections":
+          let body = JSON.stringify(this.infoCollections(username));
+          this.respond(req, resp, 200, "OK", body, {
+            "Content-Type": "application/json"
+          });
+          return;
+        case "collection_usage":
+        case "collection_counts":
+        case "quota":
+          // TODO: implement additional info methods.
+          this.respond(req, resp, 200, "OK", "TODO");
+          return;
+        default:
+          // TODO
+          this._log.warn("SyncServer: Unknown info operation " + rest);
+          throw HTTP_404;
+      }
+    }
+  }
+};
+
+/**
+ * Test helper.
+ */
+function serverForUsers(users, contents, callback) {
+  let server = new SyncServer(callback);
+  for (let [user, pass] in Iterator(users)) {
+    server.registerUser(user, pass);
+    server.createContents(user, contents);
+  }
+  server.start();
+  return server;
+}

services/sync/tests/unit/test_httpd_sync_server.js

@@ -0,0 +1,155 @@
+function run_test() {
+  run_next_test();
+}
+
+add_test(function test_creation() {
+  // Explicit callback for this one.
+  let s = new SyncServer({
+    __proto__: SyncServerCallback,
+  });
+  do_check_true(!!s);       // Just so we have a check.
+  s.start(null, function () {
+    _("Started on " + s.port);
+    s.stop(run_next_test);
+  });
+});
+
+add_test(function test_url_parsing() {
+  let s = new SyncServer();
+  let parts = s.pathRE.exec("/1.1/johnsmith/storage/crypto/keys");
+  let [all, version, username, first, rest] = parts;
+  do_check_eq(version, "1.1");
+  do_check_eq(username, "johnsmith");
+  do_check_eq(first, "storage");
+  do_check_eq(rest, "crypto/keys");
+  do_check_eq(null, s.pathRE.exec("/nothing/else"));
+  run_next_test();
+});
+
+Cu.import("resource://services-sync/rest.js");
+function localRequest(path) {
+  _("localRequest: " + path);
+  let url = "http://127.0.0.1:8080" + path;
+  _("url: " + url);
+  return new RESTRequest(url);
+}
+
+add_test(function test_basic_http() {
+  let s = new SyncServer();
+  s.registerUser("john", "password");
+  do_check_true(s.userExists("john"));
+  s.start(8080, function () {
+    _("Started on " + s.port);
+    do_check_eq(s.port, 8080);
+    Utils.nextTick(function () {
+      let req = localRequest("/1.1/john/storage/crypto/keys");
+      _("req is " + req);
+      req.get(function (err) {
+        do_check_eq(null, err);
+        Utils.nextTick(function () {
+          s.stop(run_next_test);
+        });
+      });
+    });
+  });
+});
+
+add_test(function test_info_collections() {
+  let s = new SyncServer({
+    __proto__: SyncServerCallback
+  });
+  function responseHasCorrectHeaders(r) {
+    do_check_eq(r.status, 200);
+    do_check_eq(r.headers["content-type"], "application/json");
+    do_check_true("x-weave-timestamp" in r.headers);
+  }
+
+  s.registerUser("john", "password");
+  s.start(8080, function () {
+    do_check_eq(s.port, 8080);
+    Utils.nextTick(function () {
+      let req = localRequest("/1.1/john/info/collections");
+      req.get(function (err) {
+        // Initial info/collections fetch is empty.
+        do_check_eq(null, err);
+        responseHasCorrectHeaders(this.response);
+
+        do_check_eq(this.response.body, "{}");
+        Utils.nextTick(function () {
+          // When we PUT something to crypto/keys, "crypto" appears in the response.
+          function cb(err) {
+            do_check_eq(null, err);
+            responseHasCorrectHeaders(this.response);
+            let putResponseBody = this.response.body;
+            _("PUT response body: " + JSON.stringify(putResponseBody));
+
+            req = localRequest("/1.1/john/info/collections");
+            req.get(function (err) {
+              do_check_eq(null, err);
+              responseHasCorrectHeaders(this.response);
+              let expectedColl = s.getCollection("john", "crypto");
+              do_check_true(!!expectedColl);
+              let modified = expectedColl.timestamp;
+              do_check_true(modified > 0);
+              do_check_eq(putResponseBody, modified);
+              do_check_eq(JSON.parse(this.response.body).crypto, modified);
+              Utils.nextTick(function () {
+                s.stop(run_next_test);
+              });
+            });
+          }
+          let payload = JSON.stringify({foo: "bar"});
+          localRequest("/1.1/john/storage/crypto/keys").put(payload, cb);
+        });
+      });
+    });
+  });
+});
+
+add_test(function test_storage_request() {
+  let keysURL = "/1.1/john/storage/crypto/keys?foo=bar";
+  let foosURL = "/1.1/john/storage/crypto/foos";
+  let s = new SyncServer();
+  let creation = s.timestamp();
+  s.registerUser("john", "password");
+
+  s.createContents("john", {
+    crypto: {foos: {foo: "bar"}}
+  });
+  let coll = s.user("john").collection("crypto");
+  do_check_true(!!coll);
+
+  _("We're tracking timestamps.");
+  do_check_true(coll.timestamp >= creation);
+
+  function retrieveWBONotExists(next) {
+    let req = localRequest(keysURL);
+    req.get(function (err) {
+      _("Body is " + this.response.body);
+      _("Modified is " + this.response.newModified);
+      do_check_eq(null, err);
+      do_check_eq(this.response.status, 404);
+      do_check_eq(this.response.body, "Not found");
+      Utils.nextTick(next);
+    });
+  }
+  function retrieveWBOExists(next) {
+    let req = localRequest(foosURL);
+    req.get(function (err) {
+      _("Body is " + this.response.body);
+      _("Modified is " + this.response.newModified);
+      let parsedBody = JSON.parse(this.response.body);
+      do_check_eq(parsedBody.id, "foos");
+      do_check_eq(parsedBody.modified, coll.wbo("foos").modified);
+      do_check_eq(JSON.parse(parsedBody.payload).foo, "bar");
+      Utils.nextTick(next);
+    });
+  }
+  s.start(8080, function () {
+    retrieveWBONotExists(
+      retrieveWBOExists.bind(this, function () {
+        s.stop(run_next_test);
+      })
+    );
+  });
+});

services/sync/tests/unit/xpcshell.ini

@@ -39,6 +39,7 @@ skip-if = (os == "mac" && debug) || os == "android"
 [test_history_store.js]
 [test_history_tracker.js]
 [test_hmac_error.js]
+[test_httpd_sync_server.js]
 [test_interval_triggers.js]
 [test_jpakeclient.js]
 # Bug 618233: this test produces random failures on Windows 7.