diff --git a/services/common/Makefile.in b/services/common/Makefile.in index beae39aedaa..9b310ab1717 100644 --- a/services/common/Makefile.in +++ b/services/common/Makefile.in @@ -11,6 +11,7 @@ include $(DEPTH)/config/autoconf.mk modules := \ async.js \ + bagheeraclient.js \ log4moz.js \ observers.js \ preferences.js \ @@ -23,6 +24,7 @@ modules := \ testing_modules := \ aitcserver.js \ + bagheeraserver.js \ logging.js \ storageserver.js \ utils.js \ @@ -46,20 +48,16 @@ include $(topsrcdir)/config/rules.mk # ever consolidate our Python code, and/or have a supplemental driver for the # build system, this can go away. -storage_server_hostname := localhost -storage_server_port := 8080 - -head_path = $(srcdir)/tests/unit +server_port := 8080 storage-server: $(PYTHON) $(srcdir)/tests/run_server.py $(topsrcdir) \ - $(MOZ_BUILD_ROOT) run_storage_server.js --port $(storage_server_port) - -# And the same thing for an AITC server. -aitc_server_hostname := localhost -aitc_server_port := 8080 + $(MOZ_BUILD_ROOT) run_storage_server.js --port $(server_port) aitc-server: $(PYTHON) $(srcdir)/tests/run_server.py $(topsrcdir) \ - $(MOZ_BUILD_ROOT) run_aitc_server.js --port $(aitc_server_port) + $(MOZ_BUILD_ROOT) run_aitc_server.js --port $(server_port) +bagheera-server: + $(PYTHON) $(srcdir)/tests/run_server.py $(topsrcdir) \ + $(MOZ_BUILD_ROOT) run_bagheera_server.js --port $(server_port) diff --git a/services/common/bagheeraclient.js b/services/common/bagheeraclient.js new file mode 100644 index 00000000000..5944665ed7f --- /dev/null +++ b/services/common/bagheeraclient.js @@ -0,0 +1,226 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/** + * This file contains a client API for the Bagheera data storage service. + * + * Information about Bagheera is available at + * https://github.com/mozilla-metrics/bagheera + */ + +"use strict"; + +this.EXPORTED_SYMBOLS = [ + "BagheeraClient", + "BagheeraClientRequestResult", +]; + +const {classes: Cc, interfaces: Ci, utils: Cu} = Components; + +Cu.import("resource://gre/modules/commonjs/promise/core.js"); +Cu.import("resource://gre/modules/services-common/log4moz.js"); +Cu.import("resource://gre/modules/services-common/rest.js"); +Cu.import("resource://gre/modules/services-common/utils.js"); + + +/** + * Represents the result of a Bagheera request. + */ +this.BagheeraClientRequestResult = function BagheeraClientRequestResult() { + this.transportSuccess = false; + this.serverSuccess = false; + this.request = null; +} + +Object.freeze(BagheeraClientRequestResult.prototype); + +/** + * Create a new Bagheera client instance. + * + * Each client is associated with a specific Bagheera HTTP URI endpoint. + * + * @param baseURI + * (string) The base URI of the Bagheera HTTP endpoint. + */ +this.BagheeraClient = function BagheeraClient(baseURI) { + if (!baseURI) { + throw new Error("baseURI argument must be defined."); + } + + this._log = Log4Moz.repository.getLogger("Services.BagheeraClient"); + this._log.level = Log4Moz.Level["Debug"]; + + this.baseURI = baseURI; + + if (!baseURI.endsWith("/")) { + this.baseURI += "/"; + } +} + +BagheeraClient.prototype = { + /** + * Channel load flags for all requests. + * + * Caching is not applicable, so we bypass and disable it. We also + * ignore any cookies that may be present for the domain because + * Bagheera does not utilize cookies and the release of cookies may + * inadvertantly constitute unncessary information disclosure. + */ + _loadFlags: Ci.nsIRequest.LOAD_BYPASS_CACHE | + Ci.nsIRequest.INHIBIT_CACHING | + Ci.nsIRequest.LOAD_ANONYMOUS, + + DEFAULT_TIMEOUT_MSEC: 5 * 60 * 1000, // 5 minutes. + + _RE_URI_IDENTIFIER: /^[a-zA-Z0-9_-]+$/, + + /** + * Upload a JSON payload to the server. + * + * The return value is a Promise which will be resolved with a + * BagheeraClientRequestResult when the request has finished. + * + * @param namespace + * (string) The namespace to post this data to. + * @param id + * (string) The ID of the document being uploaded. This is typically + * a UUID in hex form. + * @param payload + * (string|object) Data to upload. Can be specified as a string (which + * is assumed to be JSON) or an object. If an object, it will be fed into + * JSON.stringify() for serialization. + * @param deleteOldID + * (string) Old document ID to delete as part of upload. If not + * specified, no old documents will be deleted as part of upload. The + * string value is typically a UUID in hex form. + * + * @return Promise + */ + uploadJSON: function uploadJSON(namespace, id, payload, deleteOldID=null) { + if (!namespace) { + throw new Error("namespace argument must be defined."); + } + + if (!id) { + throw new Error("id argument must be defined."); + } + + if (!payload) { + throw new Error("payload argument must be defined."); + } + + let uri = this._submitURI(namespace, id); + + let data = payload; + + if (typeof(payload) == "object") { + data = JSON.stringify(payload); + } + + if (typeof(data) != "string") { + throw new Error("Unknown type for payload: " + typeof(data)); + } + + this._log.info("Uploading data to " + uri); + + let request = new RESTRequest(uri); + request.loadFlags = this._loadFlags; + request.timeout = this.DEFAULT_TIMEOUT_MSEC; + + if (deleteOldID) { + request.setHeader("X-Obsolete-Document", deleteOldID); + } + + let deferred = Promise.defer(); + + data = CommonUtils.convertString(data, "uncompressed", "deflate"); + // TODO proper header per bug 807134. + request.setHeader("Content-Type", "application/json+zlib; charset=utf-8"); + + this._log.info("Request body length: " + data.length); + + let result = new BagheeraClientRequestResult(); + result.namespace = namespace; + result.id = id; + + request.onComplete = this._onComplete.bind(this, request, deferred, result); + request.post(data); + + return deferred.promise; + }, + + /** + * Delete the specified document. + * + * @param namespace + * (string) Namespace from which to delete the document. + * @param id + * (string) ID of document to delete. + * + * @return Promise + */ + deleteDocument: function deleteDocument(namespace, id) { + let uri = this._submitURI(namespace, id); + + let request = new RESTRequest(uri); + request.loadFlags = this._loadFlags; + request.timeout = this.DEFAULT_TIMEOUT_MSEC; + + let result = new BagheeraClientRequestResult(); + result.namespace = namespace; + result.id = id; + let deferred = Promise.defer(); + + request.onComplete = this._onComplete.bind(this, request, deferred, result); + request.delete(); + + return deferred.promise; + }, + + _submitURI: function _submitURI(namespace, id) { + if (!this._RE_URI_IDENTIFIER.test(namespace)) { + throw new Error("Illegal namespace name. Must be alphanumeric + [_-]: " + + namespace); + } + + if (!this._RE_URI_IDENTIFIER.test(id)) { + throw new Error("Illegal id value. Must be alphanumeric + [_-]: " + id); + } + + return this.baseURI + "1.0/submit/" + namespace + "/" + id; + }, + + _onComplete: function _onComplete(request, deferred, result, error) { + result.request = request; + + if (error) { + this._log.info("Transport failure on request: " + + CommonUtils.exceptionStr(error)); + result.transportSuccess = false; + deferred.resolve(result); + return; + } + + result.transportSuccess = true; + + let response = request.response; + + switch (response.status) { + case 200: + case 201: + result.serverSuccess = true; + break; + + default: + result.serverSuccess = false; + + this._log.info("Received unexpected status code: " + response.status); + this._log.debug("Response body: " + response.body); + } + + deferred.resolve(result); + }, +}; + +Object.freeze(BagheeraClient.prototype); diff --git a/services/common/modules-testing/bagheeraserver.js b/services/common/modules-testing/bagheeraserver.js new file mode 100644 index 00000000000..35e17bd5291 --- /dev/null +++ b/services/common/modules-testing/bagheeraserver.js @@ -0,0 +1,296 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +"use strict"; + +const {utils: Cu} = Components; + +this.EXPORTED_SYMBOLS = ["BagheeraServer"]; + +Cu.import("resource://gre/modules/services-common/log4moz.js"); +Cu.import("resource://gre/modules/services-common/utils.js"); +Cu.import("resource://testing-common/httpd.js"); + + +/** + * This is an implementation of the Bagheera server. + * + * The purpose of the server is to facilitate testing of the Bagheera + * client and the Firefox Health report. It is *not* meant to be a + * production grade server. + * + * The Bagheera server is essentially a glorified document store. + */ +this.BagheeraServer = function BagheeraServer(serverURI="http://localhost") { + this._log = Log4Moz.repository.getLogger("metrics.BagheeraServer"); + + this.serverURI = serverURI; + this.server = new HttpServer(); + this.port = 8080; + this.namespaces = {}; + + this.allowAllNamespaces = false; +} + +BagheeraServer.prototype = { + /** + * Whether this server has a namespace defined. + * + * @param ns + * (string) Namepsace whose existence to query for. + * @return bool + */ + hasNamespace: function hasNamespace(ns) { + return ns in this.namespaces; + }, + + /** + * Whether this server has an ID in a particular namespace. + * + * @param ns + * (string) Namespace to look for item in. + * @param id + * (string) ID of object to look for. + * @return bool + */ + hasDocument: function hasDocument(ns, id) { + let namespace = this.namespaces[ns]; + + if (!namespace) { + return false; + } + + return id in namespace; + }, + + /** + * Obtain a document from the server. + * + * @param ns + * (string) Namespace to retrieve document from. + * @param id + * (string) ID of document to retrieve. + * + * @return string The content of the document or null if the document + * does not exist. + */ + getDocument: function getDocument(ns, id) { + let namespace = this.namespaces[ns]; + + if (!namespace) { + return null; + } + + return namespace[id]; + }, + + /** + * Set the contents of a document in the server. + * + * @param ns + * (string) Namespace to add document to. + * @param id + * (string) ID of document being added. + * @param payload + * (string) The content of the document. + */ + setDocument: function setDocument(ns, id, payload) { + let namespace = this.namespaces[ns]; + + if (!namespace) { + if (!this.allowAllNamespaces) { + throw new Error("Namespace does not exist: " + ns); + } + + this.createNamespace(ns); + namespace = this.namespaces[ns]; + } + + namespace[id] = payload; + }, + + /** + * Create a namespace in the server. + * + * The namespace will initially be empty. + * + * @param ns + * (string) The name of the namespace to create. + */ + createNamespace: function createNamespace(ns) { + if (ns in this.namespaces) { + throw new Error("Namespace already exists: " + ns); + } + + this.namespaces[ns] = {}; + }, + + start: function start(port) { + if (!port) { + throw new Error("port argument must be specified."); + } + + this.port = port; + + this.server.registerPrefixHandler("/", this._handleRequest.bind(this)); + this.server.start(port); + }, + + stop: function stop(cb) { + let handler = {onStopped: cb}; + + this.server.stop(handler); + }, + + /** + * Our root path handler. + */ + _handleRequest: function _handleRequest(request, response) { + let path = request.path; + this._log.info("Received request: " + request.method + " " + path + " " + + "HTTP/" + request.httpVersion); + + try { + if (path.startsWith("/1.0/submit/")) { + return this._handleV1Submit(request, response, + path.substr("/1.0/submit/".length)); + } else { + throw HTTP_404; + } + } catch (ex) { + if (ex instanceof HttpError) { + this._log.info("HttpError thrown: " + ex.code + " " + ex.description); + } else { + this._log.warn("Exception processing request: " + + CommonUtils.exceptionStr(ex)); + } + + throw ex; + } + }, + + /** + * Handles requests to /submit/*. + */ + _handleV1Submit: function _handleV1Submit(request, response, rest) { + if (!rest.length) { + throw HTTP_404; + } + + let namespace; + let index = rest.indexOf("/"); + if (index == -1) { + namespace = rest; + rest = ""; + } else { + namespace = rest.substr(0, index); + rest = rest.substr(index + 1); + } + + this._handleNamespaceSubmit(namespace, rest, request, response); + }, + + _handleNamespaceSubmit: function _handleNamespaceSubmit(namespace, rest, + request, response) { + if (!this.hasNamespace(namespace)) { + if (!this.allowAllNamespaces) { + this._log.info("Request to unknown namespace: " + namespace); + throw HTTP_404; + } + + this.createNamespace(namespace); + } + + if (!rest) { + this._log.info("No ID defined."); + throw HTTP_404; + } + + let id = rest; + if (id.contains("/")) { + this._log.info("URI has too many components."); + throw HTTP_404; + } + + if (request.method == "POST") { + return this._handleNamespaceSubmitPost(namespace, id, request, response); + } + + if (request.method == "DELETE") { + return this._handleNamespaceSubmitDelete(namespace, id, request, response); + } + + this._log.info("Unsupported HTTP method on namespace handler: " + + request.method); + response.setHeader("Allow", "POST,DELETE"); + throw HTTP_405; + }, + + _handleNamespaceSubmitPost: + function _handleNamespaceSubmitPost(namespace, id, request, response) { + + this._log.info("Handling data upload for " + namespace + ":" + id); + + let requestBody = CommonUtils.readBytesFromInputStream(request.bodyInputStream); + this._log.info("Raw body length: " + requestBody.length); + + if (!request.hasHeader("Content-Type")) { + this._log.info("Request does not have Content-Type header."); + throw HTTP_400; + } + + const ALLOWED_TYPES = [ + // TODO proper content types from bug 807134. + "application/json; charset=utf-8", + "application/json+zlib; charset=utf-8", + ]; + + let ct = request.getHeader("Content-Type"); + if (ALLOWED_TYPES.indexOf(ct) == -1) { + this._log.info("Unknown media type: " + ct); + // Should generate proper HTTP response headers for this error. + throw HTTP_415; + } + + if (ct.startsWith("application/json+zlib")) { + this._log.debug("Uncompressing entity body with deflate."); + requestBody = CommonUtils.convertString(requestBody, "deflate", + "uncompressed"); + } + + this._log.debug("HTTP request body: " + requestBody); + + let doc; + try { + doc = JSON.parse(requestBody); + } catch(ex) { + this._log.info("JSON parse error."); + throw HTTP_400; + } + + this.namespaces[namespace][id] = doc; + + if (request.hasHeader("X-Obsolete-Document")) { + let obsolete = request.getHeader("X-Obsolete-Document"); + delete this.namespaces[namespace][obsolete]; + } + + response.setStatusLine(request.httpVersion, 201, "Created"); + response.setHeader("Content-Type", "text/plain"); + + let body = id; + response.bodyOutputStream.write(body, body.length); + }, + + _handleNamespaceSubmitDelete: + function _handleNamespaceSubmitDelete(namespace, id, request, response) { + + delete this.namespaces[namespace][id]; + + let body = id; + response.bodyOutputStream.write(body, body.length); + }, +}; + +Object.freeze(BagheeraServer.prototype); diff --git a/services/common/tests/run_bagheera_server.js b/services/common/tests/run_bagheera_server.js new file mode 100644 index 00000000000..631cdb84377 --- /dev/null +++ b/services/common/tests/run_bagheera_server.js @@ -0,0 +1,26 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this file, + * You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/** + * This file runs a stub Bagheera server. + * + * It is meant to be executed with an xpcshell. + * + * The Makefile in this directory contains a target to run it: + * + * $ make bagheera-server + */ + +Cu.import("resource://testing-common/services-common/bagheeraserver.js"); + +initTestLogging(); + +let server = new BagheeraServer(); +server.allowAllNamespaces = true; +server.start(SERVER_PORT); +_("Bagheera server started on port " + SERVER_PORT); + +// Launch the thread manager. +_do_main(); + diff --git a/services/common/tests/unit/test_bagheera_client.js b/services/common/tests/unit/test_bagheera_client.js new file mode 100644 index 00000000000..3200fcde779 --- /dev/null +++ b/services/common/tests/unit/test_bagheera_client.js @@ -0,0 +1,89 @@ +/* Any copyright is dedicated to the Public Domain. + * http://creativecommons.org/publicdomain/zero/1.0/ */ + +"use strict"; + +Cu.import("resource://services-common/bagheeraclient.js"); +Cu.import("resource://services-common/rest.js"); +Cu.import("resource://testing-common/services-common/bagheeraserver.js"); + + +const PORT = 8080; + +function getClientAndServer(port=PORT) { + let uri = "http://localhost"; + let server = new BagheeraServer(uri); + + server.start(port); + + let client = new BagheeraClient(uri + ":" + port); + + return [client, server]; +} + +function run_test() { + initTestLogging("Trace"); + run_next_test(); +} + +add_test(function test_constructor() { + let client = new BagheeraClient("http://localhost:8080/"); + + run_next_test(); +}); + +add_test(function test_post_json_transport_failure() { + let client = new BagheeraClient("http://localhost:8080/"); + + client.uploadJSON("foo", "bar", {}).then(function onResult(result) { + do_check_false(result.transportSuccess); + + run_next_test(); + }); +}); + +add_test(function test_post_json_simple() { + let [client, server] = getClientAndServer(); + + server.createNamespace("foo"); + let promise = client.uploadJSON("foo", "bar", {foo: "bar", biz: "baz"}); + + promise.then(function onSuccess(result) { + do_check_true(result instanceof BagheeraClientRequestResult); + do_check_true(result.request instanceof RESTRequest); + do_check_true(result.transportSuccess); + do_check_true(result.serverSuccess); + + server.stop(run_next_test); + }, do_check_null); +}); + +add_test(function test_post_json_bad_data() { + let [client, server] = getClientAndServer(); + + server.createNamespace("foo"); + + client.uploadJSON("foo", "bar", "{this is invalid json}").then( + function onResult(result) { + do_check_true(result.transportSuccess); + do_check_false(result.serverSuccess); + + server.stop(run_next_test); + }); +}); + +add_test(function test_delete_document() { + let [client, server] = getClientAndServer(); + + server.createNamespace("foo"); + server.setDocument("foo", "bar", "{}"); + + client.deleteDocument("foo", "bar").then(function onResult(result) { + do_check_true(result.transportSuccess); + do_check_true(result.serverSuccess); + + do_check_null(server.getDocument("foo", "bar")); + + server.stop(run_next_test); + }); +}); diff --git a/services/common/tests/unit/test_bagheera_server.js b/services/common/tests/unit/test_bagheera_server.js new file mode 100644 index 00000000000..df467e08aa2 --- /dev/null +++ b/services/common/tests/unit/test_bagheera_server.js @@ -0,0 +1,45 @@ +/* Any copyright is dedicated to the Public Domain. + * http://creativecommons.org/publicdomain/zero/1.0/ */ + +"use strict"; + +Cu.import("resource://testing-common/services-common/bagheeraserver.js"); + +const PORT = 8080; + +function run_test() { + run_next_test(); +} + +add_test(function test_server_empty() { + let server = new BagheeraServer(); + + do_check_false(server.hasNamespace("foo")); + do_check_false(server.hasDocument("foo", "bar")); + do_check_null(server.getDocument("foo", "bar")); + + server.createNamespace("foo"); + do_check_true(server.hasNamespace("foo")); + + run_next_test(); +}); + +add_test(function test_server_start_no_port() { + let server = new BagheeraServer(); + + try { + server.start(); + } catch (ex) { + do_check_true(ex.message.startsWith("port argument must be")); + } + + run_next_test(); +}); + +add_test(function test_server_start() { + let server = new BagheeraServer(); + server.start(PORT); + + server.stop(run_next_test); +}); + diff --git a/services/common/tests/unit/test_load_modules.js b/services/common/tests/unit/test_load_modules.js index cf51e86ec07..572a58f017e 100644 --- a/services/common/tests/unit/test_load_modules.js +++ b/services/common/tests/unit/test_load_modules.js @@ -3,6 +3,7 @@ const modules = [ "async.js", + "bagheeraclient.js", "log4moz.js", "preferences.js", "rest.js", @@ -14,6 +15,7 @@ const modules = [ const test_modules = [ "aitcserver.js", + "bagheeraserver.js", "logging.js", "storageserver.js", ]; diff --git a/services/common/tests/unit/xpcshell.ini b/services/common/tests/unit/xpcshell.ini index 5709890119f..9f386c473a1 100644 --- a/services/common/tests/unit/xpcshell.ini +++ b/services/common/tests/unit/xpcshell.ini @@ -22,6 +22,8 @@ tail = [test_aitc_server.js] [test_async_chain.js] [test_async_querySpinningly.js] +[test_bagheera_server.js] +[test_bagheera_client.js] [test_log4moz.js] [test_observers.js] [test_preferences.js]