Bug 872421 - Simple module loader for workers. r=gozala

toolkit/components/moz.build

@@ -41,6 +41,7 @@ PARALLEL_DIRS += [
     'urlformatter',
     'viewconfig',
     'viewsource',
+    'workerloader',
 ]
 
 if CONFIG['MOZ_SOCIAL']:

toolkit/components/workerloader/Makefile.in

@@ -0,0 +1,20 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this file,
+# You can obtain one at http://mozilla.org/MPL/2.0/.
+
+DEPTH     = @DEPTH@
+topsrcdir = @top_srcdir@
+srcdir    = @srcdir@
+VPATH     = @srcdir@
+
+include $(DEPTH)/config/autoconf.mk
+
+WORKER_FILES := require.js \
+	$(NULL)
+
+INSTALL_TARGETS += WORKER
+
+WORKER_DEST = $(FINAL_TARGET)/modules/workers
+
+include $(topsrcdir)/config/rules.mk
+

toolkit/components/workerloader/moz.build

@@ -0,0 +1,10 @@
+# -*- Mode: python; c-basic-offset: 4; indent-tabs-mode: nil; tab-width: 40 -*-
+# vim: set filetype=python:
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+TEST_DIRS += ['tests']
+
+MODULE = 'workerloader'
+

toolkit/components/workerloader/require.js

@@ -0,0 +1,236 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+
+/**
+ * Implementation of a CommonJS module loader for workers.
+ *
+ * Use:
+ * // in the .js file loaded by the constructor of the worker
+ * importScripts("resource://gre/modules/workers/require.js");
+ * let module = require("resource://gre/modules/worker/myModule.js");
+ *
+ * // in myModule.js
+ * // Load dependencies
+ * let SimpleTest = require("resource://gre/modules/workers/SimpleTest.js");
+ * let Logger = require("resource://gre/modules/workers/Logger.js");
+ *
+ * // Define things that will not be exported
+ * let someValue = // ...
+ *
+ * // Export symbols
+ * exports.foo = // ...
+ * exports.bar = // ...
+ *
+ *
+ * Note #1:
+ * Properties |fileName| and |stack| of errors triggered from a module
+ * contain file names that do not correspond to human-readable module paths.
+ * Human readers should rather use properties |moduleName| and |moduleStack|.
+ *
+ * Note #2:
+ * The current version of |require()| only accepts absolute URIs.
+ *
+ * Note #3:
+ * By opposition to some other module loader implementations, this module
+ * loader does not enforce separation of global objects. Consequently, if
+ * a module modifies a global object (e.g. |String.prototype|), all other
+ * modules in the same worker may be affected.
+ */
+
+
+(function(exports) {
+  "use strict";
+
+  if (exports.require) {
+    // Avoid double-imports
+    return;
+  }
+
+  // Simple implementation of |require|
+  let require = (function() {
+
+    /**
+     * Mapping from module paths to module exports.
+     *
+     * @keys {string} The absolute path to a module.
+     * @values {object} The |exports| objects for that module.
+     */
+    let modules = new Map();
+
+    /**
+     * Mapping from object urls to module paths.
+     */
+    let paths = {
+      /**
+       * @keys {string} The object url holding a module.
+       * @values {string} The absolute path to that module.
+       */
+      _map: new Map(),
+      /**
+       * A regexp that may be used to search for all mapped paths.
+       */
+      get regexp() {
+        if (this._regexp) {
+          return this._regexp;
+        }
+        let objectURLs = [];
+        for (let [objectURL, _] of this._map) {
+          objectURLs.push(objectURL);
+        }
+        return this._regexp = new RegExp(objectURLs.join("|"), "g");
+      },
+      _regexp: null,
+      /**
+       * Add a mapping from an object url to a path.
+       */
+      set: function(url, path) {
+        this._regexp = null; // invalidate regexp
+        this._map.set(url, path);
+      },
+      /**
+       * Get a mapping from an object url to a path.
+       */
+      get: function(url) {
+        return this._map.get(url);
+      },
+      /**
+       * Transform a string by replacing all the instances of objectURLs
+       * appearing in that string with the corresponding module path.
+       *
+       * This is used typically to translate exception stacks.
+       *
+       * @param {string} source A source string.
+       * @return {string} The same string as |source|, in which every occurrence
+       * of an objectURL registered in this object has been replaced with the
+       * corresponding module path.
+       */
+      substitute: function(source) {
+        let map = this._map;
+        return source.replace(this.regexp, function(url) {
+          return map.get(url);
+        }, "g");
+      }
+    };
+
+    /**
+     * A human-readable version of |stack|.
+     *
+     * @type {string}
+     */
+    Object.defineProperty(Error.prototype, "moduleStack",
+    {
+      get: function() {
+        return paths.substitute(this.stack);
+      }
+    });
+    /**
+     * A human-readable version of |fileName|.
+     *
+     * @type {string}
+     */
+    Object.defineProperty(Error.prototype, "moduleName",
+    {
+      get: function() {
+        return paths.substitute(this.fileName);
+      }
+    });
+
+    /**
+     * Import a module
+     *
+     * @param {string} path The path to the module.
+     * @return {*} An object containing the properties exported by the module.
+     */
+    return function require(path) {
+      if (typeof path != "string" || path.indexOf("://") == -1) {
+        throw new TypeError("The argument to require() must be a string uri, got " + path);
+      }
+      // Determine uri for the module
+      let uri = path;
+      if (!(uri.endsWith(".js"))) {
+        uri += ".js";
+      }
+
+      // Exports provided by the module
+      let exports = Object.create(null);
+
+      // Identification of the module
+      let module = {
+        id: path,
+        uri: uri,
+        exports: exports
+      };
+
+      // Make module available immediately
+      // (necessary in case of circular dependencies)
+      if (modules.has(path)) {
+        return modules.get(path);
+      }
+      modules.set(path, exports);
+
+
+      // Load source of module, synchronously
+      let xhr = new XMLHttpRequest();
+      xhr.open("GET", uri, false);
+      xhr.responseType = "text";
+      xhr.send();
+
+
+      let source = xhr.responseText;
+      let name = ":" + path;
+      let objectURL;
+      try {
+        if (source == "") {
+          // There doesn't seem to be a better way to detect that the file couldn't be found
+          throw new Error("Could not find module " + path);
+        }
+        // From the source, build a function and an object URL. We
+        // avoid any newline at the start of the file to ensure that
+        // we do not mess up with line numbers. However, using object URLs
+        // messes up with stack traces in instances of Error().
+        source = "require._tmpModules[\"" + name + "\"] = " +
+          "function(exports, require, modules) {" +
+          source +
+        "\n}\n";
+        let blob = new Blob([(new TextEncoder()).encode(source)]);
+        objectURL = URL.createObjectURL(blob);
+        paths.set(objectURL, path);
+        importScripts(objectURL);
+        require._tmpModules[name](exports, require, modules);
+
+      } catch (ex) {
+        // Module loading has failed, exports should not be made available
+        // after all.
+        modules.delete(path);
+        throw ex;
+      } finally {
+        if (objectURL) {
+          // Clean up the object url as soon as possible. It will not be needed.
+          URL.revokeObjectURL(objectURL);
+        }
+        delete require._tmpModules[name];
+      }
+
+      Object.freeze(module.exports);
+      return module.exports;
+    };
+  })();
+
+  /**
+   * An object used to hold temporarily the module constructors
+   * while they are being loaded.
+   *
+   * @keys {string} The path to the module, prefixed with ":".
+   * @values {function} A function wrapping the module.
+   */
+  require._tmpModules = Object.create(null);
+  Object.freeze(require);
+
+  Object.defineProperty(exports, "require", {
+    value: require,
+    enumerable: true,
+    configurable: false
+  });
+})(this);

toolkit/components/workerloader/tests/Makefile.in

@@ -0,0 +1,27 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+DEPTH           = @DEPTH@
+topsrcdir       = @top_srcdir@
+srcdir          = @srcdir@
+VPATH           = @srcdir@
+relativesrcdir  = @relativesrcdir@
+
+include $(DEPTH)/config/autoconf.mk
+
+MOCHITEST_CHROME_FILES := \
+  test_loading.xul \
+  worker_test_loading.js \
+  utils_worker.js \
+  utils_mainthread.js \
+  moduleA-depends.js \
+  moduleB-dependency.js \
+  moduleC-circular.js \
+  moduleD-circular.js \
+  moduleE-throws-during-require.js \
+  moduleF-syntax-error.js \
+  moduleG-throws-later.js \
+  $(NULL)
+
+include $(topsrcdir)/config/rules.mk

toolkit/components/workerloader/tests/moduleA-depends.js

@@ -0,0 +1,14 @@
+/* Any copyright is dedicated to the Public Domain.
+   http://creativecommons.org/publicdomain/zero/1.0/ */
+
+// A trivial module that depends on an equally trivial module
+let B = require("chrome://mochitests/content/chrome/toolkit/components/workerloader/tests/moduleB-dependency.js");
+
+// Ensure that the initial set of exports is empty
+if (Object.keys(exports).length) {
+  throw new Error("exports should be empty, initially");
+}
+
+// Export some values
+exports.A = true;
+exports.importedFoo = B.foo;

toolkit/components/workerloader/tests/moduleB-dependency.js

@@ -0,0 +1,11 @@
+/* Any copyright is dedicated to the Public Domain.
+ * http://creativecommons.org/publicdomain/zero/1.0/ */
+
+exports.B = true;
+exports.foo = "foo";
+
+// Side-effect to detect if we attempt to re-execute this module.
+if ("loadedB" in self) {
+  throw new Error("B has been evaluted twice");
+}
+self.loadedB = true;

toolkit/components/workerloader/tests/moduleC-circular.js

@@ -0,0 +1,18 @@
+/* Any copyright is dedicated to the Public Domain.
+ * http://creativecommons.org/publicdomain/zero/1.0/ */
+
+// Module C and module D have circular dependencies.
+// This should not prevent from loading them.
+
+// This value is set before any circular dependency, it should be visible
+// in D.
+exports.enteredC = true;
+
+let D = require("chrome://mochitests/content/chrome/toolkit/components/workerloader/tests/moduleD-circular.js");
+
+// The following values are set after importing D.
+// copiedFromD.copiedFromC should have only one field |enteredC|
+exports.copiedFromD = JSON.parse(JSON.stringify(D));
+// exportedFromD.copiedFromC should have all the fields defined in |exports|
+exports.exportedFromD = D;
+exports.finishedC = true;

toolkit/components/workerloader/tests/moduleD-circular.js

@@ -0,0 +1,11 @@
+/* Any copyright is dedicated to the Public Domain.
+ * http://creativecommons.org/publicdomain/zero/1.0/ */
+
+// Module C and module D have circular dependencies.
+// This should not prevent from loading them.
+
+exports.enteredD = true;
+let C = require("chrome://mochitests/content/chrome/toolkit/components/workerloader/tests/moduleC-circular.js");
+exports.copiedFromC = JSON.parse(JSON.stringify(C));
+exports.exportedFromC = C;
+exports.finishedD = true;

toolkit/components/workerloader/tests/moduleE-throws-during-require.js

@@ -0,0 +1,10 @@
+/* Any copyright is dedicated to the Public Domain.
+ * http://creativecommons.org/publicdomain/zero/1.0/ */
+
+// Skip a few lines
+// 5
+// 6
+// 7
+// 8
+// 9
+throw new Error("Let's see if this error is obtained with the right origin");

toolkit/components/workerloader/tests/moduleF-syntax-error.js

@@ -0,0 +1,6 @@
+<!--
+Any copyright is dedicated to the Public Domain.
+http://creativecommons.org/publicdomain/zero/1.0/
+-->
+<?xml version="1.0" encoding="UTF-8" ?>
+<foo>Anything that doesn't parse as JavaScript</foo>

toolkit/components/workerloader/tests/moduleG-throws-later.js

@@ -0,0 +1,12 @@
+/* Any copyright is dedicated to the Public Domain.
+ * http://creativecommons.org/publicdomain/zero/1.0/ */
+
+// Skip a few lines
+// 5
+// 6
+// 7
+// 8
+// 9
+exports.doThrow = function doThrow() {
+  Array.prototype.sort.apply("foo"); // This will raise a native TypeError
+};

toolkit/components/workerloader/tests/moz.build

@@ -0,0 +1,8 @@
+# -*- Mode: python; c-basic-offset: 4; indent-tabs-mode: nil; tab-width: 40 -*-
+# vim: set filetype=python:
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+MODULE = 'workerloader'
+

toolkit/components/workerloader/tests/test_loading.xul

@@ -0,0 +1,41 @@
+<?xml version="1.0"?>
+<!--
+  Any copyright is dedicated to the Public Domain.
+  http://creativecommons.org/publicdomain/zero/1.0/
+-->
+<window title="Testing the worker loader"
+        xmlns="http://www.mozilla.org/keymaster/gatekeeper/there.is.only.xul"
+        onload="test();">
+
+  <script type="application/javascript"
+          src="chrome://mochikit/content/tests/SimpleTest/SimpleTest.js"/>
+  <script type="application/javascript"
+          src="chrome://mochikit/content/tests/SimpleTest/EventUtils.js"/>
+  <script type="application/javascript"
+          src="utils_mainthread.js"/>
+  <script type="application/javascript">
+  <![CDATA[
+
+let worker;
+let main = this;
+
+function test() {
+  info("Starting test " + document.uri);
+
+  worker = new ChromeWorker("worker_test_loading.js");
+  SimpleTest.waitForExplicitFinish();
+  info("Chrome worker created");
+  worker_handler(worker);
+  worker.postMessage(document.uri);
+  ok(true, "Test in progress");
+};
+]]>
+  </script>
+
+  <body xmlns="http://www.w3.org/1999/xhtml">
+    <p id="display"></p>
+    <div id="content" style="display:none;"></div>
+    <pre id="test"></pre>
+  </body>
+  <label id="test-result"/>
+</window>

toolkit/components/workerloader/tests/utils_mainthread.js

@@ -0,0 +1,34 @@
+/* Any copyright is dedicated to the Public Domain.
+ * http://creativecommons.org/publicdomain/zero/1.0/ */
+
+function worker_handler(worker) {
+  worker.onerror = function(error) {
+    error.preventDefault();
+    ok(false, "error "+ error.message);
+  };
+  worker.onmessage = function(msg) {
+//    ok(true, "MAIN: onmessage " + JSON.stringify(msg.data));
+    switch (msg.data.kind) {
+    case "is":
+      SimpleTest.ok(msg.data.outcome, msg.data.description +
+         "( "+ msg.data.a + " ==? " + msg.data.b + ")" );
+      return;
+    case "isnot":
+      SimpleTest.ok(msg.data.outcome, msg.data.description +
+      "( "+ msg.data.a + " !=? " + msg.data.b + ")" );
+         return;
+    case "ok":
+      SimpleTest.ok(msg.data.condition, msg.data.description);
+      return;
+    case "info":
+      SimpleTest.info(msg.data.description);
+      return;
+    case "finish":
+      SimpleTest.finish();
+      return;
+    default:
+      SimpleTest.ok(false, "test_osfile.xul: wrong message " + JSON.stringify(msg.data));
+      return;
+    }
+  };
+}

toolkit/components/workerloader/tests/utils_worker.js

@@ -0,0 +1,32 @@
+/* Any copyright is dedicated to the Public Domain.
+ * http://creativecommons.org/publicdomain/zero/1.0/ */
+
+function log(text) {
+  dump("WORKER " + text + "\n");
+}
+
+function send(message) {
+  self.postMessage(message);
+}
+
+function finish() {
+  send({kind: "finish"});
+}
+
+function ok(condition, description) {
+  send({kind: "ok", condition: !!condition, description: "" + description});
+}
+
+function is(a, b, description) {
+  let outcome = a == b; // Need to decide outcome here, as not everything can be serialized
+  send({kind: "is", outcome: outcome, description: "" + description, a: "" + a, b: "" + b});
+}
+
+function isnot(a, b, description) {
+  let outcome = a != b; // Need to decide outcome here, as not everything can be serialized
+  send({kind: "isnot", outcome: outcome, description: "" + description, a: "" + a, b: "" + b});
+}
+
+function info(description) {
+  send({kind: "info", description: "" + description});
+}

toolkit/components/workerloader/tests/worker_handler.js

@@ -0,0 +1,34 @@
+/* Any copyright is dedicated to the Public Domain.
+ * http://creativecommons.org/publicdomain/zero/1.0/ */
+
+function worker_handler(worker) {
+  worker.onerror = function(error) {
+    error.preventDefault();
+    ok(false, "error "+error);
+  }
+  worker.onmessage = function(msg) {
+    ok(true, "MAIN: onmessage " + JSON.stringify(msg.data));
+    switch (msg.data.kind) {
+    case "is":
+      SimpleTest.ok(msg.data.outcome, msg.data.description +
+         "( "+ msg.data.a + " ==? " + msg.data.b + ")" );
+      return;
+    case "isnot":
+      SimpleTest.ok(msg.data.outcome, msg.data.description +
+      "( "+ msg.data.a + " !=? " + msg.data.b + ")" );
+         return;
+    case "ok":
+      SimpleTest.ok(msg.data.condition, msg.data.description);
+      return;
+    case "info":
+      SimpleTest.info(msg.data.description);
+      return;
+    case "finish":
+      SimpleTest.finish();
+      return;
+    default:
+      SimpleTest.ok(false, "test_osfile.xul: wrong message " + JSON.stringify(msg.data));
+      return;
+    }
+  };
+}

toolkit/components/workerloader/tests/worker_test_loading.js

@@ -0,0 +1,96 @@
+/* Any copyright is dedicated to the Public Domain.
+   http://creativecommons.org/publicdomain/zero/1.0/ */
+
+importScripts("utils_worker.js"); // Test suite code
+info("Test suite configured");
+
+importScripts("resource://gre/modules/workers/require.js");
+info("Loader imported");
+
+let tests = [];
+let add_test = function(test) {
+  tests.push(test);
+};
+
+add_test(function test_setup() {
+  ok(typeof require != "undefined", "Function |require| is defined");
+});
+
+// Test simple loading (moduleA-depends.js requires moduleB-dependency.js)
+add_test(function test_load() {
+  let A = require("chrome://mochitests/content/chrome/toolkit/components/workerloader/tests/moduleA-depends.js");
+  ok(true, "Opened module A");
+
+  is(A.A, true, "Module A exported value A");
+  ok(!("B" in A), "Module A did not export value B");
+  is(A.importedFoo, "foo", "Module A re-exported B.foo");
+
+  // re-evaluating moduleB-dependency.js would cause an error, but re-requiring it shouldn't
+  let B = require("chrome://mochitests/content/chrome/toolkit/components/workerloader/tests/moduleB-dependency.js");
+  ok(true, "Managed to re-require module B");
+  is(B.B, true, "Module B exported value B");
+  is(B.foo, "foo", "Module B exported value foo");
+});
+
+// Test simple circular loading (moduleC-circular.js and moduleD-circular.js require each other)
+add_test(function test_circular() {
+  let C = require("chrome://mochitests/content/chrome/toolkit/components/workerloader/tests/moduleC-circular.js");
+  ok(true, "Loaded circular modules C and D");
+  is(C.copiedFromD.copiedFromC.enteredC, true, "Properties exported by C before requiring D can be seen by D immediately");
+
+  let D = require("chrome://mochitests/content/chrome/toolkit/components/workerloader/tests/moduleD-circular.js");
+  is(D.exportedFromC.finishedC, true, "Properties exported by C after requiring D can be seen by D eventually");
+});
+
+// Testing error cases
+add_test(function test_exceptions() {
+  let should_throw = function(f) {
+    try {
+      f();
+      return null;
+    } catch (ex) {
+      return ex;
+    }
+  };
+
+  let exn = should_throw(() => require("chrome://mochitests/content/chrome/toolkit/components/workerloader/tests/this module doesn't exist"));
+  ok(!!exn, "Attempting to load a module that doesn't exist raises an error");
+
+  exn = should_throw(() => require("chrome://mochitests/content/chrome/toolkit/components/workerloader/tests/moduleE-throws-during-require.js"));
+  ok(!!exn, "Attempting to load a module that throws at toplevel raises an error");
+  is(exn.moduleName, "chrome://mochitests/content/chrome/toolkit/components/workerloader/tests/moduleE-throws-during-require.js",
+    "moduleName is correct");
+  isnot(exn.moduleStack.indexOf("moduleE-throws-during-require.js"), -1,
+    "moduleStack contains the name of the module");
+  is(exn.lineNumber, 10, "The error comes with the right line number");
+
+  exn = should_throw(() => require("chrome://mochitests/content/chrome/toolkit/components/workerloader/tests/moduleF-syntaxerror.xml"));
+  ok(!!exn, "Attempting to load a non-well formatted module raises an error");
+
+  exn = should_throw(() => require("chrome://mochitests/content/chrome/toolkit/components/workerloader/tests/moduleG-throws-later.js").doThrow());
+  ok(!!exn, "G.doThrow() has raised an error");
+  info(exn);
+  ok(exn.toString().startsWith("TypeError"), "The exception is a TypeError.");
+  is(exn.moduleName, "chrome://mochitests/content/chrome/toolkit/components/workerloader/tests/moduleG-throws-later.js", "The name of the module is correct");
+  isnot(exn.moduleStack.indexOf("moduleG-throws-later.js"), -1,
+    "The name of the right file appears somewhere in the stack");
+  is(exn.lineNumber, 11, "The error comes with the right line number");
+});
+
+self.onmessage = function(message) {
+  for (let test of tests) {
+    info("Entering " + test.name);
+    try {
+      test();
+    } catch (ex) {
+      ok(false, "Test " + test.name + " failed");
+      info(ex);
+      info(ex.stack);
+    }
+    info("Leaving " + test.name);
+  }
+  finish();
+};
+
+
+