Bug 1199800 - [webext] Allow extensions to be generated from JSON (r=gabor)

testing/mochitest/tests/SimpleTest/ExtensionTestUtils.js

@@ -1,6 +1,6 @@
 var ExtensionTestUtils = {};
 
-ExtensionTestUtils.loadExtension = function(name)
+ExtensionTestUtils.loadExtension = function(ext)
 {
   var testResolve;
   var testDone = new Promise(resolve => { testResolve = resolve; });
@@ -37,7 +37,7 @@ ExtensionTestUtils.loadExtension = function(name)
     },
   };
 
-  var extension = SpecialPowers.loadExtension(name, handler);
+  var extension = SpecialPowers.loadExtension(ext, handler);
 
   extension.awaitMessage = (msg) => {
     return new Promise(resolve => {

testing/specialpowers/content/SpecialPowersObserverAPI.js

@@ -560,17 +560,21 @@ SpecialPowersObserverAPI.prototype = {
         let {Extension} = Components.utils.import("resource://gre/modules/Extension.jsm", {});
 
         let id = aMessage.data.id;
-        let name = aMessage.data.name;
-
-        let target = "resource://testing-common/extensions/" + name + "/";
-        let resourceHandler = Services.io.getProtocolHandler("resource")
-                                      .QueryInterface(Ci.nsISubstitutingProtocolHandler);
-        let resURI = Services.io.newURI(target, null, null);
-        let uri = Services.io.newURI(resourceHandler.resolveURI(resURI), null, null);
-        let extension = new Extension({
-          id,
-          resourceURI: uri
-        });
+        let ext = aMessage.data.ext;
+        let extension;
+        if (typeof(ext) == "string") {
+          let target = "resource://testing-common/extensions/" + ext + "/";
+          let resourceHandler = Services.io.getProtocolHandler("resource")
+                                        .QueryInterface(Ci.nsISubstitutingProtocolHandler);
+          let resURI = Services.io.newURI(target, null, null);
+          let uri = Services.io.newURI(resourceHandler.resolveURI(resURI), null, null);
+          extension = new Extension({
+            id,
+            resourceURI: uri
+          });
+        } else {
+          extension = Extension.generate(ext);
+        }
 
         let resultListener = (...args) => {
           this._sendReply(aMessage, "SPExtensionMessage", {id, type: "testResult", args});

testing/specialpowers/content/specialpowersAPI.js

@@ -2044,7 +2044,7 @@ SpecialPowersAPI.prototype = {
     this.notifyObserversInParentProcess(null, "browser:purge-domain-data", "example.com");
   },
 
-  loadExtension: function(name, handler) {
+  loadExtension: function(ext, handler) {
     let uuidGenerator = Cc["@mozilla.org/uuid-generator;1"].getService(Ci.nsIUUIDGenerator);
     let id = uuidGenerator.generateUUID().number;
 
@@ -2056,6 +2056,7 @@ SpecialPowersAPI.prototype = {
     let unloadPromise = new Promise(resolve => { resolveUnload = resolve; });
 
     handler = Cu.waiveXrays(handler);
+    ext = Cu.waiveXrays(ext);
 
     let sp = this;
     let extension = {
@@ -2074,13 +2075,13 @@ SpecialPowersAPI.prototype = {
       },
     };
 
-    this._sendAsyncMessage("SPLoadExtension", {name, id});
+    this._sendAsyncMessage("SPLoadExtension", {ext, id});
 
     let listener = (msg) => {
       if (msg.data.id == id) {
         if (msg.data.type == "extensionStarted") {
           resolveStartup();
-        } if (msg.data.type == "extensionFailed") {
+        } else if (msg.data.type == "extensionFailed") {
           rejectStartup("startup failed");
         } else if (msg.data.type == "extensionUnloaded") {
           this._removeMessageListener("SPExtensionMessage", listener);

toolkit/components/extensions/Extension.jsm

@@ -29,6 +29,8 @@ XPCOMUtils.defineLazyModuleGetter(this, "MatchPattern",
                                   "resource://gre/modules/MatchPattern.jsm");
 XPCOMUtils.defineLazyModuleGetter(this, "NetUtil",
                                   "resource://gre/modules/NetUtil.jsm");
+XPCOMUtils.defineLazyModuleGetter(this, "FileUtils",
+                                  "resource://gre/modules/FileUtils.jsm");
 
 Cu.import("resource://gre/modules/ExtensionManagement.jsm");
 
@@ -51,6 +53,7 @@ let {
   MessageBroker,
   Messenger,
   injectAPI,
+  flushJarCache,
 } = ExtensionUtils;
 
 let scriptScope = this;
@@ -316,6 +319,12 @@ this.Extension = function(addonData)
   uuid = uuid.substring(1, uuid.length - 1); // Strip of { and } off the UUID.
   this.uuid = uuid;
 
+  if (addonData.cleanupFile) {
+    Services.obs.addObserver(this, "xpcom-shutdown", false);
+    this.cleanupFile = addonData.cleanupFile || null;
+    delete addonData.cleanupFile;
+  }
+
   this.addonData = addonData;
   this.id = addonData.id;
   this.baseURI = Services.io.newURI("moz-extension://" + uuid, null, null);
@@ -336,6 +345,125 @@ this.Extension = function(addonData)
   this.emitter = new EventEmitter();
 }
 
+/**
+ * This code is designed to make it easy to test a WebExtension
+ * without creating a bunch of files. Everything is contained in a
+ * single JSON blob.
+ *
+ * Properties:
+ *   "background": "<JS code>"
+ *     A script to be loaded as the background script.
+ *     The "background" section of the "manifest" property is overwritten
+ *     if this is provided.
+ *   "manifest": {...}
+ *     Contents of manifest.json
+ *   "files": {"filename1": "contents1", ...}
+ *     Data to be included as files. Can be referenced from the manifest.
+ *     If a manifest file is provided here, it takes precedence over
+ *     a generated one. Always use "/" as a directory separator.
+ *     Directories should appear here only implicitly (as a prefix
+ *     to file names)
+ *
+ * To make things easier, the value of "background" and "files"[] can
+ * be a function, which is converted to source that is run.
+ */
+this.Extension.generate = function(data)
+{
+  let manifest = data.manifest;
+  if (!manifest) {
+    manifest = {};
+  }
+
+  let files = data.files;
+  if (!files) {
+    files = {};
+  }
+
+  function provide(obj, keys, value, override = false) {
+    if (keys.length == 1) {
+      if (!(keys[0] in obj) || override) {
+        obj[keys[0]] = value;
+      }
+    } else {
+      if (!(keys[0] in obj)) {
+        obj[keys[0]] = {};
+      }
+      provide(obj[keys[0]], keys.slice(1), value, override);
+    }
+  }
+
+  let uuidGenerator = Cc["@mozilla.org/uuid-generator;1"].getService(Ci.nsIUUIDGenerator);
+  let uuid = uuidGenerator.generateUUID().number;
+  provide(manifest, ["applications", "gecko", "id"], uuid);
+
+  provide(manifest, ["name"], "Generated extension");
+  provide(manifest, ["manifest_version"], 2);
+  provide(manifest, ["version"], "1.0");
+
+  if (data.background) {
+    let uuidGenerator = Cc["@mozilla.org/uuid-generator;1"].getService(Ci.nsIUUIDGenerator);
+    let bgScript = uuidGenerator.generateUUID().number + ".js";
+
+    provide(manifest, ["background", "scripts"], [bgScript], true);
+    files[bgScript] = data.background;
+  }
+
+  provide(files, ["manifest.json"], JSON.stringify(manifest));
+
+  let ZipWriter = Components.Constructor("@mozilla.org/zipwriter;1", "nsIZipWriter");
+  let zipW = new ZipWriter();
+
+  let file = FileUtils.getFile("TmpD", ["generated-extension.xpi"]);
+  file.createUnique(Ci.nsIFile.NORMAL_FILE_TYPE, FileUtils.PERMS_FILE);
+
+  const MODE_WRONLY = 0x02;
+  const MODE_TRUNCATE = 0x20;
+  zipW.open(file, MODE_WRONLY | MODE_TRUNCATE);
+
+  // Needs to be in microseconds for some reason.
+  let time = Date.now() * 1000;
+
+  function generateFile(filename) {
+    let components = filename.split("/");
+    let path = "";
+    for (let component of components.slice(0, -1)) {
+      path += component;
+      if (!zipW.hasEntry(path)) {
+        zipW.addEntryDirectory(path, time, false);
+      }
+
+      path += "/";
+    }
+  }
+
+  for (let filename in files) {
+    let script = files[filename];
+    if (typeof(script) == "function") {
+      script = "(" + script.toString() + ")()";
+    }
+
+    let stream = Cc["@mozilla.org/io/string-input-stream;1"].createInstance(Ci.nsIStringInputStream);
+    stream.data = script;
+
+    generateFile(filename);
+    zipW.addEntryStream(filename, time, 0, stream, false);
+  }
+
+  zipW.close();
+
+  flushJarCache(file);
+  Services.ppmm.broadcastAsyncMessage("Extension:FlushJarCache", {path: file.path});
+
+  let fileURI = Services.io.newFileURI(file);
+  let jarURI = Services.io.newURI("jar:" + fileURI.spec + "!/", null, null);
+
+  return new Extension({
+    id: uuid,
+    resourceURI: jarURI,
+    cleanupFile: file
+  });
+}
+
 Extension.prototype = {
   on(hook, f) {
     return this.emitter.on(hook, f);
@@ -578,6 +706,31 @@ Extension.prototype = {
     });
   },
 
+  cleanupGeneratedFile() {
+    if (!this.cleanupFile) {
+      return;
+    }
+
+    let file = this.cleanupFile;
+    this.cleanupFile = null;
+
+    Services.obs.removeObserver(this, "xpcom-shutdown");
+
+    let count = Services.ppmm.childCount;
+
+    Services.ppmm.addMessageListener("Extension:FlushJarCacheComplete", function listener() {
+      count--;
+      if (count == 0) {
+        // We can't delete this file until everyone using it has
+        // closed it (because Windows is dumb). So we wait for all the
+        // child processes (including the parent) to flush their JAR
+        // caches. These caches may keep the file open.
+        file.remove(false);
+      }
+    });
+    Services.ppmm.broadcastAsyncMessage("Extension:FlushJarCache", {path: file.path});
+  },
+
   shutdown() {
     this.hasShutdown = true;
     if (!this.manifest) {
@@ -599,6 +752,15 @@ Extension.prototype = {
     Services.ppmm.broadcastAsyncMessage("Extension:Shutdown", {id: this.id});
 
     ExtensionManagement.shutdownExtension(this.uuid);
+
+    // Clean up a generated file.
+    this.cleanupGeneratedFile();
+  },
+
+  observe(subject, topic, data) {
+    if (topic == "xpcom-shutdown") {
+      this.cleanupGeneratedFile();
+    }
   },
 
   hasPermission(perm) {

toolkit/components/extensions/ExtensionContent.jsm

@@ -35,6 +35,7 @@ let {
   Messenger,
   ignoreEvent,
   injectAPI,
+  flushJarCache,
 } = ExtensionUtils;
 
 function isWhenBeforeOrSame(when1, when2)
@@ -461,6 +462,7 @@ let ExtensionManager = {
   init() {
     Services.cpmm.addMessageListener("Extension:Startup", this);
     Services.cpmm.addMessageListener("Extension:Shutdown", this);
+    Services.cpmm.addMessageListener("Extension:FlushJarCache", this);
 
     if (Services.cpmm.initialProcessData && "Extension:Extensions" in Services.cpmm.initialProcessData) {
       let extensions = Services.cpmm.initialProcessData["Extension:Extensions"];
@@ -478,18 +480,29 @@ let ExtensionManager = {
   receiveMessage({name, data}) {
     let extension;
     switch (name) {
-    case "Extension:Startup":
-      extension = new BrowserExtensionContent(data);
-      this.extensions.set(data.id, extension);
-      DocumentManager.startupExtension(data.id);
-      break;
+      case "Extension:Startup": {
+        extension = new BrowserExtensionContent(data);
+        this.extensions.set(data.id, extension);
+        DocumentManager.startupExtension(data.id);
+        break;
+      }
 
-    case "Extension:Shutdown":
-      extension = this.extensions.get(data.id);
-      extension.shutdown();
-      DocumentManager.shutdownExtension(data.id);
-      this.extensions.delete(data.id);
-      break;
+      case "Extension:Shutdown": {
+        extension = this.extensions.get(data.id);
+        extension.shutdown();
+        DocumentManager.shutdownExtension(data.id);
+        this.extensions.delete(data.id);
+        break;
+      }
+
+      case "Extension:FlushJarCache": {
+        let nsIFile = Components.Constructor("@mozilla.org/file/local;1", "nsIFile",
+                                             "initWithPath");
+        let file = new nsIFile(data.path);
+        flushJarCache(file);
+        Services.cpmm.sendAsyncMessage("Extension:FlushJarCacheComplete");
+        break;
+      }
     }
   }
 };

toolkit/components/extensions/ExtensionUtils.jsm

@@ -527,6 +527,11 @@ Messenger.prototype = {
   },
 };
 
+function flushJarCache(jarFile)
+{
+  Services.obs.notifyObservers(jarFile, "flush-cache-entry", null);
+}
+
 this.ExtensionUtils = {
   runSafeWithoutClone,
   runSafe,
@@ -537,5 +542,6 @@ this.ExtensionUtils = {
   injectAPI,
   MessageBroker,
   Messenger,
+  flushJarCache,
 };
 

toolkit/components/extensions/test/mochitest/mochitest.ini

@@ -18,3 +18,4 @@ support-files =
 [test_simple_extensions.html]
 [test_extension_contentscript.html]
 [test_extension_webrequest.html]
+[test_generate_extension.html]

toolkit/components/extensions/test/mochitest/test_generate_extension.html

@@ -0,0 +1,48 @@
+<!DOCTYPE HTML>
+<html>
+<head>
+  <title>Test for generating WebExtensions</title>
+  <script type="text/javascript" src="/tests/SimpleTest/SimpleTest.js"></script>
+  <script type="text/javascript" src="/tests/SimpleTest/SpawnTask.js"></script>
+  <script type="text/javascript" src="/tests/SimpleTest/ExtensionTestUtils.js"></script>
+  <link rel="stylesheet" type="text/css" href="/tests/SimpleTest/test.css"/>
+</head>
+<body>
+
+<script type="application/javascript;version=1.8">
+
+function backgroundScript() {
+  browser.test.log("running background script");
+
+  browser.test.onMessage.addListener((x, y) => {
+    browser.test.assertEq(x, 10, "x is 10");
+    browser.test.assertEq(y, 20, "y is 20");
+
+    browser.test.notifyPass("background test passed");
+  });
+
+  browser.test.sendMessage("running", 1);
+}
+
+let extensionData = {
+  background: "(" + backgroundScript.toString() + ")()"
+};
+
+add_task(function* test_background() {
+  let extension = ExtensionTestUtils.loadExtension(extensionData);
+  info("load complete");
+  yield extension.startup();
+  let x = yield extension.awaitMessage("running");
+  is(x, 1, "got correct value from extension");
+  info("startup complete");
+  extension.sendMessage(10, 20);
+  yield extension.awaitFinish();
+  info("test complete");
+  yield extension.unload();
+  info("extension unloaded successfully");
+});
+
+</script>
+
+</body>
+</html>