Bug 1229598 - Add a mode to browser-chrome tests to summarize per-test code coverage. r=ahal

testing/mochitest/browser-test.js

@@ -180,6 +180,8 @@ function Tester(aTests, aDumper, aCallback) {
     this.SimpleTestOriginal[m] = this.SimpleTest[m];
   });
 
+  this._coverageCollector = null;
+
   this._toleratedUncaughtRejections = null;
   this._uncaughtErrorObserver = function({message, date, fileName, stack, lineNumber}) {
     let error = message;
@@ -247,6 +249,13 @@ Tester.prototype = {
     if (gConfig.repeat)
       this.repeat = gConfig.repeat;
 
+    if (gConfig.jscovDirPrefix) {
+      let coveragePath = gConfig.jscovDirPrefix;
+      let {CoverageCollector} = Cu.import("resource://testing-common/CoverageUtils.jsm",
+                                          {});
+      this._coverageCollector = new CoverageCollector(coveragePath);
+    }
+
     this.dumper.structuredLogger.info("*** Start BrowserChrome Test Results ***");
     Services.console.registerListener(this);
     Services.obs.addObserver(this, "chrome-document-global-created", false);
@@ -423,6 +432,9 @@ Tester.prototype = {
   nextTest: Task.async(function*() {
     if (this.currentTest) {
       this.Promise.Debugging.flushUncaughtErrors();
+      if (this._coverageCollector) {
+        this._coverageCollector.recordTestCoverage(this.currentTest.path);
+      }
 
       // Run cleanup functions for the current test before moving on to the
       // next one.
@@ -562,6 +574,9 @@ Tester.prototype = {
     // is invoked to start the tests.
     this.waitForWindowsState((function () {
       if (this.done) {
+        if (this._coverageCollector) {
+          this._coverageCollector.finalize();
+        }
 
         // Uninitialize a few things explicitly so that they can clean up
         // frames and browser intentionally kept alive until shutdown to

testing/mochitest/mochitest_options.py

@@ -401,6 +401,14 @@ class MochitestArguments(ArgumentContainer):
           "default": None,
           "suppress": True,
           }],
+        [["--jscov-dir-prefix"],
+         {"action": "store",
+          "help": "Directory to store per-test line coverage data as json "
+                  "(browser-chrome only). To emit lcov formatted data, set "
+                  "JS_CODE_COVERAGE_OUTPUT_DIR in the environment.",
+          "default": None,
+          "suppress": True,
+          }],
         [["--strict-content-sandbox"],
          {"action": "store_true",
           "default": False,
@@ -683,6 +691,13 @@ class MochitestArguments(ArgumentContainer):
                     "directory for %s does not exist as a destination to copy a "
                     "chrome manifest." % options.store_chrome_manifest)
 
+        if options.jscov_dir_prefix:
+            options.jscov_dir_prefix = os.path.abspath(options.jscov_dir_prefix)
+            if not os.path.isdir(options.jscov_dir_prefix):
+                parser.error(
+                    "directory %s does not exist as a destination for coverage "
+                    "data." % options.jscov_dir_prefix)
+
         if options.testingModulesDir is None:
             if build_obj:
                 options.testingModulesDir = os.path.join(

testing/mochitest/runtests.py

@@ -1158,6 +1158,8 @@ overlay chrome://browser/content/browser.xul chrome://mochikit/content/jetpack-a
         d = dict((k, v) for k, v in options.__dict__.items() if (v is None) or
                  isinstance(v, (basestring, numbers.Number)))
         d['testRoot'] = self.testRoot
+        if options.jscov_dir_prefix:
+            d['jscovDirPrefix'] = options.jscov_dir_prefix;
         if not options.keep_open:
             d['closeWhenDone'] = '1'
         content = json.dumps(d)

testing/modules/CoverageUtils.jsm

@@ -0,0 +1,129 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+"use strict";
+
+this.EXPORTED_SYMBOLS = [
+  "CoverageCollector",
+]
+
+const Cc = Components.classes;
+const Ci = Components.interfaces;
+const Cu = Components.utils;
+
+const {TextEncoder, OS} = Cu.import("resource://gre/modules/osfile.jsm", {});
+const {addDebuggerToGlobal} = Cu.import("resource://gre/modules/jsdebugger.jsm",
+                                        {});
+addDebuggerToGlobal(this);
+
+/**
+ * Records coverage for each test by way of the js debugger.
+ */
+this.CoverageCollector = function (prefix) {
+  this._prefix = prefix;
+  this._dbg = new Debugger();
+  this._dbg.collectCoverageInfo = true;
+  this._dbg.addAllGlobalsAsDebuggees();
+  this._scripts = this._dbg.findScripts();
+
+  this._dbg.onNewScript = (script) => {
+    this._scripts.push(script);
+  };
+
+  // Source -> coverage data;
+  this._allCoverage = {};
+  this._encoder = new TextEncoder();
+  this._testIndex = 0;
+}
+
+CoverageCollector.prototype._getLinesCovered = function () {
+  let coveredLines = {};
+  let currentCoverage = {};
+  this._scripts.forEach(s => {
+    let scriptName = s.url;
+    let cov = s.getOffsetsCoverage();
+    if (!cov) {
+      return;
+    }
+
+    cov.forEach(covered => {
+      let {lineNumber, columnNumber, offset, count} = covered;
+      if (!count) {
+        return;
+      }
+
+      if (!currentCoverage[scriptName]) {
+        currentCoverage[scriptName] = {};
+      }
+      if (!this._allCoverage[scriptName]) {
+        this._allCoverage[scriptName] = {};
+      }
+
+      let key = [lineNumber, columnNumber, offset].join('#');
+      if (!currentCoverage[scriptName][key]) {
+        currentCoverage[scriptName][key] = count;
+      } else {
+        currentCoverage[scriptName][key] += count;
+      }
+    });
+
+  });
+
+  // Covered lines are determined by comparing every offset mentioned as of the
+  // the completion of a test to the last time we measured coverage. If an
+  // offset in a line is novel as of this test, or a count has increased for
+  // any offset on a particular line, that line must have been covered.
+  for (let scriptName in currentCoverage) {
+    for (let key in currentCoverage[scriptName]) {
+      if (!this._allCoverage[scriptName] ||
+          !this._allCoverage[scriptName][key] ||
+          (this._allCoverage[scriptName][key] <
+           currentCoverage[scriptName][key])) {
+        let [lineNumber, colNumber, offset] = key.split('#');
+        if (!coveredLines[scriptName]) {
+          coveredLines[scriptName] = new Set();
+        }
+        coveredLines[scriptName].add(parseInt(lineNumber, 10));
+        this._allCoverage[scriptName][key] = currentCoverage[scriptName][key];
+      }
+    }
+  }
+
+  return coveredLines;
+}
+
+
+/**
+ * Records lines covered since the last time coverage was recorded,
+ * associating them with the given test name. The result is written
+ * to a json file in a specified directory.
+ */
+CoverageCollector.prototype.recordTestCoverage = function (testName) {
+  dump("Collecting coverage for: " + testName + "\n");
+  let rawLines = this._getLinesCovered(testName);
+  let result = [];
+  for (let scriptName in rawLines) {
+    let rec = {
+      testUrl: testName,
+      sourceFile: scriptName,
+      covered: []
+    };
+    for (let line of rawLines[scriptName]) {
+      rec.covered.push(line);
+    }
+    result.push(rec);
+  }
+  let arr = this._encoder.encode(JSON.stringify(result, null, 2));
+  let path = this._prefix + '/' + 'jscov_' + Date.now() + '.json';
+  dump("Writing coverage to: " + path + "\n");
+  return OS.File.writeAtomic(path, arr, {tmpPath: path + '.tmp'});
+}
+
+/**
+ * Tear down the debugger after all tests are complete.
+ */
+CoverageCollector.prototype.finalize = function () {
+  this._dbg.removeAllDebuggees();
+  this._dbg.enabled = false;
+}

testing/modules/moz.build

@@ -11,6 +11,7 @@ TESTING_JS_MODULES += [
     'AppData.jsm',
     'AppInfo.jsm',
     'Assert.jsm',
+    'CoverageUtils.jsm',
     'MockRegistrar.jsm',
     'StructuredLog.jsm',
     'TestUtils.jsm',

testing/mozharness/configs/unittests/linux_unittest.py

@@ -187,6 +187,7 @@ config = {
         "browser-chrome": ["--browser-chrome"],
         "browser-chrome-chunked": ["--browser-chrome", "--chunk-by-runtime"],
         "browser-chrome-addons": ["--browser-chrome", "--chunk-by-runtime", "--tag=addons"],
+        "browser-chrome-coverage": ["--timeout=1200"],
         "mochitest-gl": ["--subsuite=webgl"],
         "mochitest-devtools-chrome": ["--browser-chrome", "--subsuite=devtools"],
         "mochitest-devtools-chrome-chunked": ["--browser-chrome", "--subsuite=devtools", "--chunk-by-runtime"],

testing/mozharness/scripts/desktop_unittest.py

@@ -384,6 +384,10 @@ class DesktopUnittest(TestingMixin, MercurialScript, BlobUploadMixin, MozbaseMix
             if suite_category not in c["suite_definitions"]:
                 self.fatal("'%s' not defined in the config!")
 
+            if suite == 'browser-chrome-coverage':
+                base_cmd.append('--jscov-dir-prefix=%s' %
+                                dirs['abs_blob_upload_dir'])
+
             options = c["suite_definitions"][suite_category]["options"]
             if options:
                 for option in options: