Bug 647323 - Part b: Implement test harness reporting code and import testharness code from the upstream repository; r=jhammel+ted

dom/tests/mochitest/Makefile.in

@@ -57,6 +57,7 @@ DIRS	+= \
 	orientation \
 	sessionstorage \
 	storageevent \
+	w3c \
 	browser-frame \
 	$(NULL)
 

dom/tests/mochitest/w3c/Makefile.in

@@ -0,0 +1,29 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this file,
+# You can obtain one at http://mozilla.org/MPL/2.0/.
+
+DEPTH = ../../../..
+topsrcdir = @top_srcdir@
+srcdir = @srcdir@
+VPATH = @srcdir@
+relativesrcdir = dom/tests/mochitest/w3c
+
+DIRS = \
+  $(NULL)
+
+include $(DEPTH)/config/autoconf.mk
+include $(topsrcdir)/config/rules.mk
+
+_SUPPORT_FILES = \
+  testharness.js \
+  testharnessreport.js \
+  testharness.css \
+  idlharness.js \
+  WebIDLParser.js \
+  $(NULL)
+
+testharnessreport.js: testharnessreport.js.in writeReporter.py 
+	$(PYTHON_PATH) $(srcdir)/writeReporter.py $<
+
+libs:: $(_SUPPORT_FILES)
+	$(INSTALL) $(foreach f,$^,"$f") $(DEPTH)/_tests/testing/mochitest/resources

dom/tests/mochitest/w3c/testharness.css

@@ -0,0 +1,70 @@
+html {
+    font-family:DejaVu Sans, Bitstream Vera Sans, Arial, Sans;
+}
+
+section#summary {
+    margin-bottom:1em;
+}
+
+table#results {
+    border-collapse:collapse;
+    table-layout:fixed;
+    width:100%;
+}
+
+table#results th:first-child,
+table#results td:first-child {
+    width:4em;
+}
+
+table#results th:last-child,
+table#results td:last-child {
+    width:50%;
+}
+
+table#results th {
+    padding:0;
+    padding-bottom:0.5em;
+    border-bottom:medium solid black;
+}
+
+table#results td {
+    padding:1em;
+    padding-bottom:0.5em;
+    border-bottom:thin solid black;
+}
+
+tr.pass > td:first-child {
+    color:green;
+}
+
+tr.fail > td:first-child {
+    color:red;
+}
+
+tr.timeout > td:first-child {
+    color:red;
+}
+
+tr.notrun > td:first-child {
+    color:blue;
+}
+
+.pass .fail .timeout .notrun > td:first-child {
+    font-variant:small-caps;
+}
+
+table#results span {
+    display:block;
+}
+
+table#results span.expected {
+    font-family:DejaVu Sans Mono, Bitstream Vera Sans Mono, Monospace;
+    white-space:pre;
+}
+
+table#results span.actual {
+    font-family:DejaVu Sans Mono, Bitstream Vera Sans Mono, Monospace;
+    white-space:pre;
+}
+

dom/tests/mochitest/w3c/testharnessreport.js.in

@@ -0,0 +1,120 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this file,
+ * You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+var W3CTest = {
+  /**
+   * Dictionary mapping a test URL to either the string "all", which means that
+   * all tests in this file are expected to fail, or a dictionary mapping test
+   * names to either the boolean |true|, or the string "debug". The former
+   * means that this test is expected to fail in all builds, and the latter
+   * that it is only expected to fail in debug builds.
+   *
+   * This is filled in by the writeReporter.py script.
+   */
+  "expectedFailures": ${expectations},
+
+  /**
+   * List of test results, needed by TestRunner to update the UI.
+   */
+  "tests": [],
+
+  /**
+   * Reference to the TestRunner object in the parent frame.
+   */
+  "runner": parent === this ? null : parent.TestRunner || parent.wrappedJSObject.TestRunner,
+
+  /**
+   * Prefixes for the error logging. Indexed first by int(todo) and second by
+   * int(result).
+   */
+  "prefixes": [
+    ["TEST-UNEXPECTED-FAIL", "TEST-PASS"],
+    ["TEST-KNOWN-FAIL", "TEST-UNEXPECTED-PASS"]
+  ],
+
+  /**
+   * Returns the URL of the current test, relative to the root W3C tests
+   * directory. Used as a key into the expectedFailures dictionary.
+   */
+  "getURL": function() {
+    return this.runner.currentTestURL.substring("/tests/dom/tests/mochitest/w3c/".length);
+  },
+
+  /**
+   * Lets the test runner know about a test result.
+   */
+  "_log": function(test) {
+    var msg = this.prefixes[+test.todo][+test.result] + " | ";
+    if (this.runner.currentTestURL)
+      msg += this.runner.currentTestURL;
+    msg += " | " + test.message;
+    this.runner[(test.result === !test.todo) ? "log" : "error"](msg);
+  },
+
+  /**
+   * Reports a test result. The argument is an object with the following
+   * properties:
+   *
+   * o message (string): message to be reported
+   * o result (boolean): whether this test failed
+   * o todo (boolean): whether this test is expected to fail
+   */
+  "report": function(test) {
+    this.tests.push(test);
+    this._log(test);
+  },
+
+  /**
+   * Returns true if this test is expected to fail, and false otherwise.
+   */
+  "_todo": function(url, test) {
+    if (!(url in this.expectedFailures)) {
+      return false;
+    }
+    if (this.expectedFailures[url] === "all") {
+      return true;
+    }
+    var value = this.expectedFailures[url][test.name];
+    return value === true || (value === "debug" && !!SpecialPowers.isDebugBuild);
+  },
+
+  /**
+   * Callback function for testharness.js. Called when one test in a file
+   * finishes.
+   */
+  "result": function(test) {
+    var url = this.getURL();
+    this.report({
+      "message": test.message || test.name,
+      "result": test.status === test.PASS,
+      "todo": this._todo(url, test)
+    });
+  },
+
+  /**
+   * Callback function for testharness.js. Called when the entire test file
+   * finishes.
+   */
+  "finish": function(tests, status) {
+    var url = this.getURL();
+    this.report({
+      "message": "Finished test",
+      "result": status.status === status.OK,
+      "todo":
+        url in this.expectedFailures &&
+        this.expectedFailures[url] === "error"
+    });
+    this.runner.testFinished(this.tests);
+  }
+};
+(function() {
+  if (!W3CTest.runner) {
+    return;
+  }
+  add_result_callback(W3CTest.result.bind(W3CTest));
+  add_completion_callback(W3CTest.finish.bind(W3CTest));
+  setup({
+    "output": false
+  });
+})();

dom/tests/mochitest/w3c/updateTestharness.py

@@ -0,0 +1,16 @@
+#!/usr/bin/env python
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this file,
+# You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import subprocess
+
+repo = "https://dvcs.w3.org/hg/resources"
+dest = "resources-upstream"
+files = ["testharness.js", "testharness.css", "idlharness.js", "WebIDLParser.js"]
+
+subprocess.check_call(["hg", "clone", repo, dest])
+for f in files:
+  subprocess.check_call(["cp", "%s/%s" % (dest, f), f])
+  subprocess.check_call(["hg", "add", f])
+subprocess.check_call(["rm", "--recursive", "--force", dest])

dom/tests/mochitest/w3c/writeReporter.py

@@ -0,0 +1,36 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this file,
+# You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import os
+import sys
+import string
+
+try:
+  import json
+except ImportError:
+  import simplejson as json
+
+def writeReporter(src):
+  src = src.replace("testharnessreport.js.in", "")
+  expectations = {}
+  fp = open(src + "failures.txt", "rb")
+  for line in fp:
+    gp = open(src + line.strip() + ".json")
+    expectations.update(json.load(gp))
+    gp.close()
+  fp.close()
+  fp = open(src + "testharnessreport.js.in", "rb")
+  template = fp.read()
+  fp.close()
+  fp = open("testharnessreport.js", "wb")
+  expjson = json.dumps(expectations,
+                       indent = 2,
+                       sort_keys = True,
+                       separators = (',', ': '))
+  result = string.Template(template).substitute({ 'expectations': expjson })
+  fp.write(result)
+  fp.close()
+
+if __name__ == "__main__":
+  writeReporter(sys.argv[1])

testing/mochitest/tests/SimpleTest/TestRunner.js

@@ -89,12 +89,31 @@ TestRunner._currentTestStartTime = new Date().valueOf();
 TestRunner._timeoutFactor = 1;
 
 TestRunner._checkForHangs = function() {
+  function reportError(win, msg) {
+    if ("SimpleTest" in win) {
+      win.SimpleTest.ok(false, "Test timed out.");
+    } else if ("W3CTest" in win) {
+      win.W3CTest.report({
+        "message": msg,
+        "result": false,
+        "todo": false
+      });
+    }
+  }
+  function killTest(win) {
+    if ("SimpleTest" in win) {
+      win.SimpleTest.finish();
+    } else if ("W3CTest" in win) {
+      win.W3CTest.kill();
+    }
+  }
+
   if (TestRunner._currentTest < TestRunner._urls.length) {
     var runtime = new Date().valueOf() - TestRunner._currentTestStartTime;
     if (runtime >= TestRunner.timeout * TestRunner._timeoutFactor) {
       var frameWindow = $('testframe').contentWindow.wrappedJSObject ||
                           $('testframe').contentWindow;
-      frameWindow.SimpleTest.ok(false, "Test timed out.");
+      reportError(frameWindow, "Test timed out.");
 
       // If we have too many timeouts, give up. We don't want to wait hours
       // for results if some bug causes lots of tests to time out.
@@ -102,12 +121,12 @@ TestRunner._checkForHangs = function() {
         TestRunner._haltTests = true;
 
         TestRunner.currentTestURL = "(SimpleTest/TestRunner.js)";
-        frameWindow.SimpleTest.ok(false, TestRunner.maxTimeouts + " test timeouts, giving up.");
+        reportError(frameWindow, TestRunner.maxTimeouts + " test timeouts, giving up.");
         var skippedTests = TestRunner._urls.length - TestRunner._currentTest;
-        frameWindow.SimpleTest.ok(false, "Skipping " + skippedTests + " remaining tests.");
+        reportError(frameWindow, "Skipping " + skippedTests + " remaining tests.");
       }
 
-      frameWindow.SimpleTest.finish();
+      killTest(frameWindow);
 
       if (TestRunner._haltTests)
         return;

testing/mochitest/tests/SimpleTest/specialpowersAPI.js

@@ -1070,4 +1070,10 @@ SpecialPowersAPI.prototype = {
     var el = this._getElement(aWindow, target);
     return el.dispatchEvent(event);
   },
+
+  get isDebugBuild() {
+    delete this.isDebugBuild;
+    var debug = Cc["@mozilla.org/xpcom/debug;1"].getService(Ci.nsIDebug2);
+    return this.isDebugBuild = debug.isDebugBuild;
+  }
 };