Bug 404077: Add mochitest support (currently semi-disabled) for making tests fail when an unexpected number of assertions fire. r=ted

This adds support for assertion checking in all mochitest suites except
for mochitest-browser-chrome.  The checking works much like it does in
reftest, except for the mechanism for annotating expected assertions,
SimpleTest.expectAssertions() (see its in-code documentation).

The support is initially disabled in that:
 (1) It doesn't cause the tests to report failure (and thus turn the
     tree orange).
 (2) It prints TEST-DETCEPXENU-FAIL/PASS instead of
     TEST-UNEXPECTED-FAIL/PASS (so that it doesn't show up in log
     highlighting).

The assertion checking only works within the test runner (which runs
multiple tests); it does not function when running only a single test.

testing/mochitest/jar.mn

@@ -22,6 +22,7 @@ mochikit.jar:
   content/tests/SimpleTest/SimpleTest.js (tests/SimpleTest/SimpleTest.js)
   content/tests/SimpleTest/test.css (tests/SimpleTest/test.css)
   content/tests/SimpleTest/TestRunner.js (tests/SimpleTest/TestRunner.js)
+  content/tests/SimpleTest/iframe-between-tests.html (tests/SimpleTest/iframe-between-tests.html)
   content/tests/SimpleTest/WindowSnapshot.js (tests/SimpleTest/WindowSnapshot.js)
   content/tests/SimpleTest/MockObjects.js (tests/SimpleTest/MockObjects.js)
   content/tests/SimpleTest/NativeKeyCodes.js (tests/SimpleTest/NativeKeyCodes.js)

testing/mochitest/tests/SimpleTest/Makefile.in

@@ -21,6 +21,7 @@ _SIMPLETEST_FILES =	LogController.js \
 			MockObjects.js \
 			NativeKeyCodes.js \
 			paint_listener.js \
+			iframe-between-tests.html \
 			$(DEPTH)/testing/specialpowers/content/MozillaLogger.js \
 			$(DEPTH)/docshell/test/chrome/docshell_helpers.js \
 			$(NULL)

testing/mochitest/tests/SimpleTest/SimpleTest.js

@@ -468,6 +468,44 @@ SimpleTest.requestLongerTimeout = function (factor) {
     }
 }
 
+/**
+ * Note that the given range of assertions is to be expected.  When
+ * this function is not called, 0 assertions are expected.  When only
+ * one argument is given, that number of assertions are expected.
+ *
+ * A test where we expect to have assertions (which should largely be a
+ * transitional mechanism to get assertion counts down from our current
+ * situation) can call the SimpleTest.expectAssertions() function, with
+ * either one or two arguments:  one argument gives an exact number
+ * expected, and two arguments give a range.  For example, a test might do
+ * one of the following:
+ *
+ *   // Currently triggers two assertions (bug NNNNNN).
+ *   SimpleTest.expectAssertions(2);
+ *
+ *   // Currently triggers one assertion on Mac (bug NNNNNN).
+ *   if (navigator.platform.indexOf("Mac") == 0) {
+ *     SimpleTest.expectAssertions(1);
+ *   }
+ *
+ *   // Currently triggers two assertions on all platforms (bug NNNNNN),
+ *   // but intermittently triggers two additional assertions (bug NNNNNN)
+ *   // on Windows.
+ *   if (navigator.platform.indexOf("Win") == 0) {
+ *     SimpleTest.expectAssertions(2, 4);
+ *   } else {
+ *     SimpleTest.expectAssertions(2);
+ *   }
+ *
+ *   // Intermittently triggers up to three assertions (bug NNNNNN).
+ *   SimpleTest.expectAssertions(0, 3);
+ */
+SimpleTest.expectAssertions = function(min, max) {
+    if (parentRunner) {
+        parentRunner.expectAssertions(min, max);
+    }
+}
+
 SimpleTest.waitForFocus_started = false;
 SimpleTest.waitForFocus_loaded = false;
 SimpleTest.waitForFocus_focused = false;

testing/mochitest/tests/SimpleTest/TestRunner.js

@@ -72,9 +72,13 @@ function flattenArguments(lst/* ...*/) {
 var TestRunner = {};
 TestRunner.logEnabled = false;
 TestRunner._currentTest = 0;
+TestRunner._lastTestFinished = -1;
 TestRunner.currentTestURL = "";
 TestRunner.originalTestURL = "";
 TestRunner._urls = [];
+TestRunner._lastAssertionCount = 0;
+TestRunner._expectedMinAsserts = 0;
+TestRunner._expectedMaxAsserts = 0;
 
 TestRunner.timeout = 5 * 60 * 1000; // 5 minutes.
 TestRunner.maxTimeouts = 4; // halt testing after too many timeouts
@@ -146,6 +150,18 @@ TestRunner.requestLongerTimeout = function(factor) {
 TestRunner.repeat = 0;
 TestRunner._currentLoop = 0;
 
+TestRunner.expectAssertions = function(min, max) {
+    if (typeof(max) == "undefined") {
+        max = min;
+    }
+    if (typeof(min) != "number" || typeof(max) != "number" ||
+        min < 0 || max < min) {
+        throw "bad parameter to expectAssertions";
+    }
+    TestRunner._expectedMinAsserts = min;
+    TestRunner._expectedMaxAsserts = max;
+}
+
 /**
  * This function is called after generating the summary.
 **/
@@ -338,6 +354,8 @@ TestRunner.runNextTest = function() {
 
         TestRunner._currentTestStartTime = new Date().valueOf();
         TestRunner._timeoutFactor = 1;
+        TestRunner._expectedMinAsserts = 0;
+        TestRunner._expectedMaxAsserts = 0;
 
         TestRunner.log("TEST-START | " + url); // used by automation.py
 
@@ -401,6 +419,16 @@ TestRunner.expectChildProcessCrash = function() {
  * This stub is called by SimpleTest when a test is finished.
 **/
 TestRunner.testFinished = function(tests) {
+    // Prevent a test from calling finish() multiple times before we
+    // have a chance to unload it.
+    if (TestRunner._currentTest == TestRunner._lastTestFinished) {
+        TestRunner.error("TEST-UNEXPECTED-FAIL | " +
+                         TestRunner.currentTestURL +
+                         " | called finish() multiple times");
+        return;
+    }
+    TestRunner._lastTestFinished = TestRunner._currentTest;
+
     function cleanUpCrashDumpFiles() {
         if (!SpecialPowers.removeExpectedCrashDumpFiles(TestRunner._expectingProcessCrash)) {
             TestRunner.error("TEST-UNEXPECTED-FAIL | " +
@@ -440,12 +468,14 @@ TestRunner.testFinished = function(tests) {
                        " | finished in " + runtime + "ms");
 
         TestRunner.updateUI(tests);
-        TestRunner._currentTest++;
-        if (TestRunner.runSlower) {
-            setTimeout(TestRunner.runNextTest, 1000);
+
+        var interstitialURL;
+        if ($('testframe').contentWindow.location.protocol == "chrome:") {
+            interstitialURL = "tests/SimpleTest/iframe-between-tests.html";
         } else {
-            TestRunner.runNextTest();
+            interstitialURL = "/tests/SimpleTest/iframe-between-tests.html";
         }
+        TestRunner._makeIframe(interstitialURL, 0);
     }
 
     SpecialPowers.executeAfterFlushingMessageQueue(function() {
@@ -454,6 +484,35 @@ TestRunner.testFinished = function(tests) {
     });
 };
 
+TestRunner.testUnloaded = function() {
+    if (SpecialPowers.isDebugBuild) {
+        var newAssertionCount = SpecialPowers.assertionCount();
+        var numAsserts = newAssertionCount - TestRunner._lastAssertionCount;
+        TestRunner._lastAssertionCount = newAssertionCount;
+
+        var url = TestRunner._urls[TestRunner._currentTest];
+        var max = TestRunner._expectedMaxAsserts;
+        var min = TestRunner._expectedMinAsserts;
+        if (numAsserts > max) {
+            // WHEN ENABLING, change "log" to "error" and "DETCEPXENU"
+            // to "UNEXPECTED".
+            TestRunner.log("TEST-DETCEPXENU-FAIL | " + url + " | Assertion count " + numAsserts + " is greater than expected range " + min + "-" + max + " assertions.");
+        } else if (numAsserts < min) {
+            // WHEN ENABLING, change "log" to "error" and "DETCEPXENU"
+            // to "UNEXPECTED".
+            TestRunner.log("TEST-DETCEPXENU-PASS | " + url + " | Assertion count " + numAsserts + " is less than expected range " + min + "-" + max + " assertions.");
+        } else if (numAsserts > 0) {
+            TestRunner.log("TEST-KNOWN-FAIL | " + url + " | Assertion count " + numAsserts + " within expected range " + min + "-" + max + " assertions.");
+        }
+    }
+    TestRunner._currentTest++;
+    if (TestRunner.runSlower) {
+        setTimeout(TestRunner.runNextTest, 1000);
+    } else {
+        TestRunner.runNextTest();
+    }
+};
+
 /**
  * Get the results.
  */

testing/mochitest/tests/SimpleTest/iframe-between-tests.html

@@ -0,0 +1,12 @@
+<title>iframe for between tests</title>
+<!--
+  This page exists so that our accounting for assertions correctly
+  counts assertions that happen while leaving a page.  We load this page
+  after a test finishes, check the assertion counts, and then go on to
+  load the next.
+-->
+<script>
+window.addEventListener("load", function() {
+  (parent.TestRunner || parent.wrappedJSObject.TestRunner).testUnloaded();
+});
+</script>

testing/specialpowers/content/specialpowersAPI.js

@@ -1206,6 +1206,10 @@ SpecialPowersAPI.prototype = {
     var debug = Cc["@mozilla.org/xpcom/debug;1"].getService(Ci.nsIDebug2);
     return this.isDebugBuild = debug.isDebugBuild;
   },
+  assertionCount: function() {
+    var debugsvc = Cc['@mozilla.org/xpcom/debug;1'].getService(Ci.nsIDebug2);
+    return debugsvc.assertionCount;
+  },
 
   /**
    * Get the message manager associated with an <iframe mozbrowser>.