Bug 837950 - Enhance jstests framework to support Test402 tests. r=terrence

--HG--
rename : toolkit/mozapps/extensions/test/browser/addons/browser_inlinesettings1_info/bootstrap.js => toolkit/mozapps/extensions/test/browser/addons/browser_inlinesettings1/bootstrap.js
rename : toolkit/mozapps/extensions/test/browser/addons/browser_inlinesettings1_info/install.rdf => toolkit/mozapps/extensions/test/browser/addons/browser_inlinesettings1/install.rdf
rename : toolkit/mozapps/extensions/test/browser/addons/browser_inlinesettings1_info/options.xul => toolkit/mozapps/extensions/test/browser/addons/browser_inlinesettings1/options.xul
rename : toolkit/mozapps/extensions/test/browser/browser_inlinesettings_info.js => toolkit/mozapps/extensions/test/browser/browser_inlinesettings.js
extra : rebase_source : ab5a368d3fff88bad5144188689d1c969562e4d4

js/src/tests/Makefile.in

@@ -36,6 +36,7 @@ TEST_FILES = \
   js1_8/ \
   js1_8_1/ \
   js1_8_5/ \
+  test402/ \
   $(NULL)
 
 PKG_STAGE = $(DIST)/test-package-stage

js/src/tests/README.txt

@@ -5,7 +5,13 @@ The JS test suite is a fairly extensive collection of correctness and regression
 tests for the Spidermonkey engine. Two harnesses run these tests: the shell test
 harness in this directory and the "reftest" harness built into the browser, used
 by Tinderbox. The browser reftests require additional manifest files; these are
-generated automatically by the build phase 'package-tests'.
+generated automatically by the build phase 'package-tests' using the
+'--make-manifests' option to jstests.py.
+
+Creating a test
+---------------
+For general information, see
+https://developer.mozilla.org/en-US/docs/SpiderMonkey/Creating_JavaScript_tests
 
 Adding a test
 -------------
@@ -18,9 +24,9 @@ Adding a test
 Adjusting when and how a test runs
 ----------------------------------
     Put a comment at the top of the header matching the format:
-        // |reftest| <options> -- <comment>
+        // |reftest| <failure-type> -- <comment>
 
-    Where <options> is a standard reftest options string, as documented by:
+    Where <failure-type> is a standard reftest <failure-type> string, as documented by:
         http://mxr.mozilla.org/mozilla-central/source/layout/tools/reftest/README.txt
 
     Example:
@@ -29,3 +35,19 @@ Adjusting when and how a test runs
         <fineprint> Either // or /* */ style comments may be used. The entire
         comment must appear in the first 512 bytes of the file. The control
         string must be in its own comment block. </fineprint>
+
+    When adding such comments to individual files is not feasible (e.g., for
+    imported tests), reftest manifest entries can be added to jstests.list
+    instead. Combining in-file comments with entries in this manifest file for
+    the same files is not supported (the one from the manifest file will be
+    used). Only the following two forms are supported:
+        <failure-type> include <relative_path>
+        <failure-type> script <relative_path>
+    The <type> "include" indicates that <failure-type> should apply to all test
+    cases within a directory. A statement for a nested directory or script
+    overrides one for an enclosing directory.
+
+Running tests
+-------------
+See
+https://developer.mozilla.org/en-US/docs/SpiderMonkey/Running_Automated_JavaScript_Tests

js/src/tests/browser.js

@@ -2,9 +2,42 @@
 /* This Source Code Form is subject to the terms of the Mozilla Public
  * License, v. 2.0. If a copy of the MPL was not distributed with this
  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
 var gPageCompleted;
 var GLOBAL = this + '';
 
+// Variables local to jstests harness.
+var jstestsTestPassesUnlessItThrows = false;
+var jstestsRestoreFunction;
+var jstestsOptions;
+
+/*
+ * Signals to this script that the current test case should be considered to
+ * have passed if it doesn't throw an exception.
+ *
+ * Overrides the same-named function in shell.js.
+ */
+function testPassesUnlessItThrows() {
+  jstestsTestPassesUnlessItThrows = true;
+}
+
+/*
+ * Requests to load the given JavaScript file before the file containing the
+ * test case.
+ */
+function include(file) {
+  outputscripttag(file, {language: "type", mimetype: "text/javascript"});
+}
+
+/*
+ * Sets a restore function which restores the standard built-in ECMAScript
+ * properties after a destructive test case, and which will be called after
+ * the test case terminates.
+ */
+function setRestoreFunction(restore) {
+  jstestsRestoreFunction = restore;
+}
+
 function htmlesc(str) {
   if (str == '<')
     return '&lt;';
@@ -83,6 +116,16 @@ function writeFormattedResult( expect, actual, string, passed ) {
 
 window.onerror = function (msg, page, line)
 {
+  jstestsTestPassesUnlessItThrows = false;
+
+  // Restore options in case a test case used this common variable name.
+  options = jstestsOptions;
+
+  // Restore the ECMAScript environment after potentially destructive tests.
+  if (typeof jstestsRestoreFunction === "function") {
+    jstestsRestoreFunction();
+  }
+
   optionsPush();
 
   if (typeof DESCRIPTION == 'undefined')
@@ -178,11 +221,16 @@ function options(aOptionName)
 //    SpecialPowers.wrap(Components).utils[aOptionName] = options.currvalues.hasOwnProperty(aOptionName);
     Components.utils[aOptionName] =
       options.currvalues.hasOwnProperty(aOptionName);
-  }  
+  }
 
   return value;
 }
 
+// Keep a reference to options around so that we can restore it after running
+// a test case, which may have used this common name for one of its own
+// variables.
+jstestsOptions = options;
+
 function optionsInit() {
 
   // hash containing the set options.
@@ -325,8 +373,8 @@ function jsTestDriverBrowserInit()
    * since the default setting of jit changed from false to true
    * in http://hg.mozilla.org/tracemonkey/rev/685e00e68be9
    * bisections which depend upon jit settings can be thrown off.
-   * default jit(false) when not running jsreftests to make bisections 
-   * depending upon jit settings consistent over time. This is not needed 
+   * default jit(false) when not running jsreftests to make bisections
+   * depending upon jit settings consistent over time. This is not needed
    * in shell tests as the default jit setting has not changed there.
    */
 
@@ -350,6 +398,16 @@ function jsTestDriverBrowserInit()
   // is file:. insert an empty script tag, to work around it.
   document.write('<script></script>');
 
+  // Enable a test suite that has more than two levels of directories to
+  // provide browser.js and shell.js in its base directory.
+  // This assumes that suitepath is a relative path, as is the case in the
+  // try server environment. Absolute paths are not allowed.
+  if (suitepath.indexOf('/') !== -1) {
+    var base = suitepath.slice(0, suitepath.indexOf('/'));
+    outputscripttag(base + '/shell.js', properties);
+    outputscripttag(base + '/browser.js', properties);
+  }
+
   outputscripttag(suitepath + '/shell.js', properties);
   outputscripttag(suitepath + '/browser.js', properties);
   outputscripttag(suitepath + '/' + subsuite + '/shell.js', properties);
@@ -406,6 +464,20 @@ function jsTestDriverEnd()
 
   window.onerror = null;
 
+  // Restore options in case a test case used this common variable name.
+  options = jstestsOptions;
+
+  // Restore the ECMAScript environment after potentially destructive tests.
+  if (typeof jstestsRestoreFunction === "function") {
+    jstestsRestoreFunction();
+  }
+
+  if (jstestsTestPassesUnlessItThrows) {
+    var testcase = new TestCase("unknown-test-name", "", true, true);
+    print(PASSED);
+    jstestsTestPassesUnlessItThrows = false;
+  }
+
   try
   {
     optionsReset();
@@ -493,7 +565,7 @@ function closeDialog()
 
   while ( (subject = gDialogCloserSubjects.pop()) != null)
   {
-    if (subject.document instanceof XULDocument && 
+    if (subject.document instanceof XULDocument &&
         subject.document.documentURI == 'chrome://global/content/commonDialog.xul')
     {
       subject.close();

js/src/tests/jstests.list

@@ -0,0 +1,13 @@
+# Manifest entries for imported test suites whose individual test cases
+# we don't want to change.
+
+skip-if(!this.hasOwnProperty("Intl")) include test402/jstests.list # Intl is not enabled in all builds
+skip include test402/lib/jstests.list # include files
+
+skip script test402/ch09/9.2/9.2.8_4.js # bug 853702
+skip script test402/ch10/10.1/10.1.1_a.js # bug 854320
+skip script test402/ch10/10.1/10.1.1_13.js # bug 853704
+skip script test402/ch10/10.1/10.1.1_19_c.js # bug 853704
+skip script test402/ch11/11.1/11.1.1_a.js # bug 854320
+skip script test402/ch11/11.3/11.3.2_TRP.js # bug 853706
+skip script test402/ch12/12.1/12.1.1_a.js # bug 854320

js/src/tests/lib/manifest.py

@@ -276,18 +276,79 @@ def _parse_test_header(fullpath, testcase, xul_tester):
 
     _parse_one(testcase, xul_tester)
 
+def _parse_external_manifest(filename, relpath):
+    """
+    Reads an external manifest file for test suites whose individual test cases
+    can't be decorated with reftest comments.
+    filename - str: name of the manifest file
+    relpath - str: relative path of the directory containing the manifest
+                   within the test suite
+    """
+    entries = []
+
+    with open(filename, 'r') as fp:
+        manifest_re = re.compile(r'^\s*(.*)\s+(include|script)\s+(\S+)$')
+        for line in fp:
+            line, _, comment = line.partition('#')
+            line = line.strip()
+            if not line:
+                continue
+            matches = manifest_re.match(line)
+            if not matches:
+                print('warning: unrecognized line in jstests.list: {0}'.format(line))
+                continue
+
+            path = os.path.normpath(os.path.join(relpath, matches.group(3)))
+            if matches.group(2) == 'include':
+                # The manifest spec wants a reference to another manifest here,
+                # but we need just the directory. We do need the trailing
+                # separator so we don't accidentally match other paths of which
+                # this one is a prefix.
+                assert(path.endswith('jstests.list'))
+                path = path[:-len('jstests.list')]
+
+            entries.append({'path': path, 'terms': matches.group(1), 'comment': comment.strip()})
+
+    # if one directory name is a prefix of another, we want the shorter one first
+    entries.sort(key=lambda x: x["path"])
+    return entries
+
+def _apply_external_manifests(filename, testcase, entries, xul_tester):
+    for entry in entries:
+        if filename.startswith(entry["path"]):
+            # The reftest spec would require combining the terms (failure types)
+            # that may already be defined in the test case with the terms
+            # specified in entry; for example, a skip overrides a random, which
+            # overrides a fails. Since we don't necessarily know yet in which
+            # environment the test cases will be run, we'd also have to
+            # consider skip-if, random-if, and fails-if with as-yet unresolved
+            # conditions.
+            # At this point, we use external manifests only for test cases
+            # that can't have their own failure type comments, so we simply
+            # use the terms for the most specific path.
+            testcase.terms = entry["terms"]
+            testcase.comment = entry["comment"]
+            _parse_one(testcase, xul_tester)
+
 def load(location, xul_tester, reldir = ''):
     """
     Locates all tests by walking the filesystem starting at |location|.
     Uses xul_tester to evaluate any test conditions in the test header.
+    Failure type and comment for a test case can come from
+    - an external manifest entry for the test case,
+    - an external manifest entry for a containing directory,
+    - most commonly: the header of the test case itself.
     """
     # The list of tests that we are collecting.
     tests = []
 
-    # Any file who's basename matches something in this set is ignored.
+    # Any file whose basename matches something in this set is ignored.
     EXCLUDED = set(('browser.js', 'shell.js', 'jsref.js', 'template.js',
                     'user.js', 'js-test-driver-begin.js', 'js-test-driver-end.js'))
 
+    manifestFile = os.path.join(location, 'jstests.list')
+    externalManifestEntries = _parse_external_manifest(manifestFile, '')
+
     for root, basename in _find_all_js_files(location, location):
         # Skip js files in the root test directory.
         if not root:
@@ -306,9 +367,8 @@ def load(location, xul_tester, reldir = ''):
         if statbuf.st_size == 0:
             continue
 
-        # Parse the test header and load the test.
         testcase = TestCase(os.path.join(reldir, filename))
+        _apply_external_manifests(filename, testcase, externalManifestEntries, xul_tester)
         _parse_test_header(fullpath, testcase, xul_tester)
         tests.append(testcase)
     return tests
-

js/src/tests/shell.js

@@ -44,6 +44,17 @@ var DEBUG = false;
 var DESCRIPTION;
 var EXPECTED;
 
+/*
+ * Signals to results.py that the current test case should be considered to
+ * have passed if it doesn't throw an exception.
+ *
+ * When the test suite is run in the browser, this function gets overridden by
+ * the same-named function in browser.js.
+ */
+function testPassesUnlessItThrows() {
+  print(PASSED);
+}
+
 /*
  * wrapper for test case constructor that doesn't require the SECTION
  * argument.
@@ -132,7 +143,7 @@ function reportFailure (msg)
   var l;
   var funcName = currentFunc();
   var prefix = (funcName) ? "[reported from " + funcName + "] ": "";
-   
+
   for (var i=0; i<lines.length; i++)
     print (FAILED + prefix + lines[i]);
 }
@@ -376,7 +387,7 @@ function enterFunc (funcName)
 function exitFunc (funcName)
 {
   var lastFunc = callStack.pop();
-   
+
   if (funcName)
   {
     if (!funcName.match(/\(\)$/))
@@ -486,7 +497,7 @@ function BigO(data)
     {
       var Ydiff = Y[i] - this.Yavg;
       var Xdiff = X[i] - this.Xavg;
-       
+
       SUM_Ydiff2 += Ydiff * Ydiff;
       SUM_Xdiff2 += Xdiff * Xdiff;
       SUM_XdiffYdiff += Xdiff * Ydiff;
@@ -522,7 +533,7 @@ function BigO(data)
     {
       deriv.X[i] = (X[i] + X[i+1])/2;
       deriv.Y[i] = (Y[i+1] - Y[i])/(X[i+1] - X[i]);
-    } 
+    }
     return deriv;
   }
 
@@ -598,7 +609,7 @@ function optionsInit() {
 }
 
 function optionsClear() {
-       
+
   // turn off current settings
   // except jit.
   var optionNames = options().split(',');
@@ -844,5 +855,3 @@ function OptLevel( i ) {
   cx.setOptimizationLevel(i);
 }
 /* end of Rhino functions */
-
-

js/src/tests/test402/browser.js

@@ -0,0 +1,40 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+/*
+ * Undo at least the damage done by taintArray so that the jstests
+ * harness won't die. The properties added to Object.prototype by the various
+ * tests have names that are less likely to cause trouble.
+ */
+setRestoreFunction((function () {
+    var Array_indexOf = Array.prototype.indexOf;
+    var Array_join = Array.prototype.join;
+    var Array_push = Array.prototype.push;
+    var Array_slice = Array.prototype.slice;
+    var Array_sort = Array.prototype.sort;
+    return function () {
+        delete Array.prototype["0"];
+        Array.prototype.indexOf = Array_indexOf;
+        Array.prototype.join = Array_join;
+        Array.prototype.push = Array_push;
+        Array.prototype.slice = Array_slice;
+        Array.prototype.sort = Array_sort;
+    };
+}()));
+
+/*
+ * Loading include files into the browser from a script so that they become
+ * synchronously available to that same script is difficult. Instead, request
+ * all of them to be loaded before we start.
+ */
+include("test402/lib/testBuiltInObject.js");
+include("test402/lib/testIntl.js");
+
+/*
+ * Test262 function $INCLUDE loads a file with support functions for the tests.
+ * Since we've already loaded all of these files, just ignore the call.
+ * This function replaces one in shell.js.
+ */
+function $INCLUDE(file) {
+}

js/src/tests/test402/shell.js

@@ -0,0 +1,33 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+/*
+ * Test402 tests pass unless they throw. Note that this isn't true for all
+ * Test262 test cases - for the ones marked @negative the logic is inverted.
+ */
+testPassesUnlessItThrows();
+
+/*
+ * Test262 function $ERROR throws an error with the message provided. Test262
+ * test cases call it to indicate failure.
+ */
+function $ERROR(msg) {
+    throw new Error("Test402 error: " + msg);
+}
+
+/*
+ * Test262 function $INCLUDE loads a file with support functions for the tests.
+ * This function is replaced in browser.js.
+ */
+function $INCLUDE(file) {
+    loadRelativeToScript("lib/" + file);
+}
+
+/*
+ * Test262 function fnGlobalObject returns the global object.
+ */
+var __globalObject = Function("return this;")();
+function fnGlobalObject() {
+     return __globalObject;
+}