diff --git a/js/src/tests/Makefile.in b/js/src/tests/Makefile.in index b450ecfe743..318b7d78e07 100644 --- a/js/src/tests/Makefile.in +++ b/js/src/tests/Makefile.in @@ -36,6 +36,7 @@ TEST_FILES = \ js1_8/ \ js1_8_1/ \ js1_8_5/ \ + test402/ \ $(NULL) PKG_STAGE = $(DIST)/test-package-stage diff --git a/js/src/tests/README.txt b/js/src/tests/README.txt index 82e7eeb63c0..ed5e2937b85 100644 --- a/js/src/tests/README.txt +++ b/js/src/tests/README.txt @@ -5,7 +5,13 @@ The JS test suite is a fairly extensive collection of correctness and regression tests for the Spidermonkey engine. Two harnesses run these tests: the shell test harness in this directory and the "reftest" harness built into the browser, used by Tinderbox. The browser reftests require additional manifest files; these are -generated automatically by the build phase 'package-tests'. +generated automatically by the build phase 'package-tests' using the +'--make-manifests' option to jstests.py. + +Creating a test +--------------- +For general information, see +https://developer.mozilla.org/en-US/docs/SpiderMonkey/Creating_JavaScript_tests Adding a test ------------- @@ -18,9 +24,9 @@ Adding a test Adjusting when and how a test runs ---------------------------------- Put a comment at the top of the header matching the format: - // |reftest| -- + // |reftest| -- - Where is a standard reftest options string, as documented by: + Where is a standard reftest string, as documented by: http://mxr.mozilla.org/mozilla-central/source/layout/tools/reftest/README.txt Example: @@ -29,3 +35,19 @@ Adjusting when and how a test runs Either // or /* */ style comments may be used. The entire comment must appear in the first 512 bytes of the file. The control string must be in its own comment block. + + When adding such comments to individual files is not feasible (e.g., for + imported tests), reftest manifest entries can be added to jstests.list + instead. Combining in-file comments with entries in this manifest file for + the same files is not supported (the one from the manifest file will be + used). Only the following two forms are supported: + include + script + The "include" indicates that should apply to all test + cases within a directory. A statement for a nested directory or script + overrides one for an enclosing directory. + +Running tests +------------- +See +https://developer.mozilla.org/en-US/docs/SpiderMonkey/Running_Automated_JavaScript_Tests diff --git a/js/src/tests/browser.js b/js/src/tests/browser.js index d35a0b74bf6..a4d21eac3f5 100644 --- a/js/src/tests/browser.js +++ b/js/src/tests/browser.js @@ -2,9 +2,42 @@ /* This Source Code Form is subject to the terms of the Mozilla Public * License, v. 2.0. If a copy of the MPL was not distributed with this * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + var gPageCompleted; var GLOBAL = this + ''; +// Variables local to jstests harness. +var jstestsTestPassesUnlessItThrows = false; +var jstestsRestoreFunction; +var jstestsOptions; + +/* + * Signals to this script that the current test case should be considered to + * have passed if it doesn't throw an exception. + * + * Overrides the same-named function in shell.js. + */ +function testPassesUnlessItThrows() { + jstestsTestPassesUnlessItThrows = true; +} + +/* + * Requests to load the given JavaScript file before the file containing the + * test case. + */ +function include(file) { + outputscripttag(file, {language: "type", mimetype: "text/javascript"}); +} + +/* + * Sets a restore function which restores the standard built-in ECMAScript + * properties after a destructive test case, and which will be called after + * the test case terminates. + */ +function setRestoreFunction(restore) { + jstestsRestoreFunction = restore; +} + function htmlesc(str) { if (str == '<') return '<'; @@ -83,6 +116,16 @@ function writeFormattedResult( expect, actual, string, passed ) { window.onerror = function (msg, page, line) { + jstestsTestPassesUnlessItThrows = false; + + // Restore options in case a test case used this common variable name. + options = jstestsOptions; + + // Restore the ECMAScript environment after potentially destructive tests. + if (typeof jstestsRestoreFunction === "function") { + jstestsRestoreFunction(); + } + optionsPush(); if (typeof DESCRIPTION == 'undefined') @@ -178,11 +221,16 @@ function options(aOptionName) // SpecialPowers.wrap(Components).utils[aOptionName] = options.currvalues.hasOwnProperty(aOptionName); Components.utils[aOptionName] = options.currvalues.hasOwnProperty(aOptionName); - } + } return value; } +// Keep a reference to options around so that we can restore it after running +// a test case, which may have used this common name for one of its own +// variables. +jstestsOptions = options; + function optionsInit() { // hash containing the set options. @@ -325,8 +373,8 @@ function jsTestDriverBrowserInit() * since the default setting of jit changed from false to true * in http://hg.mozilla.org/tracemonkey/rev/685e00e68be9 * bisections which depend upon jit settings can be thrown off. - * default jit(false) when not running jsreftests to make bisections - * depending upon jit settings consistent over time. This is not needed + * default jit(false) when not running jsreftests to make bisections + * depending upon jit settings consistent over time. This is not needed * in shell tests as the default jit setting has not changed there. */ @@ -350,6 +398,16 @@ function jsTestDriverBrowserInit() // is file:. insert an empty script tag, to work around it. document.write(''); + // Enable a test suite that has more than two levels of directories to + // provide browser.js and shell.js in its base directory. + // This assumes that suitepath is a relative path, as is the case in the + // try server environment. Absolute paths are not allowed. + if (suitepath.indexOf('/') !== -1) { + var base = suitepath.slice(0, suitepath.indexOf('/')); + outputscripttag(base + '/shell.js', properties); + outputscripttag(base + '/browser.js', properties); + } + outputscripttag(suitepath + '/shell.js', properties); outputscripttag(suitepath + '/browser.js', properties); outputscripttag(suitepath + '/' + subsuite + '/shell.js', properties); @@ -406,6 +464,20 @@ function jsTestDriverEnd() window.onerror = null; + // Restore options in case a test case used this common variable name. + options = jstestsOptions; + + // Restore the ECMAScript environment after potentially destructive tests. + if (typeof jstestsRestoreFunction === "function") { + jstestsRestoreFunction(); + } + + if (jstestsTestPassesUnlessItThrows) { + var testcase = new TestCase("unknown-test-name", "", true, true); + print(PASSED); + jstestsTestPassesUnlessItThrows = false; + } + try { optionsReset(); @@ -493,7 +565,7 @@ function closeDialog() while ( (subject = gDialogCloserSubjects.pop()) != null) { - if (subject.document instanceof XULDocument && + if (subject.document instanceof XULDocument && subject.document.documentURI == 'chrome://global/content/commonDialog.xul') { subject.close(); diff --git a/js/src/tests/jstests.list b/js/src/tests/jstests.list new file mode 100644 index 00000000000..da507d001c8 --- /dev/null +++ b/js/src/tests/jstests.list @@ -0,0 +1,13 @@ +# Manifest entries for imported test suites whose individual test cases +# we don't want to change. + +skip-if(!this.hasOwnProperty("Intl")) include test402/jstests.list # Intl is not enabled in all builds +skip include test402/lib/jstests.list # include files + +skip script test402/ch09/9.2/9.2.8_4.js # bug 853702 +skip script test402/ch10/10.1/10.1.1_a.js # bug 854320 +skip script test402/ch10/10.1/10.1.1_13.js # bug 853704 +skip script test402/ch10/10.1/10.1.1_19_c.js # bug 853704 +skip script test402/ch11/11.1/11.1.1_a.js # bug 854320 +skip script test402/ch11/11.3/11.3.2_TRP.js # bug 853706 +skip script test402/ch12/12.1/12.1.1_a.js # bug 854320 diff --git a/js/src/tests/lib/manifest.py b/js/src/tests/lib/manifest.py index a523d20dbfd..ab849fff32c 100644 --- a/js/src/tests/lib/manifest.py +++ b/js/src/tests/lib/manifest.py @@ -276,18 +276,79 @@ def _parse_test_header(fullpath, testcase, xul_tester): _parse_one(testcase, xul_tester) +def _parse_external_manifest(filename, relpath): + """ + Reads an external manifest file for test suites whose individual test cases + can't be decorated with reftest comments. + filename - str: name of the manifest file + relpath - str: relative path of the directory containing the manifest + within the test suite + """ + entries = [] + + with open(filename, 'r') as fp: + manifest_re = re.compile(r'^\s*(.*)\s+(include|script)\s+(\S+)$') + for line in fp: + line, _, comment = line.partition('#') + line = line.strip() + if not line: + continue + matches = manifest_re.match(line) + if not matches: + print('warning: unrecognized line in jstests.list: {0}'.format(line)) + continue + + path = os.path.normpath(os.path.join(relpath, matches.group(3))) + if matches.group(2) == 'include': + # The manifest spec wants a reference to another manifest here, + # but we need just the directory. We do need the trailing + # separator so we don't accidentally match other paths of which + # this one is a prefix. + assert(path.endswith('jstests.list')) + path = path[:-len('jstests.list')] + + entries.append({'path': path, 'terms': matches.group(1), 'comment': comment.strip()}) + + # if one directory name is a prefix of another, we want the shorter one first + entries.sort(key=lambda x: x["path"]) + return entries + +def _apply_external_manifests(filename, testcase, entries, xul_tester): + for entry in entries: + if filename.startswith(entry["path"]): + # The reftest spec would require combining the terms (failure types) + # that may already be defined in the test case with the terms + # specified in entry; for example, a skip overrides a random, which + # overrides a fails. Since we don't necessarily know yet in which + # environment the test cases will be run, we'd also have to + # consider skip-if, random-if, and fails-if with as-yet unresolved + # conditions. + # At this point, we use external manifests only for test cases + # that can't have their own failure type comments, so we simply + # use the terms for the most specific path. + testcase.terms = entry["terms"] + testcase.comment = entry["comment"] + _parse_one(testcase, xul_tester) + def load(location, xul_tester, reldir = ''): """ Locates all tests by walking the filesystem starting at |location|. Uses xul_tester to evaluate any test conditions in the test header. + Failure type and comment for a test case can come from + - an external manifest entry for the test case, + - an external manifest entry for a containing directory, + - most commonly: the header of the test case itself. """ # The list of tests that we are collecting. tests = [] - # Any file who's basename matches something in this set is ignored. + # Any file whose basename matches something in this set is ignored. EXCLUDED = set(('browser.js', 'shell.js', 'jsref.js', 'template.js', 'user.js', 'js-test-driver-begin.js', 'js-test-driver-end.js')) + manifestFile = os.path.join(location, 'jstests.list') + externalManifestEntries = _parse_external_manifest(manifestFile, '') + for root, basename in _find_all_js_files(location, location): # Skip js files in the root test directory. if not root: @@ -306,9 +367,8 @@ def load(location, xul_tester, reldir = ''): if statbuf.st_size == 0: continue - # Parse the test header and load the test. testcase = TestCase(os.path.join(reldir, filename)) + _apply_external_manifests(filename, testcase, externalManifestEntries, xul_tester) _parse_test_header(fullpath, testcase, xul_tester) tests.append(testcase) return tests - diff --git a/js/src/tests/shell.js b/js/src/tests/shell.js index d0f38735083..08abb3bb518 100644 --- a/js/src/tests/shell.js +++ b/js/src/tests/shell.js @@ -44,6 +44,17 @@ var DEBUG = false; var DESCRIPTION; var EXPECTED; +/* + * Signals to results.py that the current test case should be considered to + * have passed if it doesn't throw an exception. + * + * When the test suite is run in the browser, this function gets overridden by + * the same-named function in browser.js. + */ +function testPassesUnlessItThrows() { + print(PASSED); +} + /* * wrapper for test case constructor that doesn't require the SECTION * argument. @@ -132,7 +143,7 @@ function reportFailure (msg) var l; var funcName = currentFunc(); var prefix = (funcName) ? "[reported from " + funcName + "] ": ""; - + for (var i=0; i