diff --git a/testing/web-platform/harness/README.rst b/testing/web-platform/harness/README.rst index 6e3b5fa2546..bf9b12253ba 100644 --- a/testing/web-platform/harness/README.rst +++ b/testing/web-platform/harness/README.rst @@ -25,15 +25,9 @@ following are most significant: ``--product`` (defaults to `firefox`) The product to test against: `b2g`, `chrome`, `firefox`, or `servo`. -``--binary`` (required if product is `firefox` or `servo`) +``--binary`` (required) The path to a binary file for the product (browser) to test against. -``--webdriver-binary`` (required if product is `chrome`) - The path to a `*driver` binary; e.g., a `chromedriver` binary. - -``--certutil-binary`` (required if product is `firefox` [#]_) - The path to a `certutil` binary (for tests that must be run over https). - ``--metadata`` (required) The path to a directory containing test metadata. [#]_ @@ -43,9 +37,6 @@ following are most significant: ``--prefs-root`` (required only when testing a Firefox binary) The path to a directory containing Firefox test-harness preferences. [#]_ -.. [#] The ``--certutil-binary`` option is required when the product is - ``firefox`` unless ``--ssl-type=none`` is specified. - .. [#] The ``--metadata`` path is to a directory that contains: * a ``MANIFEST.json`` file (the web-platform-tests documentation has @@ -65,29 +56,26 @@ To test a Firefox Nightly build in an OS X environment, you might start wptrunner using something similar to the following example:: wptrunner --metadata=~/web-platform-tests/ --tests=~/web-platform-tests/ \ - --binary=~/mozilla-central/obj-x86_64-apple-darwin14.3.0/dist/Nightly.app/Contents/MacOS/firefox \ - --certutil-binary=~/mozilla-central/obj-x86_64-apple-darwin14.3.0/security/nss/cmd/certutil/certutil \ - --prefs-root=~/mozilla-central/testing/profiles + --binary=~/mozilla-central/obj-x86_64-apple-darwin14.0.0/dist/Nightly.app/Contents/MacOS/firefox \ + --prefs-root=~/mozilla-central/testing/profiles And to test a Chromium build in an OS X environment, you might start wptrunner using something similar to the following example:: wptrunner --metadata=~/web-platform-tests/ --tests=~/web-platform-tests/ \ - --binary=~/chromium/src/out/Release/Chromium.app/Contents/MacOS/Chromium \ - --webdriver-binary=/usr/local/bin/chromedriver --product=chrome + --binary=~/chromium/src/out/Release/Chromium.app/Contents/MacOS/Chromium \ + --product=chrome ------------------------------------- Example: How to run a subset of tests ------------------------------------- To restrict a test run just to tests in a particular web-platform-tests -subdirectory, specify the directory name in the positional arguments after -the options; for example, run just the tests in the `dom` subdirectory:: +subdirectory, use ``--include`` with the directory name; for example:: wptrunner --metadata=~/web-platform-tests/ --tests=~/web-platform-tests/ \ - --binary=/path/to/firefox --certutil-binary=/path/to/certutil \ - --prefs-root=/path/to/testing/profiles \ - dom + --binary=/path/to/firefox --prefs-root=/path/to/testing/profiles \ + --include=dom Output ~~~~~~ @@ -107,9 +95,8 @@ log to a file and a human-readable summary to stdout, you might start wptrunner using something similar to the following example:: wptrunner --metadata=~/web-platform-tests/ --tests=~/web-platform-tests/ \ - --binary=/path/to/firefox --certutil-binary=/path/to/certutil \ - --prefs-root=/path/to/testing/profiles \ - --log-raw=output.log --log-mach=- + --binary=/path/to/firefox --prefs-root=/path/to/testing/profiles + --log-raw=output.log --log-mach=- Expectation Data ~~~~~~~~~~~~~~~~ diff --git a/testing/web-platform/harness/docs/usage.rst b/testing/web-platform/harness/docs/usage.rst index ba1375dbcc8..33471dfffba 100644 --- a/testing/web-platform/harness/docs/usage.rst +++ b/testing/web-platform/harness/docs/usage.rst @@ -56,15 +56,9 @@ takes multiple options, of which the following are most significant: ``--product`` (defaults to `firefox`) The product to test against: `b2g`, `chrome`, `firefox`, or `servo`. -``--binary`` (required if product is `firefox` or `servo`) +``--binary`` (required) The path to a binary file for the product (browser) to test against. -``--webdriver-binary`` (required if product is `chrome`) - The path to a `*driver` binary; e.g., a `chromedriver` binary. - -``--certutil-binary`` (required if product is `firefox` [#]_) - The path to a `certutil` binary (for tests that must be run over https). - ``--metadata`` (required only when not `using default paths`_) The path to a directory containing test metadata. [#]_ @@ -74,9 +68,6 @@ takes multiple options, of which the following are most significant: ``--prefs-root`` (required only when testing a Firefox binary) The path to a directory containing Firefox test-harness preferences. [#]_ -.. [#] The ``--certutil-binary`` option is required when the product is - ``firefox`` unless ``--ssl-type=none`` is specified. - .. [#] The ``--metadata`` path is to a directory that contains: * a ``MANIFEST.json`` file (the web-platform-tests documentation has @@ -98,30 +89,26 @@ To test a Firefox Nightly build in an OS X environment, you might start wptrunner using something similar to the following example:: wptrunner --metadata=~/web-platform-tests/ --tests=~/web-platform-tests/ \ - --binary=~/mozilla-central/obj-x86_64-apple-darwin14.3.0/dist/Nightly.app/Contents/MacOS/firefox \ - --certutil-binary=~/mozilla-central/obj-x86_64-apple-darwin14.3.0/security/nss/cmd/certutil/certutil \ + --binary=~/mozilla-central/obj-x86_64-apple-darwin14.0.0/dist/Nightly.app/Contents/MacOS/firefox \ --prefs-root=~/mozilla-central/testing/profiles - And to test a Chromium build in an OS X environment, you might start wptrunner using something similar to the following example:: wptrunner --metadata=~/web-platform-tests/ --tests=~/web-platform-tests/ \ --binary=~/chromium/src/out/Release/Chromium.app/Contents/MacOS/Chromium \ - --webdriver-binary=/usr/local/bin/chromedriver --product=chrome + --product=chrome -------------------- Running test subsets -------------------- To restrict a test run just to tests in a particular web-platform-tests -subdirectory, specify the directory name in the positional arguments after -the options; for example, run just the tests in the `dom` subdirectory:: +subdirectory, use ``--include`` with the directory name; for example:: wptrunner --metadata=~/web-platform-tests/ --tests=~/web-platform-tests/ \ - --binary=/path/to/firefox --certutil-binary=/path/to/certutil \ - --prefs-root=/path/to/testing/profiles \ - dom + --binary=/path/to/firefox --prefs-root=/path/to/testing/profiles \ + --include=dom ------------------- Running in parallel diff --git a/testing/web-platform/harness/test/test.cfg.example b/testing/web-platform/harness/test/test.cfg.example index db48226216c..6a4057e2200 100644 --- a/testing/web-platform/harness/test/test.cfg.example +++ b/testing/web-platform/harness/test/test.cfg.example @@ -8,13 +8,9 @@ ssl-type=none # prefs-root=/path/to/gecko-src/testing/profiles/ # [servo] -# binary=/path/to/servo-src/target/release/servo -# exclude=testharness # Because it needs a special testharness.js - -# [servodriver] -# binary=/path/to/servo-src/target/release/servo +# binary=/path/to/servo-src/components/servo/target/servo # exclude=testharness # Because it needs a special testharness.js # [chrome] # binary=/path/to/chrome -# webdriver-binary=/path/to/chromedriver +# webdriver-binary=/path/to/chromedriver \ No newline at end of file diff --git a/testing/web-platform/harness/wptrunner/browsers/servodriver.py b/testing/web-platform/harness/wptrunner/browsers/servodriver.py index 656a8b51f40..ceb1625237a 100644 --- a/testing/web-platform/harness/wptrunner/browsers/servodriver.py +++ b/testing/web-platform/harness/wptrunner/browsers/servodriver.py @@ -42,7 +42,7 @@ def browser_kwargs(**kwargs): "debug_info": kwargs["debug_info"]} -def executor_kwargs(test_type, server_config, cache_manager, run_info_data, **kwargs): +def executor_kwargs(test_type, server_config, cache_manager, **kwargs): rv = base_executor_kwargs(test_type, server_config, cache_manager, **kwargs) return rv diff --git a/testing/web-platform/harness/wptrunner/executors/executorservo.py b/testing/web-platform/harness/wptrunner/executors/executorservo.py index a94fcd26a41..987a49e26f9 100644 --- a/testing/web-platform/harness/wptrunner/executors/executorservo.py +++ b/testing/web-platform/harness/wptrunner/executors/executorservo.py @@ -62,9 +62,8 @@ class ServoTestharnessExecutor(ProcessTestExecutor): self.result_data = None self.result_flag = threading.Event() - debug_args, command = browser_command(self.binary, - ["--cpu", "--hard-fail", "-u", "Servo/wptrunner", "-z", self.test_url(test)], - self.debug_info) + debug_args, command = browser_command(self.binary, ["--cpu", "--hard-fail", "-z", self.test_url(test)], + self.debug_info) self.command = command @@ -100,18 +99,15 @@ class ServoTestharnessExecutor(ProcessTestExecutor): self.proc.wait() proc_is_running = True - - if self.result_flag.is_set(): - if self.result_data is not None: - self.result_data["test"] = test.url - result = self.convert_result(test, self.result_data) - else: - self.proc.wait() + if self.result_flag.is_set() and self.result_data is not None: + self.result_data["test"] = test.url + result = self.convert_result(test, self.result_data) + else: + if self.proc.poll() is not None: result = (test.result_cls("CRASH", None), []) proc_is_running = False - else: - result = (test.result_cls("TIMEOUT", None), []) - + else: + result = (test.result_cls("TIMEOUT", None), []) if proc_is_running: if self.pause_after_test: @@ -190,8 +186,8 @@ class ServoRefTestExecutor(ProcessTestExecutor): with TempFilename(self.tempdir) as output_path: self.command = [self.binary, "--cpu", "--hard-fail", "--exit", - "-u", "Servo/wptrunner", "-Z", "disable-text-aa", - "--output=%s" % output_path, full_url] + "-Z", "disable-text-aa", "--output=%s" % output_path, + full_url] env = os.environ.copy() env["HOST_FILE"] = self.hosts_path diff --git a/testing/web-platform/harness/wptrunner/metadata.py b/testing/web-platform/harness/wptrunner/metadata.py index 948baaa96e7..3a35619c5e7 100644 --- a/testing/web-platform/harness/wptrunner/metadata.py +++ b/testing/web-platform/harness/wptrunner/metadata.py @@ -153,32 +153,17 @@ def update_from_logs(manifests, *log_filenames, **kwargs): return expected_map -def directory_manifests(metadata_path): - rv = [] - for dirpath, dirname, filenames in os.walk(metadata_path): - if "__dir__.ini" in filenames: - rel_path = os.path.relpath(dirpath, metadata_path) - rv.append(os.path.join(rel_path, "__dir__.ini")) - return rv def write_changes(metadata_path, expected_map): # First write the new manifest files to a temporary directory temp_path = tempfile.mkdtemp(dir=os.path.split(metadata_path)[0]) write_new_expected(temp_path, expected_map) - # Keep all __dir__.ini files (these are not in expected_map because they - # aren't associated with a specific test) - keep_files = directory_manifests(metadata_path) - # Copy all files in the root to the temporary location since # these cannot be ini files - keep_files.extend(item for item in os.listdir(metadata_path) if - not os.path.isdir(os.path.join(metadata_path, item))) - + keep_files = [item for item in os.listdir(metadata_path) if + not os.path.isdir(os.path.join(metadata_path, item))] for item in keep_files: - dest_dir = os.path.dirname(os.path.join(temp_path, item)) - if not os.path.exists(dest_dir): - os.makedirs(dest_dir) shutil.copyfile(os.path.join(metadata_path, item), os.path.join(temp_path, item)) diff --git a/testing/web-platform/harness/wptrunner/testloader.py b/testing/web-platform/harness/wptrunner/testloader.py index 2e5f3a51a45..c431c840d12 100644 --- a/testing/web-platform/harness/wptrunner/testloader.py +++ b/testing/web-platform/harness/wptrunner/testloader.py @@ -496,7 +496,7 @@ class TestLoader(object): def iter_tests(self): manifest_items = [] - for manifest in sorted(self.manifests.keys()): + for manifest in self.manifests.keys(): manifest_iter = iterfilter(self.manifest_filters, manifest.itertypes(*self.test_types)) manifest_items.extend(manifest_iter) diff --git a/testing/web-platform/harness/wptrunner/update/sync.py b/testing/web-platform/harness/wptrunner/update/sync.py index c9d9ba51edc..68db2d3285b 100644 --- a/testing/web-platform/harness/wptrunner/update/sync.py +++ b/testing/web-platform/harness/wptrunner/update/sync.py @@ -124,14 +124,12 @@ class GetSyncTargetCommit(Step): class LoadManifest(Step): """Load the test manifest""" - provides = ["test_manifest", "manifest_path"] + provides = ["test_manifest"] def create(self, state): - from manifest import manifest - state.manifest_path = os.path.join(state.metadata_path, "MANIFEST.json") - # Conservatively always rebuild the manifest when doing a sync - old_manifest = manifest.load(state.tests_path, state.manifest_path) - state.test_manifest = manifest.Manifest(old_manifest.rev, "/") + state.test_manifest = testloader.ManifestLoader(state.tests_path).load_manifest( + state.tests_path, state.metadata_path, + ) class UpdateManifest(Step): @@ -140,9 +138,10 @@ class UpdateManifest(Step): provides = ["initial_rev"] def create(self, state): from manifest import manifest, update - state.initial_rev = state.test_manifest.rev - update.update(state.sync["path"], "/", state.test_manifest) - manifest.write(state.test_manifest, state.manifest_path) + test_manifest = state.test_manifest + state.initial_rev = test_manifest.rev + update.update(state.sync["path"], "/", test_manifest) + manifest.write(test_manifest, os.path.join(state.metadata_path, "MANIFEST.json")) class CopyWorkTree(Step):