diff --git a/testing/web-platform/harness/wptrunner/browsers/base.py b/testing/web-platform/harness/wptrunner/browsers/base.py index 655147d5a25..8afe887bb3d 100644 --- a/testing/web-platform/harness/wptrunner/browsers/base.py +++ b/testing/web-platform/harness/wptrunner/browsers/base.py @@ -85,11 +85,6 @@ class Browser(object): """Stop the running browser process.""" pass - @abstractmethod - def on_output(self, line): - """Callback function used with ProcessHandler to handle output from the browser process.""" - pass - @abstractmethod def pid(self): """pid of the browser process or None if there is no pid""" diff --git a/testing/web-platform/harness/wptrunner/browsers/chrome.py b/testing/web-platform/harness/wptrunner/browsers/chrome.py index 6cf2fcf08a6..370e66e7f32 100644 --- a/testing/web-platform/harness/wptrunner/browsers/chrome.py +++ b/testing/web-platform/harness/wptrunner/browsers/chrome.py @@ -2,16 +2,11 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this file, # You can obtain one at http://mozilla.org/MPL/2.0/. -import os - -import mozprocess - -from .base import get_free_port, Browser, ExecutorBrowser, require_arg, cmd_arg +from .base import Browser, ExecutorBrowser, require_arg +from .webdriver import ChromedriverLocalServer from ..executors.executorselenium import SeleniumTestharnessExecutor, required_files -here = os.path.split(__file__)[0] - __wptrunner__ = {"product": "chrome", "check_args": "check_args", "browser": "ChromeBrowser", @@ -26,19 +21,23 @@ def check_args(**kwargs): def browser_kwargs(**kwargs): - return {"binary": kwargs["binary"]} + return {"binary": kwargs["binary"], + "webdriver_binary": kwargs["webdriver_binary"]} def executor_kwargs(http_server_url, **kwargs): - from selenium import webdriver + from selenium.webdriver import DesiredCapabilities timeout_multiplier = kwargs["timeout_multiplier"] if timeout_multiplier is None: timeout_multiplier = 1 + binary = kwargs["binary"] + capabilities = dict(DesiredCapabilities.CHROME.items() + + {"chromeOptions": {"binary": binary}}.items()) return {"http_server_url": http_server_url, - "timeout_multiplier": timeout_multiplier, - "capabilities": webdriver.DesiredCapabilities.CHROME} + "capabilities": capabilities, + "timeout_multiplier": timeout_multiplier} def env_options(): @@ -48,42 +47,33 @@ def env_options(): class ChromeBrowser(Browser): - used_ports = set() + """Chrome is backed by chromedriver, which is supplied through + ``browsers.webdriver.ChromedriverLocalServer``.""" - def __init__(self, logger, binary): + def __init__(self, logger, binary, webdriver_binary="chromedriver"): + """Creates a new representation of Chrome. The `binary` argument gives + the browser binary to use for testing.""" Browser.__init__(self, logger) self.binary = binary - self.webdriver_port = get_free_port(4444, exclude=self.used_ports) - self.used_ports.add(self.webdriver_port) - self.proc = None - self.cmd = None + self.driver = ChromedriverLocalServer(self.logger, binary=webdriver_binary) def start(self): - self.cmd = [self.binary, - cmd_arg("port", str(self.webdriver_port)), - cmd_arg("url-base", "wd/url")] - self.proc = mozprocess.ProcessHandler(self.cmd, processOutputLine=self.on_output) - self.logger.debug("Starting chromedriver") - self.proc.run() + self.driver.start() def stop(self): - if self.proc is not None and hasattr(self.proc, "proc"): - self.proc.kill() + self.driver.stop() def pid(self): - if self.proc is not None: - return self.proc.pid - - def on_output(self, line): - self.logger.process_output(self.pid(), - line.decode("utf8", "replace"), - command=" ".join(self.cmd)) + return self.driver.pid def is_alive(self): - return self.pid() is not None + # TODO(ato): This only indicates the driver is alive, + # and doesn't say anything about whether a browser session + # is active. + return self.driver.is_alive() def cleanup(self): self.stop() def executor_browser(self): - return ExecutorBrowser, {"webdriver_port": self.webdriver_port} + return ExecutorBrowser, {"webdriver_url": self.driver.url} diff --git a/testing/web-platform/harness/wptrunner/browsers/webdriver.py b/testing/web-platform/harness/wptrunner/browsers/webdriver.py new file mode 100644 index 00000000000..54b1c1b5ef9 --- /dev/null +++ b/testing/web-platform/harness/wptrunner/browsers/webdriver.py @@ -0,0 +1,137 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. + +import errno +import socket +import time +import traceback +import urlparse + +import mozprocess + +from .base import get_free_port, cmd_arg + + +__all__ = ["SeleniumLocalServer", "ChromedriverLocalServer"] + + +class LocalServer(object): + used_ports = set() + default_endpoint = "/" + + def __init__(self, logger, binary, port=None, endpoint=None): + self.logger = logger + self.binary = binary + self.port = port + self.endpoint = endpoint or self.default_endpoint + + if self.port is None: + self.port = get_free_port(4444, exclude=self.used_ports) + self.used_ports.add(self.port) + self.url = "http://127.0.0.1:%i%s" % (self.port, self.endpoint) + + self.proc, self.cmd = None, None + + def start(self): + self.proc = mozprocess.ProcessHandler( + self.cmd, processOutputLine=self.on_output) + try: + self.proc.run() + except OSError as e: + if e.errno == errno.ENOENT: + raise IOError( + "chromedriver executable not found: %s" % self.binary) + raise + + self.logger.debug( + "Waiting for server to become accessible: %s" % self.url) + surl = urlparse.urlparse(self.url) + addr = (surl.hostname, surl.port) + try: + wait_service(addr) + except: + self.logger.error( + "Server was not accessible within the timeout:\n%s" % traceback.format_exc()) + raise + else: + self.logger.info("Server listening on port %i" % self.port) + + def stop(self): + if hasattr(self.proc, "proc"): + self.proc.kill() + + def is_alive(self): + if hasattr(self.proc, "proc"): + exitcode = self.proc.poll() + return exitcode is None + return False + + def on_output(self, line): + self.logger.process_output(self.pid, + line.decode("utf8", "replace"), + command=" ".join(self.cmd)) + + @property + def pid(self): + if hasattr(self.proc, "proc"): + return self.proc.pid + + +class SeleniumLocalServer(LocalServer): + default_endpoint = "/wd/hub" + + def __init__(self, logger, binary, port=None): + LocalServer.__init__(self, logger, binary, port=port) + self.cmd = ["java", + "-jar", self.binary, + "-port", str(self.port)] + + def start(self): + self.logger.debug("Starting local Selenium server") + LocalServer.start(self) + + def stop(self): + LocalServer.stop(self) + self.logger.info("Selenium server stopped listening") + + +class ChromedriverLocalServer(LocalServer): + default_endpoint = "/wd/hub" + + def __init__(self, logger, binary="chromedriver", port=None, endpoint=None): + LocalServer.__init__(self, logger, binary, port=port, endpoint=endpoint) + # TODO: verbose logging + self.cmd = [self.binary, + cmd_arg("port", str(self.port)) if self.port else "", + cmd_arg("url-base", self.endpoint) if self.endpoint else ""] + + def start(self): + self.logger.debug("Starting local chromedriver server") + LocalServer.start(self) + + def stop(self): + LocalServer.stop(self) + self.logger.info("chromedriver server stopped listening") + + +def wait_service(addr, timeout=15): + """Waits until network service given as a tuple of (host, port) becomes + available or the `timeout` duration is reached, at which point + ``socket.error`` is raised.""" + end = time.time() + timeout + while end > time.time(): + so = socket.socket() + try: + so.connect(addr) + except socket.timeout: + pass + except socket.error as e: + if e[0] != errno.ECONNREFUSED: + raise + else: + return True + finally: + so.close() + time.sleep(0.5) + raise socket.error("Service is unavailable: %s:%i" % addr) diff --git a/testing/web-platform/harness/wptrunner/executors/executormarionette.py b/testing/web-platform/harness/wptrunner/executors/executormarionette.py index f365c2c7a9a..a316b4be259 100644 --- a/testing/web-platform/harness/wptrunner/executors/executormarionette.py +++ b/testing/web-platform/harness/wptrunner/executors/executormarionette.py @@ -201,7 +201,6 @@ class MarionetteTestharnessExecutor(MarionetteTestExecutor): self.script = open(os.path.join(here, "testharness_marionette.js")).read() def do_test(self, test, timeout): - assert len(self.marionette.window_handles) == 1 if self.close_after_done: self.marionette.execute_script("if (window.wrappedJSObject.win) {window.wrappedJSObject.win.close()}") diff --git a/testing/web-platform/harness/wptrunner/executors/executorselenium.py b/testing/web-platform/harness/wptrunner/executors/executorselenium.py index 35ed5a43fb2..e6048c08155 100644 --- a/testing/web-platform/harness/wptrunner/executors/executorselenium.py +++ b/testing/web-platform/harness/wptrunner/executors/executorselenium.py @@ -32,33 +32,28 @@ def do_delayed_imports(): class SeleniumTestExecutor(TestExecutor): - def __init__(self, browser, http_server_url, timeout_multiplier=1, - **kwargs): + def __init__(self, browser, http_server_url, capabilities, + timeout_multiplier=1, **kwargs): do_delayed_imports() TestExecutor.__init__(self, browser, http_server_url, timeout_multiplier) - self.webdriver_port = browser.webdriver_port + self.capabilities = capabilities + self.url = browser.webdriver_url self.webdriver = None - self.timer = None self.window_id = str(uuid.uuid4()) - self.capabilities = kwargs.pop("capabilities") def setup(self, runner): """Connect to browser via Selenium's WebDriver implementation.""" self.runner = runner - url = "http://localhost:%i/wd/url" % self.webdriver_port - self.logger.debug("Connecting to Selenium on URL: %s" % url) + self.logger.debug("Connecting to Selenium on URL: %s" % self.url) session_started = False try: - time.sleep(1) self.webdriver = webdriver.Remote( - url, desired_capabilities=self.capabilities) - time.sleep(10) + self.url, desired_capabilities=self.capabilities) except: self.logger.warning( "Connecting to Selenium failed:\n%s" % traceback.format_exc()) - time.sleep(1) else: self.logger.debug("Selenium session started") session_started = True @@ -78,6 +73,7 @@ class SeleniumTestExecutor(TestExecutor): self.runner.send_message("init_succeeded") def teardown(self): + self.logger.debug("Hanging up on Selenium session") try: self.webdriver.quit() except: diff --git a/testing/web-platform/harness/wptrunner/testrunner.py b/testing/web-platform/harness/wptrunner/testrunner.py index 8a88a73c2b5..57c4e965b84 100644 --- a/testing/web-platform/harness/wptrunner/testrunner.py +++ b/testing/web-platform/harness/wptrunner/testrunner.py @@ -322,8 +322,8 @@ class TestRunnerManager(threading.Thread): self.child_stop_flag.set() with self.init_lock: - # To guard against cases where we fail to connect with marionette for - # whatever reason + # Guard against problems initialising the browser or the browser + # remote control method self.init_timer = threading.Timer(self.browser.init_timeout, init_failed) test_queue = self.test_source.get_queue() if test_queue is None: @@ -348,16 +348,16 @@ class TestRunnerManager(threading.Thread): self.init_failed() def init_succeeded(self): - """Callback when we have started the browser, connected via - marionette, and we are ready to start testing""" + """Callback when we have started the browser, started the remote + control connection, and we are ready to start testing.""" self.logger.debug("Init succeeded") self.init_timer.cancel() self.init_fail_count = 0 self.start_next_test() def init_failed(self): - """Callback when we can't connect to the browser via - marionette for some reason""" + """Callback when starting the browser or the remote control connect + fails.""" self.init_fail_count += 1 self.logger.warning("Init failed %i" % self.init_fail_count) self.init_timer.cancel() diff --git a/testing/web-platform/harness/wptrunner/update.py b/testing/web-platform/harness/wptrunner/update.py index 905c193e8ec..e0c9fba41f8 100644 --- a/testing/web-platform/harness/wptrunner/update.py +++ b/testing/web-platform/harness/wptrunner/update.py @@ -333,12 +333,7 @@ def sync_tests(paths, local_tree, wpt, bug): "metadata_path": paths["sync_dest"]["metadata_path"]}} manifest_loader = testloader.ManifestLoader(sync_paths) - test_manifest = manifest_loader.load_manifest(**sync_paths["/"]) - - initial_rev = test_manifest.rev - manifest.update(sync_paths["/"]["tests_path"], "/", test_manifest) - manifest.write(test_manifest, os.path.join(sync_paths["/"]["metadata_path"], "MANIFEST.json")) - + initial_manifests = manifest_loader.load() wpt.copy_work_tree(paths["sync_dest"]["tests_path"]) local_tree.create_patch("web-platform-tests_update_%s" % wpt.rev, @@ -354,7 +349,7 @@ def sync_tests(paths, local_tree, wpt, bug): finally: pass # wpt.clean() - return initial_rev + return initial_manifests def update_metadata(paths, local_tree, initial_rev, bug, log_files, ignore_existing, @@ -445,7 +440,12 @@ expected data.""" wpt_repo = WebPlatformTests(config["web-platform-tests"]["remote_url"], paths["sync"], rev=rev) - initial_rev = sync_tests(paths, local_tree, wpt_repo, bug) + initial_manifests = sync_tests(paths, local_tree, wpt_repo, bug) + initial_rev = None + for manifest, path_data in initial_manifests.iteritems(): + if path_data["url_base"] == "/": + initial_rev = manifest.rev + break if kwargs["run_log"]: update_metadata(paths, diff --git a/testing/web-platform/harness/wptrunner/wptcommandline.py b/testing/web-platform/harness/wptrunner/wptcommandline.py index 659cd2a96cb..6317586bc09 100644 --- a/testing/web-platform/harness/wptrunner/wptcommandline.py +++ b/testing/web-platform/harness/wptrunner/wptcommandline.py @@ -10,9 +10,11 @@ from collections import OrderedDict import config + def abs_path(path): return os.path.abspath(os.path.expanduser(path)) + def url_or_path(path): import urlparse @@ -66,6 +68,8 @@ def create_parser(product_choices=None): parser.add_argument("--binary", action="store", type=abs_path, help="Binary to run tests against") + parser.add_argument("--webdriver-binary", action="store", metavar="BINARY", + type=abs_path, help="WebDriver server binary to use") parser.add_argument("--test-types", action="store", nargs="*", default=["testharness", "reftest"], choices=["testharness", "reftest"], @@ -283,12 +287,14 @@ def parse_args(): check_args(rv) return rv + def parse_args_update(): parser = create_parser_update() rv = vars(parser.parse_args()) set_from_config(rv) return rv + def parse_args_reduce(): parser = create_parser_reduce() rv = vars(parser.parse_args())