Bug 1108786 - Update to most recent wptrunner. r=jgraham

testing/web-platform/harness/wptrunner/browsers/base.py

@@ -85,11 +85,6 @@ class Browser(object):
         """Stop the running browser process."""
         pass
 
-    @abstractmethod
-    def on_output(self, line):
-        """Callback function used with ProcessHandler to handle output from the browser process."""
-        pass
-
     @abstractmethod
     def pid(self):
         """pid of the browser process or None if there is no pid"""

testing/web-platform/harness/wptrunner/browsers/chrome.py

@@ -2,16 +2,11 @@
 # License, v. 2.0. If a copy of the MPL was not distributed with this file,
 # You can obtain one at http://mozilla.org/MPL/2.0/.
 
-import os
-
-import mozprocess
-
-from .base import get_free_port, Browser, ExecutorBrowser, require_arg, cmd_arg
+from .base import Browser, ExecutorBrowser, require_arg
+from .webdriver import ChromedriverLocalServer
 from ..executors.executorselenium import SeleniumTestharnessExecutor, required_files
 
 
-here = os.path.split(__file__)[0]
-
 __wptrunner__ = {"product": "chrome",
                  "check_args": "check_args",
                  "browser": "ChromeBrowser",
@@ -26,19 +21,23 @@ def check_args(**kwargs):
 
 
 def browser_kwargs(**kwargs):
-    return {"binary": kwargs["binary"]}
+    return {"binary": kwargs["binary"],
+            "webdriver_binary": kwargs["webdriver_binary"]}
 
 
 def executor_kwargs(http_server_url, **kwargs):
-    from selenium import webdriver
+    from selenium.webdriver import DesiredCapabilities
 
     timeout_multiplier = kwargs["timeout_multiplier"]
     if timeout_multiplier is None:
         timeout_multiplier = 1
+    binary = kwargs["binary"]
+    capabilities = dict(DesiredCapabilities.CHROME.items() +
+                        {"chromeOptions": {"binary": binary}}.items())
 
     return {"http_server_url": http_server_url,
-            "timeout_multiplier": timeout_multiplier,
-            "capabilities": webdriver.DesiredCapabilities.CHROME}
+            "capabilities": capabilities,
+            "timeout_multiplier": timeout_multiplier}
 
 
 def env_options():
@@ -48,42 +47,33 @@ def env_options():
 
 
 class ChromeBrowser(Browser):
-    used_ports = set()
+    """Chrome is backed by chromedriver, which is supplied through
+    ``browsers.webdriver.ChromedriverLocalServer``."""
 
-    def __init__(self, logger, binary):
+    def __init__(self, logger, binary, webdriver_binary="chromedriver"):
+        """Creates a new representation of Chrome.  The `binary` argument gives
+        the browser binary to use for testing."""
         Browser.__init__(self, logger)
         self.binary = binary
-        self.webdriver_port = get_free_port(4444, exclude=self.used_ports)
-        self.used_ports.add(self.webdriver_port)
-        self.proc = None
-        self.cmd = None
+        self.driver = ChromedriverLocalServer(self.logger, binary=webdriver_binary)
 
     def start(self):
-        self.cmd = [self.binary,
-                    cmd_arg("port", str(self.webdriver_port)),
-                    cmd_arg("url-base", "wd/url")]
-        self.proc = mozprocess.ProcessHandler(self.cmd, processOutputLine=self.on_output)
-        self.logger.debug("Starting chromedriver")
-        self.proc.run()
+        self.driver.start()
 
     def stop(self):
-        if self.proc is not None and hasattr(self.proc, "proc"):
-            self.proc.kill()
+        self.driver.stop()
 
     def pid(self):
-        if self.proc is not None:
-            return self.proc.pid
-
-    def on_output(self, line):
-        self.logger.process_output(self.pid(),
-                                   line.decode("utf8", "replace"),
-                                   command=" ".join(self.cmd))
+        return self.driver.pid
 
     def is_alive(self):
-        return self.pid() is not None
+        # TODO(ato): This only indicates the driver is alive,
+        # and doesn't say anything about whether a browser session
+        # is active.
+        return self.driver.is_alive()
 
     def cleanup(self):
         self.stop()
 
     def executor_browser(self):
-        return ExecutorBrowser, {"webdriver_port": self.webdriver_port}
+        return ExecutorBrowser, {"webdriver_url": self.driver.url}

testing/web-platform/harness/wptrunner/browsers/webdriver.py

@@ -0,0 +1,137 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this file,
+# You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import errno
+import socket
+import time
+import traceback
+import urlparse
+
+import mozprocess
+
+from .base import get_free_port, cmd_arg
+
+
+__all__ = ["SeleniumLocalServer", "ChromedriverLocalServer"]
+
+
+class LocalServer(object):
+    used_ports = set()
+    default_endpoint = "/"
+
+    def __init__(self, logger, binary, port=None, endpoint=None):
+        self.logger = logger
+        self.binary = binary
+        self.port = port
+        self.endpoint = endpoint or self.default_endpoint
+
+        if self.port is None:
+            self.port = get_free_port(4444, exclude=self.used_ports)
+        self.used_ports.add(self.port)
+        self.url = "http://127.0.0.1:%i%s" % (self.port, self.endpoint)
+
+        self.proc, self.cmd = None, None
+
+    def start(self):
+        self.proc = mozprocess.ProcessHandler(
+            self.cmd, processOutputLine=self.on_output)
+        try:
+            self.proc.run()
+        except OSError as e:
+            if e.errno == errno.ENOENT:
+                raise IOError(
+                    "chromedriver executable not found: %s" % self.binary)
+            raise
+
+        self.logger.debug(
+            "Waiting for server to become accessible: %s" % self.url)
+        surl = urlparse.urlparse(self.url)
+        addr = (surl.hostname, surl.port)
+        try:
+            wait_service(addr)
+        except:
+            self.logger.error(
+                "Server was not accessible within the timeout:\n%s" % traceback.format_exc())
+            raise
+        else:
+            self.logger.info("Server listening on port %i" % self.port)
+
+    def stop(self):
+        if hasattr(self.proc, "proc"):
+            self.proc.kill()
+
+    def is_alive(self):
+        if hasattr(self.proc, "proc"):
+            exitcode = self.proc.poll()
+            return exitcode is None
+        return False
+
+    def on_output(self, line):
+        self.logger.process_output(self.pid,
+                                   line.decode("utf8", "replace"),
+                                   command=" ".join(self.cmd))
+
+    @property
+    def pid(self):
+        if hasattr(self.proc, "proc"):
+            return self.proc.pid
+
+
+class SeleniumLocalServer(LocalServer):
+    default_endpoint = "/wd/hub"
+
+    def __init__(self, logger, binary, port=None):
+        LocalServer.__init__(self, logger, binary, port=port)
+        self.cmd = ["java",
+                    "-jar", self.binary,
+                    "-port", str(self.port)]
+
+    def start(self):
+        self.logger.debug("Starting local Selenium server")
+        LocalServer.start(self)
+
+    def stop(self):
+        LocalServer.stop(self)
+        self.logger.info("Selenium server stopped listening")
+
+
+class ChromedriverLocalServer(LocalServer):
+    default_endpoint = "/wd/hub"
+
+    def __init__(self, logger, binary="chromedriver", port=None, endpoint=None):
+        LocalServer.__init__(self, logger, binary, port=port, endpoint=endpoint)
+        # TODO: verbose logging
+        self.cmd = [self.binary,
+                    cmd_arg("port", str(self.port)) if self.port else "",
+                    cmd_arg("url-base", self.endpoint) if self.endpoint else ""]
+
+    def start(self):
+        self.logger.debug("Starting local chromedriver server")
+        LocalServer.start(self)
+
+    def stop(self):
+        LocalServer.stop(self)
+        self.logger.info("chromedriver server stopped listening")
+
+
+def wait_service(addr, timeout=15):
+    """Waits until network service given as a tuple of (host, port) becomes
+    available or the `timeout` duration is reached, at which point
+    ``socket.error`` is raised."""
+    end = time.time() + timeout
+    while end > time.time():
+        so = socket.socket()
+        try:
+            so.connect(addr)
+        except socket.timeout:
+            pass
+        except socket.error as e:
+            if e[0] != errno.ECONNREFUSED:
+                raise
+        else:
+            return True
+        finally:
+            so.close()
+        time.sleep(0.5)
+    raise socket.error("Service is unavailable: %s:%i" % addr)

testing/web-platform/harness/wptrunner/executors/executormarionette.py

@@ -201,7 +201,6 @@ class MarionetteTestharnessExecutor(MarionetteTestExecutor):
         self.script = open(os.path.join(here, "testharness_marionette.js")).read()
 
     def do_test(self, test, timeout):
-        assert len(self.marionette.window_handles) == 1
         if self.close_after_done:
             self.marionette.execute_script("if (window.wrappedJSObject.win) {window.wrappedJSObject.win.close()}")
 

testing/web-platform/harness/wptrunner/executors/executorselenium.py

@@ -32,33 +32,28 @@ def do_delayed_imports():
 
 
 class SeleniumTestExecutor(TestExecutor):
-    def __init__(self, browser, http_server_url, timeout_multiplier=1,
-                 **kwargs):
+    def __init__(self, browser, http_server_url, capabilities,
+                 timeout_multiplier=1, **kwargs):
         do_delayed_imports()
         TestExecutor.__init__(self, browser, http_server_url, timeout_multiplier)
-        self.webdriver_port = browser.webdriver_port
+        self.capabilities = capabilities
+        self.url = browser.webdriver_url
         self.webdriver = None
-
         self.timer = None
         self.window_id = str(uuid.uuid4())
-        self.capabilities = kwargs.pop("capabilities")
 
     def setup(self, runner):
         """Connect to browser via Selenium's WebDriver implementation."""
         self.runner = runner
-        url = "http://localhost:%i/wd/url" % self.webdriver_port
-        self.logger.debug("Connecting to Selenium on URL: %s" % url)
+        self.logger.debug("Connecting to Selenium on URL: %s" % self.url)
 
         session_started = False
         try:
-            time.sleep(1)
             self.webdriver = webdriver.Remote(
-                url, desired_capabilities=self.capabilities)
-            time.sleep(10)
+                self.url, desired_capabilities=self.capabilities)
         except:
             self.logger.warning(
                 "Connecting to Selenium failed:\n%s" % traceback.format_exc())
-            time.sleep(1)
         else:
             self.logger.debug("Selenium session started")
             session_started = True
@@ -78,6 +73,7 @@ class SeleniumTestExecutor(TestExecutor):
                 self.runner.send_message("init_succeeded")
 
     def teardown(self):
+        self.logger.debug("Hanging up on Selenium session")
         try:
             self.webdriver.quit()
         except:

testing/web-platform/harness/wptrunner/testrunner.py

@@ -322,8 +322,8 @@ class TestRunnerManager(threading.Thread):
                 self.child_stop_flag.set()
 
         with self.init_lock:
-            # To guard against cases where we fail to connect with marionette for
-            # whatever reason
+	    # Guard against problems initialising the browser or the browser
+	    # remote control method
             self.init_timer = threading.Timer(self.browser.init_timeout, init_failed)
             test_queue = self.test_source.get_queue()
             if test_queue is None:
@@ -348,16 +348,16 @@ class TestRunnerManager(threading.Thread):
             self.init_failed()
 
     def init_succeeded(self):
-        """Callback when we have started the browser, connected via
-        marionette, and we are ready to start testing"""
+	"""Callback when we have started the browser, started the remote
+	control connection, and we are ready to start testing."""
         self.logger.debug("Init succeeded")
         self.init_timer.cancel()
         self.init_fail_count = 0
         self.start_next_test()
 
     def init_failed(self):
-        """Callback when we can't connect to the browser via
-        marionette for some reason"""
+	"""Callback when starting the browser or the remote control connect
+	fails."""
         self.init_fail_count += 1
         self.logger.warning("Init failed %i" % self.init_fail_count)
         self.init_timer.cancel()

testing/web-platform/harness/wptrunner/update.py

@@ -333,12 +333,7 @@ def sync_tests(paths, local_tree, wpt, bug):
                             "metadata_path": paths["sync_dest"]["metadata_path"]}}
 
         manifest_loader = testloader.ManifestLoader(sync_paths)
-        test_manifest = manifest_loader.load_manifest(**sync_paths["/"])
-
-        initial_rev = test_manifest.rev
-        manifest.update(sync_paths["/"]["tests_path"], "/", test_manifest)
-        manifest.write(test_manifest, os.path.join(sync_paths["/"]["metadata_path"], "MANIFEST.json"))
-
+        initial_manifests = manifest_loader.load()
         wpt.copy_work_tree(paths["sync_dest"]["tests_path"])
 
         local_tree.create_patch("web-platform-tests_update_%s" % wpt.rev,
@@ -354,7 +349,7 @@ def sync_tests(paths, local_tree, wpt, bug):
     finally:
         pass  # wpt.clean()
 
-    return initial_rev
+    return initial_manifests
 
 
 def update_metadata(paths, local_tree, initial_rev, bug, log_files, ignore_existing,
@@ -445,7 +440,12 @@ expected data."""
         wpt_repo = WebPlatformTests(config["web-platform-tests"]["remote_url"],
                                     paths["sync"],
                                     rev=rev)
-        initial_rev = sync_tests(paths, local_tree, wpt_repo, bug)
+        initial_manifests = sync_tests(paths, local_tree, wpt_repo, bug)
+        initial_rev = None
+        for manifest, path_data in initial_manifests.iteritems():
+            if path_data["url_base"] == "/":
+                initial_rev = manifest.rev
+                break
 
     if kwargs["run_log"]:
         update_metadata(paths,

testing/web-platform/harness/wptrunner/wptcommandline.py

@@ -10,9 +10,11 @@ from collections import OrderedDict
 
 import config
 
+
 def abs_path(path):
     return os.path.abspath(os.path.expanduser(path))
 
+
 def url_or_path(path):
     import urlparse
 
@@ -66,6 +68,8 @@ def create_parser(product_choices=None):
 
     parser.add_argument("--binary", action="store",
                         type=abs_path, help="Binary to run tests against")
+    parser.add_argument("--webdriver-binary", action="store", metavar="BINARY",
+                        type=abs_path, help="WebDriver server binary to use")
     parser.add_argument("--test-types", action="store",
                         nargs="*", default=["testharness", "reftest"],
                         choices=["testharness", "reftest"],
@@ -283,12 +287,14 @@ def parse_args():
     check_args(rv)
     return rv
 
+
 def parse_args_update():
     parser = create_parser_update()
     rv = vars(parser.parse_args())
     set_from_config(rv)
     return rv
 
+
 def parse_args_reduce():
     parser = create_parser_reduce()
     rv = vars(parser.parse_args())