Bug 1195288 - consider using python webserver for production talos. r=jmaher

Always use an in-process webserver, removing the need for apache - and
hopefuly providing better accuracy for numbers.

This means that we know have to copy the pagesets in the talos dir on
harness.

On windows, some pagesets paths were too long due to that, so the
solution is to replace "page_load_test" with "tests".

testing/mozharness/configs/talos/linux_config.py

@@ -35,8 +35,6 @@ config = {
         "install",
         "run-tests",
     ],
-    "python_webserver": False,
-    "webroot": '%s/../talos-data' % os.getcwd(),
     "default_blob_upload_servers": [
         "https://blobupload.elasticbeanstalk.com",
     ],

testing/mozharness/configs/talos/mac_config.py

@@ -38,8 +38,6 @@ config = {
         "install",
         "run-tests",
     ],
-    "python_webserver": False,
-    "webroot": '%s/../talos-data' % os.getcwd(),
     "run_cmd_checks_enabled": True,
     "preflight_run_cmd_suites": [
         SCREEN_RESOLUTION_CHECK,

testing/mozharness/configs/talos/windows_config.py

@@ -37,10 +37,6 @@ config = {
         "install",
         "run-tests",
     ],
-    "python_webserver": False,
-    "webroot": 'c:/slave/talos-data',
-    # Srsly gly? Ys
-    "webroot_extract_cmd": r'''c:/mozilla-build/msys/bin/bash -c "PATH=/c/mozilla-build/msys/bin:$PATH tar zx --strip-components=1 -f '%(tarball)s' --wildcards '**/talos/'"''',
     "default_blob_upload_servers": [
         "https://blobupload.elasticbeanstalk.com",
     ],

testing/mozharness/mozharness/mozilla/testing/talos.py

@@ -157,11 +157,6 @@ class Talos(TestingMixin, MercurialScript, BlobUploadMixin):
         self.talos_json_config = self.config.get("talos_json_config")
         self.tests = None
         self.pagesets_url = None
-        self.pagesets_parent_dir_path = None
-        self.pagesets_manifest_path = None
-        self.abs_pagesets_paths = None
-        self.pagesets_manifest_filename = None
-        self.pagesets_manifest_parent_path = None
         self.sps_profile = self.config.get('sps_profile')
         self.sps_profile_interval = self.config.get('sps_profile_interval')
 
@@ -270,62 +265,6 @@ class Talos(TestingMixin, MercurialScript, BlobUploadMixin):
             self.pagesets_url = self.talos_json_config['suites'][self.config['suite']].get('pagesets_url')
             return self.pagesets_url
 
-    def query_pagesets_parent_dir_path(self):
-        """ We have to copy the pageset into the webroot separately.
-
-        Helper method to avoid hardcodes.
-        """
-        if self.pagesets_parent_dir_path:
-            return self.pagesets_parent_dir_path
-        if self.query_talos_json_config():
-            self.pagesets_parent_dir_path = self.talos_json_config['suites'][self.config['suite']].get('pagesets_parent_dir_path')
-            return self.pagesets_parent_dir_path
-
-    def query_pagesets_manifest_path(self):
-        """ We have to copy the tp manifest from webroot to talos root when
-        those two directories aren't the same, until bug 795172 is fixed.
-
-        Helper method to avoid hardcodes.
-        """
-        if self.pagesets_manifest_path:
-            return self.pagesets_manifest_path
-        if self.query_talos_json_config():
-            self.pagesets_manifest_path = self.talos_json_config['suites'][self.config['suite']].get('pagesets_manifest_path')
-            return self.pagesets_manifest_path
-
-    def query_pagesets_manifest_filename(self):
-        if self.pagesets_manifest_filename:
-            return self.pagesets_manifest_filename
-        else:
-            manifest_path = self.query_pagesets_manifest_path()
-            self.pagesets_manifest_filename = os.path.basename(manifest_path)
-            return self.pagesets_manifest_filename
-
-    def query_pagesets_manifest_parent_path(self):
-        if self.pagesets_manifest_parent_path:
-            return self.pagesets_manifest_parent_path
-        if self.query_talos_json_config():
-            manifest_path = self.query_pagesets_manifest_path()
-            self.pagesets_manifest_parent_path = os.path.dirname(manifest_path)
-            return self.pagesets_manifest_parent_path
-
-    def query_abs_pagesets_paths(self):
-        """ Returns a bunch of absolute pagesets directory paths.
-        We need this to make the dir and copy the manifest to the local dir.
-        """
-        if self.abs_pagesets_paths:
-            return self.abs_pagesets_paths
-        else:
-            paths = {}
-            manifest_parent_path = self.query_pagesets_manifest_parent_path()
-            paths['pagesets_manifest_parent'] = os.path.join(self.talos_path, manifest_parent_path)
-
-            manifest_path = self.query_pagesets_manifest_path()
-            paths['pagesets_manifest'] = os.path.join(self.talos_path, manifest_path)
-
-            self.abs_pagesets_paths = paths
-            return self.abs_pagesets_paths
-
     def talos_options(self, args=None, **kw):
         """return options to talos"""
         # binary path
@@ -335,8 +274,6 @@ class Talos(TestingMixin, MercurialScript, BlobUploadMixin):
 
         # talos options
         options = []
-        if self.config.get('python_webserver', True):
-            options.append('--develop')
         # talos can't gather data if the process name ends with '.exe'
         if binary_path.endswith('.exe'):
             binary_path = binary_path[:-4]
@@ -376,12 +313,6 @@ class Talos(TestingMixin, MercurialScript, BlobUploadMixin):
     def populate_webroot(self):
         """Populate the production test slaves' webroots"""
         c = self.config
-        if not c.get('webroot'):
-            self.fatal("webroot need to be set to populate_webroot!")
-        self.info("Populating webroot %s..." % c['webroot'])
-        talos_webdir = os.path.join(c['webroot'], 'talos')
-        self.mkdir_p(c['webroot'], error_level=FATAL)
-        self.rmtree(talos_webdir, error_level=FATAL)
 
         self.talos_path = os.path.join(
             self.query_abs_dirs()['abs_work_dir'], 'tests', 'talos'
@@ -389,25 +320,12 @@ class Talos(TestingMixin, MercurialScript, BlobUploadMixin):
         if c.get('run_local'):
             self.talos_path = os.path.dirname(self.talos_json)
 
-        # the apache server needs the talos directory (talos/talos)
-        # to be in the webroot
         src_talos_webdir = os.path.join(self.talos_path, 'talos')
-        self.copytree(src_talos_webdir, talos_webdir)
 
         if self.query_pagesets_url():
             self.info("Downloading pageset...")
-            pagesets_path = os.path.join(c['webroot'], self.query_pagesets_parent_dir_path())
-            self._download_unzip(self.pagesets_url, pagesets_path)
-
-            # mkdir for the missing manifest directory in talos_repo/talos/page_load_test directory
-            abs_pagesets_paths = self.query_abs_pagesets_paths()
-            abs_manifest_parent_path = abs_pagesets_paths['pagesets_manifest_parent']
-            self.mkdir_p(abs_manifest_parent_path, error_level=FATAL)
-
-            # copy all the manifest file from unzipped zip file into the manifest dir
-            src_manifest_file = os.path.join(c['webroot'], self.query_pagesets_manifest_path())
-            dest_manifest_file = abs_pagesets_paths['pagesets_manifest']
-            self.copyfile(src_manifest_file, dest_manifest_file, error_level=FATAL)
+            src_talos_pageset = os.path.join(src_talos_webdir, 'tests')
+            self._download_unzip(self.pagesets_url, src_talos_pageset)
 
     # Action methods. {{{1
     # clobber defined in BaseScript
@@ -452,20 +370,6 @@ class Talos(TestingMixin, MercurialScript, BlobUploadMixin):
                                        'requirements.txt')]
         )
 
-    def postflight_create_virtualenv(self):
-        """ This belongs in download_and_install() but requires the
-        virtualenv to be set up :(
-
-        The real fix here may be a --tpmanifest option for PerfConfigurator.
-        """
-        c = self.config
-        if not c.get('python_webserver', True) and self.query_pagesets_url():
-            pagesets_path = self.query_pagesets_manifest_path()
-            manifest_source = os.path.join(c['webroot'], pagesets_path)
-            manifest_target = os.path.join(self.query_python_site_packages_path(), pagesets_path)
-            self.mkdir_p(os.path.dirname(manifest_target))
-            self.copyfile(manifest_source, manifest_target)
-
     def run_tests(self, args=None, **kw):
         """run Talos tests"""
 

testing/talos/mach_commands.py

@@ -45,7 +45,6 @@ class TalosRunner(MozbuildObject):
 
     def init_variables(self, talos_args):
         self.talos_dir = os.path.join(self.topsrcdir, 'testing', 'talos')
-        self.talos_webroot = os.path.join(self.topobjdir, 'testing', 'talos')
         self.mozharness_dir = os.path.join(self.topsrcdir, 'testing',
                                            'mozharness')
         self.config_dir = os.path.join(self.mozharness_dir, 'configs', 'talos')
@@ -79,16 +78,13 @@ class TalosRunner(MozbuildObject):
                 'create-virtualenv',
                 'run-tests',
             ],
-            'python_webserver': False,
             'talos_extra_options': ['--develop'] + self.talos_args,
         }
 
     def make_args(self):
         self.args = {
-            'config': {
-                'webroot': self.talos_webroot,
-            },
-           'initial_config_file': self.config_file_path,
+            'config': {},
+            'initial_config_file': self.config_file_path,
        }
 
     def write_config(self):

testing/talos/talos.json

@@ -44,29 +44,21 @@
         },
         "g1": {
             "tests": ["tp5o_scroll", "glterrain"],
-            "pagesets_url": "http://talos-bundles.pvt.build.mozilla.org/zips/tp5n.zip",
-            "pagesets_parent_dir_path": "talos/page_load_test/",
-            "pagesets_manifest_path": "talos/page_load_test/tp5n/tp5o.manifest"
+            "pagesets_url": "http://talos-bundles.pvt.build.mozilla.org/zips/tp5n.zip"
         },
         "g1-e10s": {
             "tests": ["tp5o_scroll", "glterrain"],
             "talos_options": ["--e10s"],
-            "pagesets_url": "http://talos-bundles.pvt.build.mozilla.org/zips/tp5n.zip",
-            "pagesets_parent_dir_path": "talos/page_load_test/",
-            "pagesets_manifest_path": "talos/page_load_test/tp5n/tp5o.manifest"
+            "pagesets_url": "http://talos-bundles.pvt.build.mozilla.org/zips/tp5n.zip"
         },
         "g2": {
             "tests": ["damp", "tps"],
-            "pagesets_url": "http://talos-bundles.pvt.build.mozilla.org/zips/tp5n.zip",
-            "pagesets_parent_dir_path": "talos/page_load_test/",
-            "pagesets_manifest_path": "talos/page_load_test/tp5n/tp5o.manifest"
+            "pagesets_url": "http://talos-bundles.pvt.build.mozilla.org/zips/tp5n.zip"
         },
         "g2-e10s": {
             "tests": ["damp", "tps"],
             "talos_options": ["--e10s"],
-            "pagesets_url": "http://talos-bundles.pvt.build.mozilla.org/zips/tp5n.zip",
-            "pagesets_parent_dir_path": "talos/page_load_test/",
-            "pagesets_manifest_path": "talos/page_load_test/tp5n/tp5o.manifest"
+            "pagesets_url": "http://talos-bundles.pvt.build.mozilla.org/zips/tp5n.zip"
         },
         "g3": {
             "tests": ["dromaeo_dom"]
@@ -83,22 +75,16 @@
         },
         "tp5o": {
             "tests": ["tp5o"],
-            "pagesets_url": "http://talos-bundles.pvt.build.mozilla.org/zips/tp5n.zip",
-            "pagesets_parent_dir_path": "talos/page_load_test/",
-            "pagesets_manifest_path": "talos/page_load_test/tp5n/tp5o.manifest"
+            "pagesets_url": "http://talos-bundles.pvt.build.mozilla.org/zips/tp5n.zip"
         },
         "tp5o-e10s": {
             "tests": ["tp5o"],
             "talos_options": ["--e10s"],
-            "pagesets_url": "http://talos-bundles.pvt.build.mozilla.org/zips/tp5n.zip",
-            "pagesets_parent_dir_path": "talos/page_load_test/",
-            "pagesets_manifest_path": "talos/page_load_test/tp5n/tp5o.manifest"
+            "pagesets_url": "http://talos-bundles.pvt.build.mozilla.org/zips/tp5n.zip"
         },
         "xperf": {
             "tests": ["tp5n"],
             "pagesets_url": "http://talos-bundles.pvt.build.mozilla.org/zips/tp5n.zip",
-            "pagesets_parent_dir_path": "talos/page_load_test/",
-            "pagesets_manifest_path": "talos/page_load_test/tp5n/tp5n.manifest",
             "talos_options": [
                 "--xperf_path",
                 "\"c:/Program Files/Microsoft Windows Performance Toolkit/xperf.exe\""
@@ -107,8 +93,6 @@
         "xperf-e10s": {
             "tests": ["tp5n"],
             "pagesets_url": "http://talos-bundles.pvt.build.mozilla.org/zips/tp5n.zip",
-            "pagesets_parent_dir_path": "talos/page_load_test/",
-            "pagesets_manifest_path": "talos/page_load_test/tp5n/tp5n.manifest",
             "talos_options": [
                 "--e10s",
                 "--xperf_path",

testing/talos/talos/cmdline.py

@@ -98,12 +98,11 @@ def create_parser(mach_interface=False):
             metavar="PREF=VALUE",
             help="defines an extra user preference")
     add_arg('--webServer', dest='webserver',
-            help="address of the webserver hosting the talos files")
+            help="DEPRECATED")
     if not mach_interface:
         add_arg('--develop', action='store_true', default=False,
                 help="useful for running tests on a developer machine."
-                     " Creates a local webserver and doesn't upload to the"
-                     " graph servers.")
+                     " Doesn't upload to the graph servers.")
     add_arg('--responsiveness', action='store_true',
             help="turn on responsiveness collection")
     add_arg("--cycles", type=int,

testing/talos/talos/config.py

@@ -252,9 +252,15 @@ def fix_xperf(config):
 
 
 @validator
-def check_webserver(config):
-    if config['develop'] and not config['webserver']:
-        config['webserver'] = 'localhost:15707'
+def set_webserver(config):
+    # pick a free port
+    import socket
+    sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+    sock.bind(('', 0))
+    port = sock.getsockname()[1]
+    sock.close()
+
+    config['webserver'] = 'localhost:%d' % port
 
 
 @validator
@@ -338,8 +344,9 @@ def build_manifest(config, manifestName):
     # write modified manifest lines
     with open(manifestName + '.develop', 'w') as newHandle:
         for line in manifestLines:
-            newHandle.write(line.replace('localhost',
-                                         config['webserver']))
+            newline = line.replace('localhost', config['webserver'])
+            newline = newline.replace('page_load_test', 'tests')
+            newHandle.write(newline)
 
     newManifestName = manifestName + '.develop'
 
@@ -368,7 +375,7 @@ def get_test(config, global_overrides, counters, test_instance):
 
     # fix up tpmanifest
     tpmanifest = getattr(test_instance, 'tpmanifest', None)
-    if tpmanifest and config.get('develop'):
+    if tpmanifest:
         test_instance.tpmanifest = \
             build_manifest(config, utils.interpolate(tpmanifest))
 
@@ -405,7 +412,7 @@ def tests(config):
 
 def get_browser_config(config):
     required = ('preferences', 'extensions', 'browser_path', 'browser_wait',
-                'extra_args', 'buildid', 'env', 'init_url')
+                'extra_args', 'buildid', 'env', 'init_url', 'webserver')
     optional = {'bcontroller_config': '${talos}/bcontroller.json',
                 'branch_name': '',
                 'child_process': 'plugin-container',
@@ -417,7 +424,6 @@ def get_browser_config(config):
                 'symbols_path': None,
                 'test_name_extension': '',
                 'test_timeout': 1200,
-                'webserver': '',
                 'xperf_path': None,
                 'error_filename': None,
                 }

testing/talos/talos/filter.py

@@ -94,7 +94,7 @@ def dromaeo(series):
     dromaeo: https://wiki.mozilla.org/Dromaeo, pull the internal calculation
     out
       * This is for 'runs/s' based tests, not 'ms' tests.
-      * chunksize: defined in dromaeo: page_load_test/dromaeo/webrunner.js#l8
+      * chunksize: defined in dromaeo: tests/dromaeo/webrunner.js#l8
     """
     means = []
     chunksize = 5

testing/talos/talos/generate-tart-xpi.html

@@ -11,7 +11,7 @@
   <script src="scripts/xpigen.js"></script>
 
   <script>
-    var base = "page_load_test/tart/addon/";
+    var base = "tests/tart/addon/";
     var files = [
       "chrome.manifest",
       "install.rdf",
@@ -67,7 +67,7 @@
   </p>
   <p>
     You can also build the addon yourself on a local talos repo: zip the content
-    of <span style="font-family: monospace;">talos/page_load_test/tart/addon/</span> and rename the zip to xpi.
+    of <span style="font-family: monospace;">talos/tests/tart/addon/</span> and rename the zip to xpi.
   </p>
 
 </body>

testing/talos/talos/getInfo.html

@@ -6,7 +6,7 @@
   <meta charset="utf-8">
   <head>
     <script language="Javascript" type="text/javascript" src="scripts/MozillaFileLogger.js"></script>
-    <script language="Javascript" type="text/javascript" src="page_load_test/quit.js"></script>
+    <script language="Javascript" type="text/javascript" src="tests/quit.js"></script>
     <title>shutdown script</title>
   </head>
 

testing/talos/talos/page_load_test/canvasmark/canvasmark.manifest

@@ -1,3 +0,0 @@
-% http://localhost/page_load_test/canvasmark/index.html?auto=true
-
-

testing/talos/talos/page_load_test/dromaeo/css.manifest

@@ -1,6 +0,0 @@
-% http://localhost/page_load_test/dromaeo/cssquery-dojo.html
-% http://localhost/page_load_test/dromaeo/cssquery-ext.html
-% http://localhost/page_load_test/dromaeo/cssquery-jquery.html
-% http://localhost/page_load_test/dromaeo/cssquery-mootools.html
-% http://localhost/page_load_test/dromaeo/cssquery-prototype.html
-% http://localhost/page_load_test/dromaeo/cssquery-yui.html

testing/talos/talos/page_load_test/dromaeo/dom.manifest

@@ -1,4 +0,0 @@
-% http://localhost/page_load_test/dromaeo/dom-attr.html
-% http://localhost/page_load_test/dromaeo/dom-modify.html
-% http://localhost/page_load_test/dromaeo/dom-query.html
-% http://localhost/page_load_test/dromaeo/dom-traverse.html

testing/talos/talos/page_load_test/kraken/kraken.manifest

@@ -1,14 +0,0 @@
-% http://localhost/page_load_test/kraken/driver.html?testName=ai-astar
-% http://localhost/page_load_test/kraken/driver.html?testName=audio-beat-detection
-% http://localhost/page_load_test/kraken/driver.html?testName=audio-dft
-% http://localhost/page_load_test/kraken/driver.html?testName=audio-fft
-% http://localhost/page_load_test/kraken/driver.html?testName=audio-oscillator
-% http://localhost/page_load_test/kraken/driver.html?testName=imaging-gaussian-blur
-% http://localhost/page_load_test/kraken/driver.html?testName=imaging-darkroom
-% http://localhost/page_load_test/kraken/driver.html?testName=imaging-desaturate
-% http://localhost/page_load_test/kraken/driver.html?testName=json-parse-financial
-% http://localhost/page_load_test/kraken/driver.html?testName=json-stringify-tinderbox
-% http://localhost/page_load_test/kraken/driver.html?testName=stanford-crypto-aes
-% http://localhost/page_load_test/kraken/driver.html?testName=stanford-crypto-ccm
-% http://localhost/page_load_test/kraken/driver.html?testName=stanford-crypto-pbkdf2
-% http://localhost/page_load_test/kraken/driver.html?testName=stanford-crypto-sha256-iterative

testing/talos/talos/page_load_test/scroll/scroll.manifest

@@ -1,6 +0,0 @@
-% http://localhost/page_load_test/scroll/tiled.html
-% http://localhost/page_load_test/scroll/tiled-fixed.html
-% http://localhost/page_load_test/scroll/tiled-downscale.html
-% http://localhost/page_load_test/scroll/tiled-fixed-downscale.html
-% http://localhost/page_load_test/scroll/iframe.svg
-% http://localhost/page_load_test/scroll/reader.htm

testing/talos/talos/page_load_test/svg_opacity/svg_opacity.manifest

@@ -1,4 +0,0 @@
-# opacity tests
-
-http://localhost/page_load_test/svg_opacity/big-optimizable-group-opacity-2500.svg
-http://localhost/page_load_test/svg_opacity/small-group-opacity-2500.svg

testing/talos/talos/page_load_test/svgx/svgm.manifest

@@ -1,17 +0,0 @@
-# gearflowers image
-http://localhost/page_load_test/svgx/gearflowers.svg
-
-# some generic image compositing tests
-http://localhost/page_load_test/svgx/composite-scale.svg
-http://localhost/page_load_test/svgx/composite-scale-opacity.svg
-http://localhost/page_load_test/svgx/composite-scale-rotate.svg
-http://localhost/page_load_test/svgx/composite-scale-rotate-opacity.svg
-
-# Painting multiple complex paths
-% http://localhost/page_load_test/svgx/hixie-001.xml
-# Painting text
-% http://localhost/page_load_test/svgx/hixie-003.xml
-# Painting linear gradients
-% http://localhost/page_load_test/svgx/hixie-005.xml
-# World Map
-% http://localhost/page_load_test/svgx/hixie-007.xml

testing/talos/talos/page_load_test/svgx/svgx.manifest

@@ -1,23 +0,0 @@
-# gearflowers image
-http://localhost/page_load_test/svgx/gearflowers.svg
-
-# some generic image compositing tests
-http://localhost/page_load_test/svgx/composite-scale.svg
-http://localhost/page_load_test/svgx/composite-scale-opacity.svg
-http://localhost/page_load_test/svgx/composite-scale-rotate.svg
-http://localhost/page_load_test/svgx/composite-scale-rotate-opacity.svg
-
-# Painting multiple complex paths
-% http://localhost/page_load_test/svgx/hixie-001.xml
-# Painting multiple complex paths with transparency
-% http://localhost/page_load_test/svgx/hixie-002.xml
-# Painting text
-% http://localhost/page_load_test/svgx/hixie-003.xml
-# Painting images
-% http://localhost/page_load_test/svgx/hixie-004.xml
-# Painting linear gradients
-% http://localhost/page_load_test/svgx/hixie-005.xml
-# Painting radial gradients
-% http://localhost/page_load_test/svgx/hixie-006.xml
-# World Map
-% http://localhost/page_load_test/svgx/hixie-007.xml

testing/talos/talos/page_load_test/v8_7/v8.manifest

@@ -1 +0,0 @@
-% http://localhost/page_load_test/v8_7/run.html

testing/talos/talos/page_load_test/webgl/glterrain.manifest

@@ -1 +0,0 @@
-% http://localhost/page_load_test/webgl/benchmarks/terrain/perftest.html

testing/talos/talos/pageloader/chrome/Profiler.js

@@ -5,8 +5,8 @@
 // - NOTE: This file is duplicated verbatim at:
 //         - talos/scripts/Profiler.js
 //         - talos/pageloader/chrome/Profiler.js
-//         - talos/page_load_test/devtools/addon/content/Profiler.js
-//         - talos/page_load_test/tart/addon/content/Profiler.js
+//         - talos/tests/devtools/addon/content/Profiler.js
+//         - talos/tests/tart/addon/content/Profiler.js
 //         - talos/startup_test/tresize/addon/content/Profiler.js
 //
 //  - Please keep these copies in sync.

testing/talos/talos/pageloader/chrome/tscroll.js

@@ -1,5 +1,5 @@
 // Note: The content from here upto '// End scroll test' is duplicated at:
-//       - talos/page_load_test/scroll/scroll-test.js
+//       - talos/tests/scroll/scroll-test.js
 //       - inside talos/pageloader/chrome/tscroll.js
 //
 // - Please keep these copies in sync.

testing/talos/talos/run_tests.py

@@ -12,8 +12,8 @@ import sys
 import time
 import traceback
 import urllib
-import urlparse
 import utils
+import mozhttpd
 
 from talos.results import TalosResults
 from talos.ttest import TTest
@@ -68,28 +68,10 @@ def buildCommandLine(test):
 
 def setup_webserver(webserver):
     """use mozhttpd to setup a webserver"""
+    logging.info("starting webserver on %r" % webserver)
 
-    scheme = "http://"
-    if (webserver.startswith('http://') or
-        webserver.startswith('chrome://') or
-        webserver.startswith('file:///')):  # noqa
-
-        scheme = ""
-    elif '://' in webserver:
-        print "Unable to parse user defined webserver: '%s'" % (webserver)
-        sys.exit(2)
-
-    url = urlparse.urlparse('%s%s' % (scheme, webserver))
-    port = url.port
-
-    if port:
-        import mozhttpd
-        return mozhttpd.MozHttpd(host=url.hostname, port=int(port),
-                                 docroot=here)
-    else:
-        print ("WARNING: unable to start web server without custom port"
-               " configured")
-        return None
+    host, port = webserver.split(':')
+    return mozhttpd.MozHttpd(host=host, port=int(port), docroot=here)
 
 
 def run_tests(config, browser_config):
@@ -120,6 +102,8 @@ def run_tests(config, browser_config):
         test['cleanup'] = utils.interpolate(test['cleanup'])
 
     # pass --no-remote to firefox launch, if --develop is specified
+    # we do that to allow locally the user to have another running firefox
+    # instance
     if browser_config['develop']:
         browser_config['extra_args'] = '--no-remote'
 
@@ -195,52 +179,41 @@ def run_tests(config, browser_config):
         )
     talos_results.check_output_formats(results_urls)
 
-    # setup a webserver, if --develop is specified
-    httpd = None
-    if browser_config['develop']:
-        httpd = setup_webserver(browser_config['webserver'])
-        if httpd:
-            httpd.start()
+    httpd = setup_webserver(browser_config['webserver'])
+    httpd.start()
 
-    # run the tests
-    timer = utils.Timer()
-    logging.info("Starting test suite %s", title)
-    for test in tests:
-        testname = test['name']
-        testtimer = utils.Timer()
-        logging.info("Starting test %s", testname)
+    testname = None
+    try:
+        # run the tests
+        timer = utils.Timer()
+        logging.info("Starting test suite %s", title)
+        for test in tests:
+            testname = test['name']
+            testtimer = utils.Timer()
+            logging.info("Starting test %s", testname)
 
-        try:
             mytest = TTest()
-            if mytest:
-                talos_results.add(mytest.runTest(browser_config, test))
-            else:
-                logging.error("Error found while running %s", testname)
-        except TalosRegression:
-            logging.error("Detected a regression for %s", testname)
-            if httpd:
-                httpd.stop()
-            # by returning 1, we report an orange to buildbot
-            # http://docs.buildbot.net/latest/developer/results.html
-            return 1
-        except (TalosCrash, TalosError):
-            # NOTE: if we get into this condition, talos has an internal
-            # problem and cannot continue
-            #       this will prevent future tests from running
-            traceback.print_exception(*sys.exc_info())
-            if httpd:
-                httpd.stop()
-            # indicate a failure to buildbot, turn the job red
-            return 2
+            talos_results.add(mytest.runTest(browser_config, test))
 
-        logging.info("Completed test %s (%s)", testname, testtimer.elapsed())
+            logging.info("Completed test %s (%s)", testname, testtimer.elapsed())
+
+    except TalosRegression:
+        logging.error("Detected a regression for %s", testname)
+        # by returning 1, we report an orange to buildbot
+        # http://docs.buildbot.net/latest/developer/results.html
+        return 1
+    except (TalosCrash, TalosError):
+        # NOTE: if we get into this condition, talos has an internal
+        # problem and cannot continue
+        #       this will prevent future tests from running
+        traceback.print_exception(*sys.exc_info())
+        # indicate a failure to buildbot, turn the job red
+        return 2
+    finally:
+        httpd.stop()
 
     logging.info("Completed test suite (%s)", timer.elapsed())
 
-    # stop the webserver if running
-    if httpd:
-        httpd.stop()
-
     # output results
     if results_urls:
         talos_results.output(results_urls)

testing/talos/talos/scripts/Profiler.js

@@ -5,8 +5,8 @@
 // - NOTE: This file is duplicated verbatim at:
 //         - talos/scripts/Profiler.js
 //         - talos/pageloader/chrome/Profiler.js
-//         - talos/page_load_test/devtools/addon/content/Profiler.js
-//         - talos/page_load_test/tart/addon/content/Profiler.js
+//         - talos/tests/devtools/addon/content/Profiler.js
+//         - talos/tests/tart/addon/content/Profiler.js
 //         - talos/startup_test/tresize/addon/content/Profiler.js
 //
 //  - Please keep these copies in sync.

testing/talos/talos/startup_test/media/html/media_tests.html

@@ -17,7 +17,7 @@
     <div id="log"></div>
     <script language="Javascript" src="/scripts/MozillaFileLogger.js"></script>
     <script language="Javascript" src="/scripts/Profiler.js"></script>
-    <script language="Javascript" src="/page_load_test/quit.js"></script>
+    <script language="Javascript" src="/tests/quit.js"></script>
     <script language="Javascript" src="/startup_test/media/html/media_api.js"></script>
     <script language="Javascript" src="/startup_test/media/html/audio_playback.js"></script>
     <script language="Javascript" src="/startup_test/media/html/pc_audio_quality.js"></script>

testing/talos/talos/startup_test/sessionrestore/index.html

@@ -5,7 +5,7 @@
 <title>Session Restore Regression Test</title>
 
 <script type="text/javascript" src="../../scripts/MozillaFileLogger.js"></script>
-<script type="text/javascript" src="../../page_load_test/quit.js"></script>
+<script type="text/javascript" src="../../tests/quit.js"></script>
 <script type="text/javascript" src="../../scripts/Profiler.js"></script>
 <script type="text/javascript" src="./main.js">
 </script>

testing/talos/talos/startup_test/tpaint.html

@@ -2,7 +2,7 @@
 <meta charset='utf-8'>
 <script language="JavaScript" type="text/javascript" src="../scripts/MozillaFileLogger.js"></script>
 <script language="JavaScript" type="text/javascript" src="../scripts/Profiler.js"></script>
-<script language="JavaScript" type="text/javascript" src="../page_load_test/quit.js"></script>
+<script language="JavaScript" type="text/javascript" src="../tests/quit.js"></script>
 <script>
 var OPENER_DELAY = 1000; // ms delay between tests
 var REPEAT_COUNT = 20;

testing/talos/talos/startup_test/tresize/addon/content/Profiler.js

@@ -5,8 +5,8 @@
 // - NOTE: This file is duplicated verbatim at:
 //         - talos/scripts/Profiler.js
 //         - talos/pageloader/chrome/Profiler.js
-//         - talos/page_load_test/devtools/addon/content/Profiler.js
-//         - talos/page_load_test/tart/addon/content/Profiler.js
+//         - talos/tests/devtools/addon/content/Profiler.js
+//         - talos/tests/tart/addon/content/Profiler.js
 //         - talos/startup_test/tresize/addon/content/Profiler.js
 //
 //  - Please keep these copies in sync.

testing/talos/talos/startup_test/tresize/addon/content/tresize-test.html

@@ -5,7 +5,7 @@
 <head>
 <script language="Javascript" type="text/javascript" src="../../../../scripts/Profiler.js"></script>
 <script language="Javascript" type="text/javascript" src="../../../../scripts/MozillaFileLogger.js"></script>
-<script language="JavaScript" type="text/javascript" src="../../../../page_load_test/quit.js"></script>
+<script language="JavaScript" type="text/javascript" src="../../../../tests/quit.js"></script>
 <script language="javascript" type="text/javascript">
 
 function runTest() {

testing/talos/talos/startup_test/tspaint_test.html

@@ -8,7 +8,7 @@
 <!-- In-line this to avoid compilation. -->
 <script language="Javascript" type="text/javascript" src="../scripts/MozillaFileLogger.js"></script>
 <script language="Javascript" type="text/javascript" src="../scripts/Profiler.js"></script>
-<script language="JavaScript" type="text/javascript" src="../page_load_test/quit.js"></script>
+<script language="JavaScript" type="text/javascript" src="../tests/quit.js"></script>
 <script language="javascript" type="text/javascript">
 
 function painted() {

testing/talos/talos/test.py

@@ -247,14 +247,14 @@ class tps(PageloaderTest):
     """
     Tests the amount of time it takes to switch between tabs
     """
-    extensions = '${talos}/page_load_test/tabswitch'
-    tpmanifest = '${talos}/page_load_test/tabswitch/tps.manifest'
+    extensions = '${talos}/tests/tabswitch'
+    tpmanifest = '${talos}/tests/tabswitch/tps.manifest'
     tppagecycles = 5
     sps_profile_entries = 1000000
     tploadnocache = True
     preferences = {
         'addon.test.tabswitch.urlfile': os.path.join('${talos}',
-                                                     'page_load_test',
+                                                     'tests',
                                                      'tp5o.html'),
         'addon.test.tabswitch.webserver': '${webserver}',
     }
@@ -283,8 +283,8 @@ class tart(PageloaderTest):
       - half: average interval over the 2nd half of the animation.
       - all: average interval over all recorded intervals.
     """
-    tpmanifest = '${talos}/page_load_test/tart/tart.manifest'
-    extensions = '${talos}/page_load_test/tart/addon'
+    tpmanifest = '${talos}/tests/tart/tart.manifest'
+    extensions = '${talos}/tests/tart/addon'
     tpcycles = 1
     tppagecycles = 25
     tploadnocache = True
@@ -319,8 +319,8 @@ class cart(PageloaderTest):
     2-customize-exit  - exiting customize
     3-customize-enter-css - only the CSS animation part of entering customize
     """
-    tpmanifest = '${talos}/page_load_test/tart/cart.manifest'
-    extensions = '${talos}/page_load_test/tart/addon'
+    tpmanifest = '${talos}/tests/tart/cart.manifest'
+    extensions = '${talos}/tests/tart/addon'
     tpcycles = 1
     tppagecycles = 25
     tploadnocache = True
@@ -344,8 +344,8 @@ class damp(PageloaderTest):
     Tests the speed of DevTools toolbox open, close, and page reload
     for each tool, across a very simple and very complicated page.
     """
-    tpmanifest = '${talos}/page_load_test/devtools/damp.manifest'
-    extensions = '${talos}/page_load_test/devtools/addon'
+    tpmanifest = '${talos}/tests/devtools/damp.manifest'
+    extensions = '${talos}/tests/devtools/addon'
     tpcycles = 1
     tppagecycles = 25
     tploadnocache = True
@@ -366,7 +366,7 @@ class glterrain(PageloaderTest):
     antialias as canvas properties.
     Each of these 4 runs is reported as a different test name.
     """
-    tpmanifest = '${talos}/page_load_test/webgl/glterrain.manifest'
+    tpmanifest = '${talos}/tests/webgl/glterrain.manifest'
     tpcycles = 1
     tppagecycles = 25
     tploadnocache = True
@@ -393,7 +393,7 @@ class tp5n(PageloaderTest):
     """
     resolution = 20
     shutdown = True
-    tpmanifest = '${talos}/page_load_test/tp5n/tp5n.manifest'
+    tpmanifest = '${talos}/tests/tp5n/tp5n.manifest'
     tpcycles = 1
     tppagecycles = 1
     cycles = 1
@@ -437,7 +437,7 @@ class tp5o(PageloaderTest):
     tptimeout = 5000
     rss = True
     mainthread = False
-    tpmanifest = '${talos}/page_load_test/tp5n/tp5o.manifest'
+    tpmanifest = '${talos}/tests/tp5n/tp5o.manifest'
     win_counters = ['Main_RSS', 'Private Bytes', '% Processor Time']
     w7_counters = ['Main_RSS', 'Private Bytes', '% Processor Time',
                    'Modified Page List Bytes']
@@ -455,7 +455,7 @@ class tp5o_scroll(PageloaderTest):
     """
     Tests scroll (like tscrollx does, including ASAP) but on the tp5o pageset.
     """
-    tpmanifest = '${talos}/page_load_test/tp5n/tp5o.manifest'
+    tpmanifest = '${talos}/tests/tp5n/tp5o.manifest'
     tpcycles = 1
     tppagecycles = 12
     sps_profile_interval = 2
@@ -479,7 +479,7 @@ class v8_7(PageloaderTest):
     The previous version of this test is V8 version 5 which was run on
     selective branches and operating systems.
     """
-    tpmanifest = '${talos}/page_load_test/v8_7/v8.manifest'
+    tpmanifest = '${talos}/tests/v8_7/v8.manifest'
     sps_profile_interval = 1
     sps_profile_entries = 1000000
     tpcycles = 1
@@ -495,7 +495,7 @@ class kraken(PageloaderTest):
     This is the Kraken javascript benchmark taken verbatim and slightly
     modified to fit into our pageloader extension and talos harness.
     """
-    tpmanifest = '${talos}/page_load_test/kraken/kraken.manifest'
+    tpmanifest = '${talos}/tests/kraken/kraken.manifest'
     tpcycles = 1
     tppagecycles = 1
     sps_profile_interval = 0.1
@@ -510,7 +510,7 @@ class tcanvasmark(PageloaderTest):
     """
     CanvasMark benchmark v0.6
     """
-    tpmanifest = '${talos}/page_load_test/canvasmark/canvasmark.manifest'
+    tpmanifest = '${talos}/tests/canvasmark/canvasmark.manifest'
     win_counters = w7_counters = linux_counters = mac_counters = None
     tpcycles = 5
     tppagecycles = 1
@@ -538,7 +538,7 @@ class dromaeo_css(dromaeo):
     """
     sps_profile_interval = 2
     sps_profile_entries = 10000000
-    tpmanifest = '${talos}/page_load_test/dromaeo/css.manifest'
+    tpmanifest = '${talos}/tests/dromaeo/css.manifest'
 
 
 @register_test()
@@ -552,7 +552,7 @@ class dromaeo_dom(dromaeo):
     """
     sps_profile_interval = 2
     sps_profile_entries = 10000000
-    tpmanifest = '${talos}/page_load_test/dromaeo/dom.manifest'
+    tpmanifest = '${talos}/tests/dromaeo/dom.manifest'
     tpdisable_e10s = True
 
 
@@ -561,7 +561,7 @@ class tsvgm(PageloaderTest):
     """
     An svg-only number that measures SVG rendering performance.
     """
-    tpmanifest = '${talos}/page_load_test/svgx/svgm.manifest'
+    tpmanifest = '${talos}/tests/svgx/svgm.manifest'
     tpcycles = 1
     tppagecycles = 7
     tpmozafterpaint = False
@@ -579,7 +579,7 @@ class tsvgx(PageloaderTest):
     """
     An svg-only number that measures SVG rendering performance.
     """
-    tpmanifest = '${talos}/page_load_test/svgx/svgx.manifest'
+    tpmanifest = '${talos}/tests/svgx/svgx.manifest'
     tpcycles = 1
     tppagecycles = 25
     tpmozafterpaint = False
@@ -597,7 +597,7 @@ class tsvgr_opacity(PageloaderTest):
     """
     An svg-only number that measures SVG rendering performance.
     """
-    tpmanifest = '${talos}/page_load_test/svg_opacity/svg_opacity.manifest'
+    tpmanifest = '${talos}/tests/svg_opacity/svg_opacity.manifest'
     tpcycles = 1
     tppagecycles = 25
     sps_profile_interval = 1
@@ -610,7 +610,7 @@ class tscrollx(PageloaderTest):
     """
     This test does some scrolly thing.
     """
-    tpmanifest = '${talos}/page_load_test/scroll/scroll.manifest'
+    tpmanifest = '${talos}/tests/scroll/scroll.manifest'
     tpcycles = 1
     tppagecycles = 25
     tpmozafterpaint = False
@@ -629,7 +629,7 @@ class a11yr(PageloaderTest):
     This test ensures basic a11y tables and permutations do not cause
     performance regressions.
     """
-    tpmanifest = '${talos}/page_load_test/a11y/a11y.manifest'
+    tpmanifest = '${talos}/tests/a11y/a11y.manifest'
     tpcycles = 1
     tppagecycles = 25
     tpmozafterpaint = True

testing/talos/talos/tests/canvasmark/canvasmark.manifest

@@ -0,0 +1,3 @@
+% http://localhost/tests/canvasmark/index.html?auto=true
+
+

testing/talos/talos/tests/devtools/addon/content/Profiler.js

@@ -5,8 +5,8 @@
 // - NOTE: This file is duplicated verbatim at:
 //         - talos/scripts/Profiler.js
 //         - talos/pageloader/chrome/Profiler.js
-//         - talos/page_load_test/devtools/addon/content/Profiler.js
-//         - talos/page_load_test/tart/addon/content/Profiler.js
+//         - talos/tests/devtools/addon/content/Profiler.js
+//         - talos/tests/tart/addon/content/Profiler.js
 //         - talos/startup_test/tresize/addon/content/Profiler.js
 //
 //  - Please keep these copies in sync.

testing/talos/talos/tests/devtools/addon/content/damp.html

@@ -56,7 +56,7 @@ function updateConfig() {
 
 Utilities:
   <a href="pages/simple.html">simple page</a>&nbsp;&nbsp;&nbsp;
-  <a href="http://localhost/page_load_test/tp5n/bild.de/www.bild.de/index.html">complicated page</a>&nbsp;&nbsp;&nbsp;
+  <a href="http://localhost/tests/tp5n/bild.de/www.bild.de/index.html">complicated page</a>&nbsp;&nbsp;&nbsp;
 <br/><br/>
 <b>Configure DAMP</b> (CTRL-F5 to reset to talos defaults) <button type="button" onclick="deselectAll()">Deselect all tests</button><br/>
 <script>

testing/talos/talos/tests/devtools/addon/content/damp.js

@@ -6,7 +6,7 @@ const { getActiveTab } = devtools.require("sdk/tabs/utils");
 const { getMostRecentBrowserWindow } = devtools.require("sdk/window/utils");
 
 const SIMPLE_URL = "chrome://damp/content/pages/simple.html";
-const COMPLICATED_URL = "http://localhost/page_load_test/tp5n/bild.de/www.bild.de/index.html";
+const COMPLICATED_URL = "http://localhost/tests/tp5n/bild.de/www.bild.de/index.html";
 
 function Damp() {
 }

testing/talos/talos/tests/dromaeo/css.manifest

@@ -0,0 +1,6 @@
+% http://localhost/tests/dromaeo/cssquery-dojo.html
+% http://localhost/tests/dromaeo/cssquery-ext.html
+% http://localhost/tests/dromaeo/cssquery-jquery.html
+% http://localhost/tests/dromaeo/cssquery-mootools.html
+% http://localhost/tests/dromaeo/cssquery-prototype.html
+% http://localhost/tests/dromaeo/cssquery-yui.html

testing/talos/talos/tests/dromaeo/dom.manifest

@@ -0,0 +1,4 @@
+% http://localhost/tests/dromaeo/dom-attr.html
+% http://localhost/tests/dromaeo/dom-modify.html
+% http://localhost/tests/dromaeo/dom-query.html
+% http://localhost/tests/dromaeo/dom-traverse.html