From c5ccd2f839deef5b0403b3cb3d45c7667a5150a5 Mon Sep 17 00:00:00 2001 From: Andrew Halberstadt Date: Fri, 5 Jun 2015 13:28:29 -0400 Subject: [PATCH] Bug 1171971 - Move test_paths argument out of mach and into mochitest; remove --test-path, r=chmanchester --- .../config/mozharness/b2g_emulator_config.py | 4 +- testing/mochitest/bisection.py | 2 - testing/mochitest/browser-harness.xul | 9 +- testing/mochitest/browser-test.js | 2 +- testing/mochitest/chrome-harness.js | 73 +----- testing/mochitest/jetpack-addon-harness.js | 2 +- testing/mochitest/jetpack-package-harness.js | 2 +- testing/mochitest/mach_commands.py | 33 +-- testing/mochitest/mochitest_options.py | 26 +- testing/mochitest/runtests.py | 246 ++++++++---------- testing/mochitest/runtestsb2g.py | 2 + testing/mochitest/runtestsremote.py | 4 +- .../manifestparser/manifestparser/filters.py | 28 ++ .../tests/b2g_emulator_mochitest_media.yml | 2 +- testing/testsuite-targets.mk | 21 +- 15 files changed, 189 insertions(+), 267 deletions(-) diff --git a/testing/config/mozharness/b2g_emulator_config.py b/testing/config/mozharness/b2g_emulator_config.py index d93827c6ef8..40aa6b008db 100644 --- a/testing/config/mozharness/b2g_emulator_config.py +++ b/testing/config/mozharness/b2g_emulator_config.py @@ -72,7 +72,7 @@ config = { "--quiet", "--log-raw=%(raw_log_file)s", "--certificate-path=%(certificate_path)s", - "--test-path=%(test_path)s" + "%(test_path)s" ], "run_filename": "runtestsb2g.py", "testsdir": "mochitest" @@ -93,7 +93,7 @@ config = { "--chrome", "--log-raw=%(raw_log_file)s", "--certificate-path=%(certificate_path)s", - "--test-path=%(test_path)s" + "%(test_path)s" ], "run_filename": "runtestsb2g.py", "testsdir": "mochitest" diff --git a/testing/mochitest/bisection.py b/testing/mochitest/bisection.py index 5ffd2771922..9c6d69d2b51 100644 --- a/testing/mochitest/bisection.py +++ b/testing/mochitest/bisection.py @@ -10,8 +10,6 @@ class Bisect(object): super(Bisect, self).__init__() self.summary = [] self.contents = {} - self.testRoot = harness.testRoot - self.testRootAbs = harness.testRootAbs self.repeat = 10 self.failcount = 0 self.max_failures = 3 diff --git a/testing/mochitest/browser-harness.xul b/testing/mochitest/browser-harness.xul index 388cb5810e7..cbf8d18b6ee 100644 --- a/testing/mochitest/browser-harness.xul +++ b/testing/mochitest/browser-harness.xul @@ -100,14 +100,7 @@ function TestStart() { gConfig = readConfig(); - // If MochiTest was started with the --test-path flag specifying a subset - // of tests to run, put that path in the label of the "Run Tests" button - // so the tester knows which tests will run when they press that button. - if (gConfig.testPath) - document.getElementById("runTestsButton").label = - "Run " + gConfig.testPath + " tests"; - - // Similarly, update the title for --start-at and --end-at. + // Update the title for --start-at and --end-at. if (gConfig.startAt || gConfig.endAt) document.getElementById("runTestsButton").label = "Run subset of tests"; diff --git a/testing/mochitest/browser-test.js b/testing/mochitest/browser-test.js index 48dd88f1db6..edac456e0f3 100644 --- a/testing/mochitest/browser-test.js +++ b/testing/mochitest/browser-test.js @@ -350,7 +350,7 @@ Tester.prototype = { this.dumper.structuredLogger.testEnd("browser-test.js", "FAIL", "PASS", - "No tests to run. Did you pass an invalid --test-path?"); + "No tests to run. Did you pass invalid test_paths?"); } this.dumper.structuredLogger.info("*** End BrowserChrome Test Results ***"); diff --git a/testing/mochitest/chrome-harness.js b/testing/mochitest/chrome-harness.js index 83e8c5c67ec..569fa2ce608 100644 --- a/testing/mochitest/chrome-harness.js +++ b/testing/mochitest/chrome-harness.js @@ -57,62 +57,6 @@ function getChromeDir(resolvedURI) { return chromeDir.parent.QueryInterface(Components.interfaces.nsILocalFile); } -/** - * basePath: the URL base path to search from such as chrome://mochikit/content/a11y - * testPath: the optional testPath passed into the test such as dom/tests/mochitest - * dir: the test dir to append to the uri after getting a directory interface - * srvScope: loaded javascript to server.js so we have aComponents.classesess to the list() function - * - * return value: - * single test: [json object, path to test] - * list of tests: [json object, null] <- directory [heirarchy] - */ -function getFileListing(basePath, testPath, dir, srvScope) -{ - var uri = getResolvedURI(basePath); - var chromeDir = getChromeDir(uri); - chromeDir.appendRelativePath(dir); - basePath += '/' + dir.replace(/\\/g, '/'); - - if (testPath == "false" || testPath == false) { - testPath = ""; - } - testPath = testPath.replace(/\\\\/g, '\\').replace(/\\/g, '/'); - - var ioSvc = Components.classes["@mozilla.org/network/io-service;1"]. - getService(Components.interfaces.nsIIOService); - var testsDirURI = ioSvc.newFileURI(chromeDir); - var testsDir = ioSvc.newURI(testPath, null, testsDirURI) - .QueryInterface(Components.interfaces.nsIFileURL).file; - - if (testPath != undefined) { - var extraPath = testPath; - - var fileNameRegexp = /(browser|test)_.+\.(xul|html|js)$/; - - // Invalid testPath... - if (!testsDir.exists()) - return null; - - if (testsDir.isFile()) { - if (fileNameRegexp.test(testsDir.leafName)) { - var singlePath = basePath + '/' + testPath; - var links = {}; - links[singlePath] = true; - return links; - } - // We were passed a file that's not a test... - return null; - } - - // otherwise, we were passed a directory of tests - basePath += "/" + testPath; - } - var [links, count] = srvScope.list(basePath, testsDir, true); - return links; -} - - //used by tests to determine their directory based off window.location.path function getRootDirectory(path, chromeURI) { if (chromeURI === undefined) @@ -313,19 +257,6 @@ function getTestList(params, callback) { } } params = config; - if (params.manifestFile) { - getTestManifest("http://mochi.test:8888/" + params.manifestFile, params, callback); - return; - } - - var links = {}; - // load server.js in so we can share template functions - var scriptLoader = Cc["@mozilla.org/moz/jssubscript-loader;1"]. - getService(Ci.mozIJSSubScriptLoader); - var srvScope = {}; - scriptLoader.loadSubScript('chrome://mochikit/content/server.js', - srvScope); - - links = getFileListing(baseurl, params.testPath, params.testRoot, srvScope); - callback(links); + getTestManifest("http://mochi.test:8888/" + params.manifestFile, params, callback); + return; } diff --git a/testing/mochitest/jetpack-addon-harness.js b/testing/mochitest/jetpack-addon-harness.js index 6f9db61ddf4..47f5a157617 100644 --- a/testing/mochitest/jetpack-addon-harness.js +++ b/testing/mochitest/jetpack-addon-harness.js @@ -197,7 +197,7 @@ function testInit() { function finish() { if (passed + failed == 0) { dump("TEST-UNEXPECTED-FAIL | jetpack-addon-harness.js | " + - "No tests to run. Did you pass an invalid --test-path?\n"); + "No tests to run. Did you pass invalid test_paths?\n"); } else { dump("Jetpack Addon Test Summary\n"); diff --git a/testing/mochitest/jetpack-package-harness.js b/testing/mochitest/jetpack-package-harness.js index ee93fdeb90e..70867a72313 100644 --- a/testing/mochitest/jetpack-package-harness.js +++ b/testing/mochitest/jetpack-package-harness.js @@ -218,7 +218,7 @@ function testInit() { function finish() { if (passed + failed == 0) { dump("TEST-UNEXPECTED-FAIL | jetpack-package-harness.js | " + - "No tests to run. Did you pass an invalid --test-path?\n"); + "No tests to run. Did you pass invalid test_paths?\n"); } else { dump("Jetpack Package Test Summary\n"); diff --git a/testing/mochitest/mach_commands.py b/testing/mochitest/mach_commands.py index eedb5970042..8735111101f 100644 --- a/testing/mochitest/mach_commands.py +++ b/testing/mochitest/mach_commands.py @@ -236,10 +236,6 @@ class MochitestRunner(MozbuildObject): if test_objects: return test_objects - # Ensure test paths are relative to topobjdir or topsrcdir. - test_paths = test_paths or [] - test_paths = [self._wrap_path_argument(tp).relpath() for tp in test_paths] - from mozbuild.testing import TestResolver resolver = self._spawn(TestResolver) tests = list(resolver.resolve_tests(paths=test_paths, cwd=cwd)) @@ -440,12 +436,7 @@ class MachCommands(MachCommandBase): metavar='{{{}}}'.format(', '.join(CANONICAL_FLAVORS)), choices=SUPPORTED_FLAVORS, help='Only run tests of this flavor.') - @CommandArgument('test_paths', nargs='*', metavar='TEST', default=None, - help='Test to run. Can be a single test file or a directory of tests ' - '(to run recursively). If omitted, the entire suite is run.') - def run_mochitest_general(self, test_paths, flavor=None, test_objects=None, - **kwargs): - + def run_mochitest_general(self, flavor=None, test_objects=None, **kwargs): buildapp = None for app in SUPPORTED_APPS: if is_buildapp_in(app)(self): @@ -469,19 +460,22 @@ class MachCommands(MachCommandBase): driver = self._spawn(BuildDriver) driver.install_tests(remove=False) + test_paths = kwargs['test_paths'] + kwargs['test_paths'] = [] + if test_paths and buildapp == 'b2g': # In B2G there is often a 'gecko' directory, though topsrcdir is actually # elsewhere. This little hack makes test paths like 'gecko/dom' work, even if # GECKO_PATH is set in the .userconfig gecko_path = mozpath.abspath(mozpath.join(kwargs['b2gPath'], 'gecko')) if gecko_path != self.topsrcdir: - old_paths = test_paths[:] - test_paths = [] - for tp in old_paths: + new_paths = [] + for tp in test_paths: if mozpath.abspath(tp).startswith(gecko_path): - test_paths.append(mozpath.relpath(tp, gecko_path)) + new_paths.append(mozpath.relpath(tp, gecko_path)) else: - test_paths.append(tp) + new_paths.append(tp) + test_paths = new_paths mochitest = self._spawn(MochitestRunner) tests = mochitest.resolve_tests(test_paths, test_objects, cwd=self._mach_context.cwd) @@ -576,14 +570,10 @@ class RobocopCommands(MachCommandBase): conditions=[conditions.is_android], description='Run a Robocop test.', parser=setup_argument_parser) - @CommandArgument('test_paths', nargs='*', metavar='TEST', default=None, - help='Test to run. Can be a single Robocop test file (like "testLoad.java") ' - ' or a directory of tests ' - '(to run recursively). If omitted, the entire Robocop suite is run.') @CommandArgument('--serve', default=False, action='store_true', help='Run no tests but start the mochi.test web server and launch ' 'Fennec with a test profile.') - def run_robocop(self, test_paths, serve=False, **kwargs): + def run_robocop(self, serve=False, **kwargs): if serve: kwargs['autorun'] = False @@ -601,6 +591,9 @@ class RobocopCommands(MachCommandBase): driver = self._spawn(BuildDriver) driver.install_tests(remove=False) + test_paths = kwargs['test_paths'] + kwargs['test_paths'] = [] + from mozbuild.testing import TestResolver resolver = self._spawn(TestResolver) tests = list(resolver.resolve_tests(paths=test_paths, cwd=self._mach_context.cwd, diff --git a/testing/mochitest/mochitest_options.py b/testing/mochitest/mochitest_options.py index 0e9487a23d1..28eafc031e4 100644 --- a/testing/mochitest/mochitest_options.py +++ b/testing/mochitest/mochitest_options.py @@ -58,6 +58,13 @@ class MochitestArguments(ArgumentContainer): LEVEL_STRING = ", ".join(LOG_LEVELS) args = [ + [["test_paths"], + {"nargs": "*", + "metavar": "TEST", + "default": [], + "help": "Test to run. Can be a single test file or a directory of tests " + "(to run recursively). If omitted, the entire suite is run.", + }], [["--keep-open"], {"action": "store_false", "dest": "closeWhenDone", @@ -158,13 +165,6 @@ class MochitestArguments(ArgumentContainer): "default": False, "suppress": True, }], - [["--test-path"], - {"dest": "testPath", - "default": "", - "help": "Run the given test or recursively run the given directory of tests.", - # if running from mach, a test_paths arg is exposed instead - "suppress": build_obj is not None, - }], [["--bisect-chunk"], {"dest": "bisectChunk", "default": None, @@ -552,6 +552,9 @@ class MochitestArguments(ArgumentContainer): options.gmp_path = os.pathsep.join( os.path.join(build_obj.bindir, *p) for p in gmp_modules) + if options.ipcplugins: + options.test_paths.append('dom/plugins/test/mochitest') + if options.totalChunks is not None and options.thisChunk is None: parser.error( "thisChunk must be specified when totalChunks is specified") @@ -728,6 +731,15 @@ class MochitestArguments(ArgumentContainer): if mozinfo.isWin: options.ignoreMissingLeaks.append("tab") + # XXX We can't normalize test_paths in the non build_obj case here, + # because testRoot depends on the flavor, which is determined by the + # mach command and therefore not finalized yet. Conversely, test paths + # need to be normalized here for the mach case. + if options.test_paths and build_obj: + # Normalize test paths so they are relative to test root + options.test_paths = [build_obj._wrap_path_argument(p).relpath() + for p in options.test_paths] + return options diff --git a/testing/mochitest/runtests.py b/testing/mochitest/runtests.py index 30556124ca6..bce7d649877 100644 --- a/testing/mochitest/runtests.py +++ b/testing/mochitest/runtests.py @@ -13,7 +13,6 @@ SCRIPT_DIR = os.path.abspath(os.path.realpath(os.path.dirname(__file__))) sys.path.insert(0, SCRIPT_DIR) from argparse import Namespace -from urlparse import urlparse import ctypes import glob import json @@ -51,10 +50,11 @@ from manifestparser.filters import ( chunk_by_dir, chunk_by_runtime, chunk_by_slice, + pathprefix, subsuite, tags, ) -from mochitest_options import MochitestArgumentParser +from mochitest_options import MochitestArgumentParser, build_obj from mozprofile import Profile, Preferences from mozprofile.permissions import ServerLocations from urllib import quote_plus as encodeURIComponent @@ -63,12 +63,6 @@ from mozlog.structured import commandline here = os.path.abspath(os.path.dirname(__file__)) -try: - from mozbuild.base import MozbuildObject - build_obj = MozbuildObject.from_environment(cwd=here) -except ImportError: - build_obj = None - ########################### # Option for NSPR logging # @@ -638,14 +632,14 @@ class MochitestUtilsMixin(object): self.urlOpts.append("runUntilFailure=1") if options.repeat: self.urlOpts.append("repeat=%d" % options.repeat) - if os.path.isfile( + if len(options.test_paths) == 1 and options.repeat > 0 and os.path.isfile( os.path.join( self.oldcwd, os.path.dirname(__file__), self.TEST_PATH, - options.testPath)) and options.repeat > 0: - self.urlOpts.append("testname=%s" % - ("/").join([self.TEST_PATH, options.testPath])) + options.test_paths[0])): + self.urlOpts.append("testname=%s" % "/".join( + [self.TEST_PATH, options.test_paths[0]])) if options.manifestFile: self.urlOpts.append("manifestFile=%s" % options.manifestFile) if options.failureFile: @@ -716,50 +710,43 @@ class MochitestUtilsMixin(object): return (testPattern.match(pathPieces[-1]) and not re.search(r'\^headers\^$', filename)) - def getTestPath(self, options): - if options.ipcplugins: - return "dom/plugins/test/mochitest" - else: - return options.testPath - def setTestRoot(self, options): - if hasattr(self, "testRoot"): - return self.testRoot, self.testRootAbs - else: - if options.browserChrome: - if options.immersiveMode: - self.testRoot = 'metro' - else: - self.testRoot = 'browser' - elif options.jetpackPackage: - self.testRoot = 'jetpack-package' - elif options.jetpackAddon: - self.testRoot = 'jetpack-addon' - elif options.a11y: - self.testRoot = 'a11y' - elif options.webapprtChrome: - self.testRoot = 'webapprtChrome' - elif options.webapprtContent: - self.testRoot = 'webapprtContent' - elif options.chrome: - self.testRoot = 'chrome' + if options.browserChrome: + if options.immersiveMode: + self.testRoot = 'metro' else: - self.testRoot = self.TEST_PATH - self.testRootAbs = os.path.join(SCRIPT_DIR, self.testRoot) + self.testRoot = 'browser' + elif options.jetpackPackage: + self.testRoot = 'jetpack-package' + elif options.jetpackAddon: + self.testRoot = 'jetpack-addon' + elif options.a11y: + self.testRoot = 'a11y' + elif options.webapprtChrome: + self.testRoot = 'webapprtChrome' + elif options.webapprtContent: + self.testRoot = 'webapprtContent' + elif options.chrome: + self.testRoot = 'chrome' + else: + self.testRoot = self.TEST_PATH + self.testRootAbs = os.path.join(SCRIPT_DIR, self.testRoot) def buildTestURL(self, options): testHost = "http://mochi.test:8888" - testPath = self.getTestPath(options) - testURL = "/".join([testHost, self.TEST_PATH, testPath]) - if os.path.isfile( - os.path.join( - self.oldcwd, - os.path.dirname(__file__), - self.TEST_PATH, - testPath)) and options.repeat > 0: - testURL = "/".join([testHost, - self.TEST_PATH, - os.path.dirname(testPath)]) + testURL = "/".join([testHost, self.TEST_PATH]) + + if len(options.test_paths) == 1 : + if options.repeat > 0 and os.path.isfile( + os.path.join( + self.oldcwd, + os.path.dirname(__file__), + self.TEST_PATH, + options.test_paths[0])): + testURL = "/".join([testURL, os.path.dirname(options.test_paths[0])]) + else: + testURL = "/".join([testURL, options.test_paths[0]]) + if options.chrome or options.a11y: testURL = "/".join([testHost, self.CHROME_PATH]) elif options.browserChrome or options.jetpackPackage or options.jetpackAddon: @@ -1698,7 +1685,6 @@ class Mochitest(MochitestUtilsMixin): onLaunch=None, detectShutdownLeaks=False, screenshotOnFail=False, - testPath=None, bisectChunk=None, quiet=False): """ @@ -1775,8 +1761,7 @@ class Mochitest(MochitestUtilsMixin): proc, utilityPath, debuggerInfo, - browserProcessId, - testPath) + browserProcessId) kp_kwargs = {'kill_on_timeout': False, 'cwd': SCRIPT_DIR, 'onTimeout': [timeoutHandler]} @@ -1896,6 +1881,16 @@ class Mochitest(MochitestUtilsMixin): return os.path.join(data_dir, '{}-{}.runtimes.json'.format( base, flavor)) + def normalize_paths(self, paths): + # Normalize test paths so they are relative to test root + norm_paths = [] + for p in paths: + abspath = os.path.abspath(os.path.join(self.oldcwd, p)) + if abspath.startswith(self.testRootAbs): + norm_paths.append(os.path.relpath(abspath, self.testRootAbs)) + else: + norm_paths.append(p) + return norm_paths def getActiveTests(self, options, disabled=True): """ @@ -1904,87 +1899,73 @@ class Mochitest(MochitestUtilsMixin): if self._active_tests: return self._active_tests - self.setTestRoot(options) manifest = self.getTestManifest(options) if manifest: info = mozinfo.info - # Bug 883858 - return all tests including disabled tests - testPath = self.getTestPath(options) - testPath = testPath.replace('\\', '/') - if testPath.endswith('.html') or \ - testPath.endswith('.xhtml') or \ - testPath.endswith('.xul') or \ - testPath.endswith('.js'): - # In the case where we have a single file, we don't want to - # filter based on options such as subsuite. - tests = manifest.active_tests( - exists=False, disabled=disabled, **info) - for test in tests: - if 'disabled' in test: - del test['disabled'] + # Bug 1089034 - imptest failure expectations are encoded as + # test manifests, even though they aren't tests. This gross + # hack causes several problems in automation including + # throwing off the chunking numbers. Remove them manually + # until bug 1089034 is fixed. + def remove_imptest_failure_expectations(tests, values): + return (t for t in tests + if 'imptests/failures' not in t['path']) - else: - # Bug 1089034 - imptest failure expectations are encoded as - # test manifests, even though they aren't tests. This gross - # hack causes several problems in automation including - # throwing off the chunking numbers. Remove them manually - # until bug 1089034 is fixed. - def remove_imptest_failure_expectations(tests, values): - return (t for t in tests - if 'imptests/failures' not in t['path']) + filters = [ + remove_imptest_failure_expectations, + subsuite(options.subsuite), + ] - filters = [ - remove_imptest_failure_expectations, - subsuite(options.subsuite), - ] + if options.test_tags: + filters.append(tags(options.test_tags)) - # Add chunking filters if specified - if options.totalChunks: - if options.chunkByRuntime: - runtime_file = self.resolve_runtime_file(options, info) - if not os.path.exists(runtime_file): - self.log.warning("runtime file %s not found; defaulting to chunk-by-dir" % - runtime_file) - options.chunkByRuntime = None - flavor = self.getTestFlavor(options) - if flavor in ('browser-chrome', 'devtools-chrome'): - # these values match current mozharness configs - options.chunkbyDir = 5 - else: - options.chunkByDir = 4 + if options.test_paths: + options.test_paths = self.normalize_paths(options.test_paths) + filters.append(pathprefix(options.test_paths)) - if options.chunkByDir: - filters.append(chunk_by_dir(options.thisChunk, - options.totalChunks, - options.chunkByDir)) - elif options.chunkByRuntime: - with open(runtime_file, 'r') as f: - runtime_data = json.loads(f.read()) - runtimes = runtime_data['runtimes'] - default = runtime_data['excluded_test_average'] - filters.append( - chunk_by_runtime(options.thisChunk, - options.totalChunks, - runtimes, - default_runtime=default)) - else: - filters.append(chunk_by_slice(options.thisChunk, - options.totalChunks)) + # Add chunking filters if specified + if options.totalChunks: + if options.chunkByRuntime: + runtime_file = self.resolve_runtime_file(options, info) + if not os.path.exists(runtime_file): + self.log.warning("runtime file %s not found; defaulting to chunk-by-dir" % + runtime_file) + options.chunkByRuntime = None + flavor = self.getTestFlavor(options) + if flavor in ('browser-chrome', 'devtools-chrome'): + # these values match current mozharness configs + options.chunkbyDir = 5 + else: + options.chunkByDir = 4 - if options.test_tags: - filters.append(tags(options.test_tags)) + if options.chunkByDir: + filters.append(chunk_by_dir(options.thisChunk, + options.totalChunks, + options.chunkByDir)) + elif options.chunkByRuntime: + with open(runtime_file, 'r') as f: + runtime_data = json.loads(f.read()) + runtimes = runtime_data['runtimes'] + default = runtime_data['excluded_test_average'] + filters.append( + chunk_by_runtime(options.thisChunk, + options.totalChunks, + runtimes, + default_runtime=default)) + else: + filters.append(chunk_by_slice(options.thisChunk, + options.totalChunks)) - tests = manifest.active_tests( - exists=False, disabled=disabled, filters=filters, **info) + tests = manifest.active_tests( + exists=False, disabled=disabled, filters=filters, **info) - if len(tests) == 0: - self.log.error("no tests to run using specified " - "combination of filters: {}".format( - manifest.fmt_filters())) + if len(tests) == 0: + self.log.error("no tests to run using specified " + "combination of filters: {}".format( + manifest.fmt_filters())) paths = [] - for test in tests: if len(tests) == 1 and 'disabled' in test: del test['disabled'] @@ -1993,10 +1974,6 @@ class Mochitest(MochitestUtilsMixin): assert pathAbs.startswith(self.testRootAbs) tp = pathAbs[len(self.testRootAbs):].replace('\\', '/').strip('/') - # Filter out tests if we are using --test-path - if testPath and not tp.startswith(testPath): - continue - if not self.isTest(options, tp): self.log.warning( 'Warning: %s from manifest %s is not a valid test' % @@ -2132,11 +2109,7 @@ class Mochitest(MochitestUtilsMixin): dirs = self.getDirectories(options) result = 1 # default value, if no tests are run. - inputTestPath = self.getTestPath(options) for d in dirs: - if inputTestPath and not inputTestPath.startswith(d): - continue - print "dir: %s" % d tests_in_dir = [t for t in testsToRun if os.path.dirname(t) == d] @@ -2296,7 +2269,6 @@ class Mochitest(MochitestUtilsMixin): onLaunch=onLaunch, detectShutdownLeaks=detectShutdownLeaks, screenshotOnFail=options.screenshotOnFail, - testPath=options.testPath, bisectChunk=options.bisectChunk, quiet=options.quiet ) @@ -2337,17 +2309,12 @@ class Mochitest(MochitestUtilsMixin): proc, utilityPath, debuggerInfo, - browserProcessId, - testPath=None): + browserProcessId): """handle process output timeout""" # TODO: bug 913975 : _processOutput should call self.processOutputLine # one more time one timeout (I think) - if testPath: - error_message = "TEST-UNEXPECTED-TIMEOUT | %s | application timed out after %d seconds with no output on %s" % ( - self.lastTestSeen, int(timeout), testPath) - else: - error_message = "TEST-UNEXPECTED-TIMEOUT | %s | application timed out after %d seconds with no output" % ( - self.lastTestSeen, int(timeout)) + error_message = "TEST-UNEXPECTED-TIMEOUT | %s | application timed out after %d seconds with no output" % ( + self.lastTestSeen, int(timeout)) self.message_logger.dump_buffered() self.message_logger.buffering = False @@ -2563,7 +2530,6 @@ class Mochitest(MochitestUtilsMixin): def makeTestConfig(self, options): "Creates a test configuration file for customizing test execution." options.logFile = options.logFile.replace("\\", "\\\\") - options.testPath = options.testPath.replace("\\", "\\\\") if "MOZ_HIDE_RESULTS_TABLE" in os.environ and os.environ[ "MOZ_HIDE_RESULTS_TABLE"] == "1": diff --git a/testing/mochitest/runtestsb2g.py b/testing/mochitest/runtestsb2g.py index 788dcd79f5f..75d89bbd8a5 100644 --- a/testing/mochitest/runtestsb2g.py +++ b/testing/mochitest/runtestsb2g.py @@ -121,6 +121,8 @@ class B2GMochitest(MochitestUtilsMixin): def run_tests(self, options): """ Prepare, configure, run tests and cleanup """ + self.setTestRoot(options) + manifest = self.build_profile(options) self.logPreamble(self.getActiveTests(options)) diff --git a/testing/mochitest/runtestsremote.py b/testing/mochitest/runtestsremote.py index 562acfb7844..3ad1b014850 100644 --- a/testing/mochitest/runtestsremote.py +++ b/testing/mochitest/runtestsremote.py @@ -547,13 +547,13 @@ def run_test_harness(options): if not options.autorun: # Force a single loop iteration. The iteration will start Fennec and # the httpd server, but not actually run a test. - options.testPath = robocop_tests[0]['name'] + options.test_paths = [robocop_tests[0]['name']] retVal = None # Filtering tests active_tests = [] for test in robocop_tests: - if options.testPath and options.testPath != test['name']: + if options.test_paths and test['name'] not in options.test_paths: continue if 'disabled' in test: diff --git a/testing/mozbase/manifestparser/manifestparser/filters.py b/testing/mozbase/manifestparser/manifestparser/filters.py index b0d64e37bcc..d24aa5c0768 100644 --- a/testing/mozbase/manifestparser/manifestparser/filters.py +++ b/testing/mozbase/manifestparser/manifestparser/filters.py @@ -339,6 +339,34 @@ class tags(InstanceFilter): yield test +class pathprefix(InstanceFilter): + """ + Removes tests that don't start with any of the given test paths. + + :param paths: A list of test paths to filter on + """ + + def __init__(self, paths): + InstanceFilter.__init__(self, paths) + if isinstance(paths, basestring): + paths = [paths] + self.paths = paths + + def __call__(self, tests, values): + for test in tests: + for tp in self.paths: + tp = os.path.normpath(tp) + if not os.path.normpath(test['relpath']).startswith(tp): + continue + + # any test path that points to a single file will be run no + # matter what, even if it's disabled + if 'disabled' in test and os.path.normpath(test['relpath']) == tp: + del test['disabled'] + yield test + break + + # filter container DEFAULT_FILTERS = ( diff --git a/testing/taskcluster/tasks/tests/b2g_emulator_mochitest_media.yml b/testing/taskcluster/tasks/tests/b2g_emulator_mochitest_media.yml index 5437721a8f6..8d5b34c76e3 100644 --- a/testing/taskcluster/tasks/tests/b2g_emulator_mochitest_media.yml +++ b/testing/taskcluster/tasks/tests/b2g_emulator_mochitest_media.yml @@ -18,10 +18,10 @@ task: --config-file ./mozharness_configs/remove_executables.py --download-symbols ondemand --test-suite mochitest - --test-path dom/media/tests/ --installer-url {{build_url}} --test-url {{tests_url}} --xre-url https://queue.taskcluster.net/v1/task/wXAHAaxDQpqxoWF1iljJjg/runs/0/artifacts/public/cache/xulrunner-sdk-40.zip + dom/media/tests artifacts: 'public/build': type: directory diff --git a/testing/testsuite-targets.mk b/testing/testsuite-targets.mk index 98f3871aaa6..249c1984f2b 100644 --- a/testing/testsuite-targets.mk +++ b/testing/testsuite-targets.mk @@ -3,14 +3,13 @@ # file, You can obtain one at http://mozilla.org/MPL/2.0/. -# Shortcut for mochitest* and xpcshell-tests targets, -# replaces 'EXTRA_TEST_ARGS=--test-path=...'. +# Shortcut for mochitest* and xpcshell-tests targets ifdef TEST_PATH -TEST_PATH_ARG := --test-path='$(TEST_PATH)' -IPCPLUGINS_PATH_ARG := --test-path='$(TEST_PATH)' +TEST_PATH_ARG := '$(TEST_PATH)' +IPCPLUGINS_PATH_ARG := '$(TEST_PATH)' else TEST_PATH_ARG := -IPCPLUGINS_PATH_ARG := --test-path=dom/plugins/test +IPCPLUGINS_PATH_ARG := dom/plugins/test endif # include automation-build.mk to get the path to the binary @@ -33,7 +32,7 @@ RUN_MOCHITEST_B2G_DESKTOP = \ --log-tbpl=./$@.log \ --desktop --profile ${GAIA_PROFILE_DIR} \ --failure-file=$(abspath _tests/testing/mochitest/makefailures.json) \ - $(TEST_PATH_ARG) $(EXTRA_TEST_ARGS) + $(EXTRA_TEST_ARGS) $(TEST_PATH_ARG) RUN_MOCHITEST = \ rm -f ./$@.log && \ @@ -41,7 +40,7 @@ RUN_MOCHITEST = \ --log-tbpl=./$@.log \ --failure-file=$(abspath _tests/testing/mochitest/makefailures.json) \ --testing-modules-dir=$(abspath _tests/modules) \ - $(SYMBOLS_PATH) $(TEST_PATH_ARG) $(EXTRA_TEST_ARGS) + $(SYMBOLS_PATH) $(EXTRA_TEST_ARGS) $(TEST_PATH_ARG) RERUN_MOCHITEST = \ rm -f ./$@.log && \ @@ -49,7 +48,7 @@ RERUN_MOCHITEST = \ --log-tbpl=./$@.log \ --run-only-tests=makefailures.json \ --testing-modules-dir=$(abspath _tests/modules) \ - $(SYMBOLS_PATH) $(TEST_PATH_ARG) $(EXTRA_TEST_ARGS) + $(SYMBOLS_PATH) $(EXTRA_TEST_ARGS) $(TEST_PATH_ARG) RUN_MOCHITEST_REMOTE = \ rm -f ./$@.log && \ @@ -57,7 +56,7 @@ RUN_MOCHITEST_REMOTE = \ --log-tbpl=./$@.log $(DM_FLAGS) --dm_trans=$(DM_TRANS) \ --app=$(TEST_PACKAGE_NAME) --deviceIP=${TEST_DEVICE} --xre-path=${MOZ_HOST_BIN} \ --testing-modules-dir=$(abspath _tests/modules) \ - $(SYMBOLS_PATH) $(TEST_PATH_ARG) $(EXTRA_TEST_ARGS) + $(SYMBOLS_PATH) $(EXTRA_TEST_ARGS) $(TEST_PATH_ARG) RUN_MOCHITEST_ROBOCOP = \ rm -f ./$@.log && \ @@ -67,7 +66,7 @@ RUN_MOCHITEST_ROBOCOP = \ --robocop-ini=_tests/testing/mochitest/robocop.ini \ --log-tbpl=./$@.log $(DM_FLAGS) --dm_trans=$(DM_TRANS) \ --app=$(TEST_PACKAGE_NAME) --deviceIP=${TEST_DEVICE} --xre-path=${MOZ_HOST_BIN} \ - $(SYMBOLS_PATH) $(TEST_PATH_ARG) $(EXTRA_TEST_ARGS) + $(SYMBOLS_PATH) $(EXTRA_TEST_ARGS) $(TEST_PATH_ARG) ifndef NO_FAIL_ON_TEST_ERRORS define check_test_error_internal @@ -162,7 +161,7 @@ ifeq (powerpc,$(TARGET_CPU)) $(RUN_MOCHITEST) --setpref=dom.ipc.plugins.enabled.ppc.test.plugin=false $(IPCPLUGINS_PATH_ARG) endif else - $(RUN_MOCHITEST) --setpref=dom.ipc.plugins.enabled=false --test-path=dom/plugins/test + $(RUN_MOCHITEST) --setpref=dom.ipc.plugins.enabled=false dom/plugins/test endif $(CHECK_TEST_ERROR)