# This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. from __future__ import unicode_literals import logging import mozpack.path import os import platform import sys import warnings import which from mozbuild.base import ( MachCommandBase, MachCommandConditions as conditions, MozbuildObject, ) from mach.decorators import ( CommandArgument, CommandProvider, Command, ) from mach.logging import StructuredHumanFormatter ADB_NOT_FOUND = ''' The %s command requires the adb binary to be on your path. If you have a B2G build, this can be found in '%s/out/host//bin'. '''.lstrip() GAIA_PROFILE_NOT_FOUND = ''' The %s command requires a non-debug gaia profile. Either pass in --profile, or set the GAIA_PROFILE environment variable. If you do not have a non-debug gaia profile, you can build one: $ git clone https://github.com/mozilla-b2g/gaia $ cd gaia $ make The profile should be generated in a directory called 'profile'. '''.lstrip() GAIA_PROFILE_IS_DEBUG = ''' The %s command requires a non-debug gaia profile. The specified profile, %s, is a debug profile. If you do not have a non-debug gaia profile, you can build one: $ git clone https://github.com/mozilla-b2g/gaia $ cd gaia $ make The profile should be generated in a directory called 'profile'. '''.lstrip() MARIONETTE_DISABLED = ''' The %s command requires a marionette enabled build. Add 'ENABLE_MARIONETTE=1' to your mozconfig file and re-build the application. Your currently active mozconfig is %s. '''.lstrip() class UnexpectedFilter(logging.Filter): def filter(self, record): msg = getattr(record, 'params', {}).get('msg', '') return 'TEST-UNEXPECTED-' in msg class MochitestRunner(MozbuildObject): """Easily run mochitests. This currently contains just the basics for running mochitests. We may want to hook up result parsing, etc. """ def get_webapp_runtime_path(self): import mozinfo appname = 'webapprt-stub' + mozinfo.info.get('bin_suffix', '') if sys.platform.startswith('darwin'): appname = os.path.join(self.distdir, self.substs['MOZ_MACBUNDLE_NAME'], 'Contents', 'MacOS', appname) else: appname = os.path.join(self.distdir, 'bin', appname) return appname def __init__(self, *args, **kwargs): MozbuildObject.__init__(self, *args, **kwargs) # TODO Bug 794506 remove once mach integrates with virtualenv. build_path = os.path.join(self.topobjdir, 'build') if build_path not in sys.path: sys.path.append(build_path) self.tests_dir = os.path.join(self.topobjdir, '_tests') self.mochitest_dir = os.path.join(self.tests_dir, 'testing', 'mochitest') def run_b2g_test(self, test_file=None, b2g_home=None, xre_path=None, **kwargs): """Runs a b2g mochitest. test_file is a path to a test file. It can be a relative path from the top source directory, an absolute filename, or a directory containing test files. """ # Need to call relpath before os.chdir() below. test_path = '' if test_file: test_path = self._wrap_path_argument(test_file).relpath() # TODO without os.chdir, chained imports fail below os.chdir(self.mochitest_dir) # The imp module can spew warnings if the modules below have # already been imported, ignore them. with warnings.catch_warnings(): warnings.simplefilter('ignore') import imp path = os.path.join(self.mochitest_dir, 'runtestsb2g.py') with open(path, 'r') as fh: imp.load_module('mochitest', fh, path, ('.py', 'r', imp.PY_SOURCE)) import mochitest from mochitest_options import B2GOptions parser = B2GOptions() options = parser.parse_args([])[0] if test_path: test_root_file = mozpack.path.join(self.mochitest_dir, 'tests', test_path) if not os.path.exists(test_root_file): print('Specified test path does not exist: %s' % test_root_file) return 1 options.testPath = test_path elif conditions.is_b2g_desktop: options.testManifest = 'b2g-desktop.json' else: options.testManifest = 'b2g.json' for k, v in kwargs.iteritems(): setattr(options, k, v) options.consoleLevel = 'INFO' if conditions.is_b2g_desktop(self): if self.substs.get('ENABLE_MARIONETTE') != '1': print(MARIONETTE_DISABLED % ('mochitest-b2g-desktop', self.mozconfig['path'])) return 1 options.profile = options.profile or os.environ.get('GAIA_PROFILE') if not options.profile: print(GAIA_PROFILE_NOT_FOUND % 'mochitest-b2g-desktop') return 1 if os.path.isfile(os.path.join(options.profile, 'extensions', \ 'httpd@gaiamobile.org')): print(GAIA_PROFILE_IS_DEBUG % ('mochitest-b2g-desktop', options.profile)) return 1 options.desktop = True options.app = self.get_binary_path() if not options.app.endswith('-bin'): options.app = '%s-bin' % options.app if not os.path.isfile(options.app): options.app = options.app[:-len('-bin')] return mochitest.run_desktop_mochitests(parser, options) try: which.which('adb') except which.WhichError: # TODO Find adb automatically if it isn't on the path print(ADB_NOT_FOUND % ('mochitest-remote', b2g_home)) return 1 options.b2gPath = b2g_home options.logcat_dir = self.mochitest_dir options.httpdPath = self.mochitest_dir options.xrePath = xre_path return mochitest.run_remote_mochitests(parser, options) def run_desktop_test(self, suite=None, test_file=None, debugger=None, debugger_args=None, shuffle=False, keep_open=False, rerun_failures=False, no_autorun=False, repeat=0, run_until_failure=False, slow=False, chunk_by_dir=0, total_chunks=None, this_chunk=None, jsdebugger=False, debug_on_failure=False, start_at=None, end_at=None, e10s=False): """Runs a mochitest. test_file is a path to a test file. It can be a relative path from the top source directory, an absolute filename, or a directory containing test files. suite is the type of mochitest to run. It can be one of ('plain', 'chrome', 'browser', 'metro', 'a11y'). debugger is a program name or path to a binary (presumably a debugger) to run the test in. e.g. 'gdb' debugger_args are the arguments passed to the debugger. shuffle is whether test order should be shuffled (defaults to false). keep_open denotes whether to keep the browser open after tests complete. """ if rerun_failures and test_file: print('Cannot specify both --rerun-failures and a test path.') return 1 # Need to call relpath before os.chdir() below. test_path = '' if test_file: test_path = self._wrap_path_argument(test_file).relpath() failure_file_path = os.path.join(self.statedir, 'mochitest_failures.json') if rerun_failures and not os.path.exists(failure_file_path): print('No failure file present. Did you run mochitests before?') return 1 from StringIO import StringIO # runtests.py is ambiguous, so we load the file/module manually. if 'mochitest' not in sys.modules: import imp path = os.path.join(self.mochitest_dir, 'runtests.py') with open(path, 'r') as fh: imp.load_module('mochitest', fh, path, ('.py', 'r', imp.PY_SOURCE)) import mozinfo import mochitest # This is required to make other components happy. Sad, isn't it? os.chdir(self.topobjdir) # Automation installs its own stream handler to stdout. Since we want # all logging to go through us, we just remove their handler. remove_handlers = [l for l in logging.getLogger().handlers if isinstance(l, logging.StreamHandler)] for handler in remove_handlers: logging.getLogger().removeHandler(handler) runner = mochitest.Mochitest() opts = mochitest.MochitestOptions() options, args = opts.parse_args([]) # Need to set the suite options before verifyOptions below. if suite == 'plain': # Don't need additional options for plain. pass elif suite == 'chrome': options.chrome = True elif suite == 'browser': options.browserChrome = True elif suite == 'metro': options.immersiveMode = True options.browserChrome = True elif suite == 'a11y': options.a11y = True elif suite == 'webapprt-content': options.webapprtContent = True options.app = self.get_webapp_runtime_path() elif suite == 'webapprt-chrome': options.webapprtChrome = True options.app = self.get_webapp_runtime_path() options.browserArgs.append("-test-mode") else: raise Exception('None or unrecognized mochitest suite type.') options.autorun = not no_autorun options.closeWhenDone = not keep_open options.shuffle = shuffle options.consoleLevel = 'INFO' options.repeat = repeat options.runUntilFailure = run_until_failure options.runSlower = slow options.testingModulesDir = os.path.join(self.tests_dir, 'modules') options.extraProfileFiles.append(os.path.join(self.distdir, 'plugins')) options.symbolsPath = os.path.join(self.distdir, 'crashreporter-symbols') options.chunkByDir = chunk_by_dir options.totalChunks = total_chunks options.thisChunk = this_chunk options.jsdebugger = jsdebugger options.debugOnFailure = debug_on_failure options.startAt = start_at options.endAt = end_at options.e10s = e10s mozinfo.update({"e10s": e10s}) # for test manifest parsing. options.failureFile = failure_file_path if test_path: test_root = runner.getTestRoot(options) test_root_file = mozpack.path.join(self.mochitest_dir, test_root, test_path) if not os.path.exists(test_root_file): print('Specified test path does not exist: %s' % test_root_file) print('You may need to run |mach build| to build the test files.') return 1 options.testPath = test_path if rerun_failures: options.testManifest = failure_file_path if debugger: options.debugger = debugger if debugger_args: if options.debugger == None: print("--debugger-args passed, but no debugger specified.") return 1 options.debuggerArgs = debugger_args options = opts.verifyOptions(options, runner) if options is None: raise Exception('mochitest option validator failed.') # We need this to enable colorization of output. self.log_manager.enable_unstructured() # Output processing is a little funky here. The old make targets # grepped the log output from TEST-UNEXPECTED-* and printed these lines # after test execution. Ideally the test runner would expose a Python # API for obtaining test results and we could just format failures # appropriately. Unfortunately, it doesn't yet do that. So, we capture # all output to a buffer then "grep" the buffer after test execution. # Bug 858197 tracks a Python API that would facilitate this. test_output = StringIO() handler = logging.StreamHandler(test_output) handler.addFilter(UnexpectedFilter()) handler.setFormatter(StructuredHumanFormatter(0, write_times=False)) logging.getLogger().addHandler(handler) result = runner.runTests(options) # Need to remove our buffering handler before we echo failures or else # it will catch them again! logging.getLogger().removeHandler(handler) self.log_manager.disable_unstructured() if test_output.getvalue(): result = 1 for line in test_output.getvalue().splitlines(): self.log(logging.INFO, 'unexpected', {'msg': line}, '{msg}') return result def MochitestCommand(func): """Decorator that adds shared command arguments to mochitest commands.""" # This employs light Python magic. Keep in mind a decorator is just a # function that takes a function, does something with it, then returns a # (modified) function. Here, we chain decorators onto the passed in # function. debugger = CommandArgument('--debugger', '-d', metavar='DEBUGGER', help='Debugger binary to run test in. Program name or path.') func = debugger(func) debugger_args = CommandArgument('--debugger-args', metavar='DEBUGGER_ARGS', help='Arguments to pass to the debugger.') func = debugger_args(func) shuffle = CommandArgument('--shuffle', action='store_true', help='Shuffle execution order.') func = shuffle(func) keep_open = CommandArgument('--keep-open', action='store_true', help='Keep the browser open after tests complete.') func = keep_open(func) rerun = CommandArgument('--rerun-failures', action='store_true', help='Run only the tests that failed during the last test run.') func = rerun(func) autorun = CommandArgument('--no-autorun', action='store_true', help='Do not starting running tests automatically.') func = autorun(func) repeat = CommandArgument('--repeat', type=int, default=0, help='Repeat the test the given number of times.') func = repeat(func) runUntilFailure = CommandArgument("--run-until-failure", action='store_true', help='Run tests repeatedly and stops on the first time a test fails. ' \ 'Default cap is 30 runs, which can be overwritten ' \ 'with the --repeat parameter.') func = runUntilFailure(func) slow = CommandArgument('--slow', action='store_true', help='Delay execution between tests.') func = slow(func) end_at = CommandArgument('--end-at', type=str, help='Stop running the test sequence at this test.') func = end_at(func) start_at = CommandArgument('--start-at', type=str, help='Start running the test sequence at this test.') func = start_at(func) chunk_dir = CommandArgument('--chunk-by-dir', type=int, help='Group tests together in chunks by this many top directories.') func = chunk_dir(func) chunk_total = CommandArgument('--total-chunks', type=int, help='Total number of chunks to split tests into.') func = chunk_total(func) this_chunk = CommandArgument('--this-chunk', type=int, help='If running tests by chunks, the number of the chunk to run.') func = this_chunk(func) debug_on_failure = CommandArgument('--debug-on-failure', action='store_true', help='Breaks execution and enters the JS debugger on a test failure. ' \ 'Should be used together with --jsdebugger.') func = debug_on_failure(func) jsdebugger = CommandArgument('--jsdebugger', action='store_true', help='Start the browser JS debugger before running the test. Implies --no-autorun.') func = jsdebugger(func) this_chunk = CommandArgument('--e10s', action='store_true', help='Run tests with electrolysis preferences and test filtering enabled.') func = this_chunk(func) path = CommandArgument('test_file', default=None, nargs='?', metavar='TEST', help='Test to run. Can be specified as a single file, a ' \ 'directory, or omitted. If omitted, the entire test suite is ' \ 'executed.') func = path(func) return func def B2GCommand(func): """Decorator that adds shared command arguments to b2g mochitest commands.""" busybox = CommandArgument('--busybox', default=None, help='Path to busybox binary to install on device') func = busybox(func) logcatdir = CommandArgument('--logcat-dir', default=None, help='directory to store logcat dump files') func = logcatdir(func) profile = CommandArgument('--profile', default=None, help='for desktop testing, the path to the \ gaia profile to use') func = profile(func) geckopath = CommandArgument('--gecko-path', default=None, help='the path to a gecko distribution that should \ be installed on the emulator prior to test') func = geckopath(func) nowindow = CommandArgument('--no-window', action='store_true', default=False, help='Pass --no-window to the emulator') func = nowindow(func) sdcard = CommandArgument('--sdcard', default="10MB", help='Define size of sdcard: 1MB, 50MB...etc') func = sdcard(func) emulator = CommandArgument('--emulator', default='arm', help='Architecture of emulator to use: x86 or arm') func = emulator(func) marionette = CommandArgument('--marionette', default=None, help='host:port to use when connecting to Marionette') func = marionette(func) chunk_total = CommandArgument('--total-chunks', type=int, help='Total number of chunks to split tests into.') func = chunk_total(func) this_chunk = CommandArgument('--this-chunk', type=int, help='If running tests by chunks, the number of the chunk to run.') func = this_chunk(func) path = CommandArgument('test_file', default=None, nargs='?', metavar='TEST', help='Test to run. Can be specified as a single file, a ' \ 'directory, or omitted. If omitted, the entire test suite is ' \ 'executed.') func = path(func) return func @CommandProvider class MachCommands(MachCommandBase): @Command('mochitest-plain', category='testing', conditions=[conditions.is_firefox], description='Run a plain mochitest.') @MochitestCommand def run_mochitest_plain(self, test_file, **kwargs): return self.run_mochitest(test_file, 'plain', **kwargs) @Command('mochitest-chrome', category='testing', conditions=[conditions.is_firefox], description='Run a chrome mochitest.') @MochitestCommand def run_mochitest_chrome(self, test_file, **kwargs): return self.run_mochitest(test_file, 'chrome', **kwargs) @Command('mochitest-browser', category='testing', conditions=[conditions.is_firefox], description='Run a mochitest with browser chrome.') @MochitestCommand def run_mochitest_browser(self, test_file, **kwargs): return self.run_mochitest(test_file, 'browser', **kwargs) @Command('mochitest-metro', category='testing', conditions=[conditions.is_firefox], description='Run a mochitest with metro browser chrome.') @MochitestCommand def run_mochitest_metro(self, test_file, **kwargs): return self.run_mochitest(test_file, 'metro', **kwargs) @Command('mochitest-a11y', category='testing', conditions=[conditions.is_firefox], description='Run an a11y mochitest.') @MochitestCommand def run_mochitest_a11y(self, test_file, **kwargs): return self.run_mochitest(test_file, 'a11y', **kwargs) @Command('webapprt-test-chrome', category='testing', conditions=[conditions.is_firefox], description='Run a webapprt chrome mochitest.') @MochitestCommand def run_mochitest_webapprt_chrome(self, test_file, **kwargs): return self.run_mochitest(test_file, 'webapprt-chrome', **kwargs) @Command('webapprt-test-content', category='testing', conditions=[conditions.is_firefox], description='Run a webapprt content mochitest.') @MochitestCommand def run_mochitest_webapprt_content(self, test_file, **kwargs): return self.run_mochitest(test_file, 'webapprt-content', **kwargs) def run_mochitest(self, test_file, flavor, **kwargs): from mozbuild.controller.building import BuildDriver self._ensure_state_subdir_exists('.') driver = self._spawn(BuildDriver) driver.install_tests(remove=False) mochitest = self._spawn(MochitestRunner) return mochitest.run_desktop_test(test_file=test_file, suite=flavor, **kwargs) # TODO For now b2g commands will only work with the emulator, # they should be modified to work with all devices. def is_emulator(cls): """Emulator needs to be configured.""" return cls.device_name in ('emulator', 'emulator-jb') @CommandProvider class B2GCommands(MachCommandBase): """So far these are only mochitest plain. They are implemented separately because their command lines are completely different. """ def __init__(self, context): MachCommandBase.__init__(self, context) for attr in ('b2g_home', 'xre_path', 'device_name'): setattr(self, attr, getattr(context, attr, None)) @Command('mochitest-remote', category='testing', description='Run a remote mochitest.', conditions=[conditions.is_b2g, is_emulator]) @B2GCommand def run_mochitest_remote(self, test_file, **kwargs): from mozbuild.controller.building import BuildDriver self._ensure_state_subdir_exists('.') driver = self._spawn(BuildDriver) driver.install_tests(remove=False) mochitest = self._spawn(MochitestRunner) return mochitest.run_b2g_test(b2g_home=self.b2g_home, xre_path=self.xre_path, test_file=test_file, **kwargs) @Command('mochitest-b2g-desktop', category='testing', conditions=[conditions.is_b2g_desktop], description='Run a b2g desktop mochitest.') @B2GCommand def run_mochitest_b2g_desktop(self, test_file, **kwargs): from mozbuild.controller.building import BuildDriver self._ensure_state_subdir_exists('.') driver = self._spawn(BuildDriver) driver.install_tests(remove=False) mochitest = self._spawn(MochitestRunner) return mochitest.run_b2g_test(test_file=test_file, **kwargs)