# # ***** BEGIN LICENSE BLOCK ***** # Version: MPL 1.1/GPL 2.0/LGPL 2.1 # # The contents of this file are subject to the Mozilla Public License Version # 1.1 (the "License"); you may not use this file except in compliance with # the License. You may obtain a copy of the License at # http://www.mozilla.org/MPL/ # # Software distributed under the License is distributed on an "AS IS" basis, # WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License # for the specific language governing rights and limitations under the # License. # # The Original Code is mozilla.org code. # # The Initial Developer of the Original Code is # Mozilla Foundation. # Portions created by the Initial Developer are Copyright (C) 1998 # the Initial Developer. All Rights Reserved. # # Contributor(s): # Robert Sayre # Jeff Walden # # Alternatively, the contents of this file may be used under the terms of # either the GNU General Public License Version 2 or later (the "GPL"), or # the GNU Lesser General Public License Version 2.1 or later (the "LGPL"), # in which case the provisions of the GPL or the LGPL are applicable instead # of those above. If you wish to allow use of your version of this file only # under the terms of either the GPL or the LGPL, and not to allow others to # use your version of this file under the terms of the MPL, indicate your # decision by deleting the provisions above and replace them with the notice # and other provisions required by the GPL or the LGPL. If you do not delete # the provisions above, a recipient may use your version of this file under # the terms of any one of the MPL, the GPL or the LGPL. # # ***** END LICENSE BLOCK ***** """ Runs the Mochitest test harness. """ from datetime import datetime import optparse import os import os.path import sys import time import shutil from urllib import quote_plus as encodeURIComponent import urllib2 import commands import automation from automationutils import addCommonOptions # Path to the test script on the server TEST_SERVER_HOST = "localhost:8888" TEST_PATH = "/tests/" CHROME_PATH = "/redirect.html"; A11Y_PATH = "/redirect-a11y.html" TESTS_URL = "http://" + TEST_SERVER_HOST + TEST_PATH CHROMETESTS_URL = "http://" + TEST_SERVER_HOST + CHROME_PATH A11YTESTS_URL = "http://" + TEST_SERVER_HOST + A11Y_PATH SERVER_SHUTDOWN_URL = "http://" + TEST_SERVER_HOST + "/server/shutdown" # main browser chrome URL, same as browser.chromeURL pref #ifdef MOZ_SUITE BROWSER_CHROME_URL = "chrome://navigator/content/navigator.xul" #else BROWSER_CHROME_URL = "chrome://browser/content/browser.xul" #endif # Max time in seconds to wait for server startup before tests will fail -- if # this seems big, it's mostly for debug machines where cold startup # (particularly after a build) takes forever. SERVER_STARTUP_TIMEOUT = 45 oldcwd = os.getcwd() SCRIPT_DIRECTORY = os.path.abspath(os.path.realpath(os.path.dirname(sys.argv[0]))) os.chdir(SCRIPT_DIRECTORY) PROFILE_DIRECTORY = os.path.abspath("./mochitesttestingprofile") LEAK_REPORT_FILE = PROFILE_DIRECTORY + "/" + "leaks-report.log" # Map of debugging programs to information about them, like default arguments # and whether or not they are interactive. DEBUGGER_INFO = { # gdb requires that you supply the '--args' flag in order to pass arguments # after the executable name to the executable. "gdb": { "interactive": True, "args": "-q --args" }, # valgrind doesn't explain much about leaks unless you set the # '--leak-check=full' flag. "valgrind": { "interactive": False, "args": "--leak-check=full" } } ####################### # COMMANDLINE OPTIONS # ####################### class MochitestOptions(optparse.OptionParser): """Parses Mochitest commandline options.""" def __init__(self, **kwargs): optparse.OptionParser.__init__(self, **kwargs) defaults = {} # we want to pass down everything from automation.__all__ addCommonOptions(self, defaults=dict(zip(automation.__all__, [getattr(automation, x) for x in automation.__all__]))) self.add_option("--close-when-done", action = "store_true", dest = "closeWhenDone", help = "close the application when tests are done running") defaults["closeWhenDone"] = False self.add_option("--appname", action = "store", type = "string", dest = "app", help = "absolute path to application, overriding default") defaults["app"] = os.path.join(SCRIPT_DIRECTORY, automation.DEFAULT_APP) self.add_option("--utility-path", action = "store", type = "string", dest = "utilityPath", help = "absolute path to directory containing utility programs (xpcshell, ssltunnel, certutil)") defaults["utilityPath"] = automation.DIST_BIN self.add_option("--certificate-path", action = "store", type = "string", dest = "certPath", help = "absolute path to directory containing certificate store to use testing profile") defaults["certPath"] = automation.CERTS_SRC_DIR self.add_option("--log-file", action = "store", type = "string", dest = "logFile", metavar = "FILE", help = "file to which logging occurs") defaults["logFile"] = "" self.add_option("--autorun", action = "store_true", dest = "autorun", help = "start running tests when the application starts") defaults["autorun"] = False LOG_LEVELS = ("DEBUG", "INFO", "WARNING", "ERROR", "FATAL") LEVEL_STRING = ", ".join(LOG_LEVELS) self.add_option("--console-level", action = "store", type = "choice", dest = "consoleLevel", choices = LOG_LEVELS, metavar = "LEVEL", help = "one of %s to determine the level of console " "logging" % LEVEL_STRING) defaults["consoleLevel"] = None self.add_option("--file-level", action = "store", type = "choice", dest = "fileLevel", choices = LOG_LEVELS, metavar = "LEVEL", help = "one of %s to determine the level of file " "logging if a file has been specified, defaulting " "to INFO" % LEVEL_STRING) defaults["fileLevel"] = "INFO" self.add_option("--chrome", action = "store_true", dest = "chrome", help = "run chrome Mochitests") defaults["chrome"] = False self.add_option("--test-path", action = "store", type = "string", dest = "testPath", help = "start in the given directory's tests") defaults["testPath"] = "" self.add_option("--browser-chrome", action = "store_true", dest = "browserChrome", help = "run browser chrome Mochitests") defaults["browserChrome"] = False self.add_option("--a11y", action = "store_true", dest = "a11y", help = "run accessibility Mochitests"); self.add_option("--setenv", action = "append", type = "string", dest = "environment", metavar = "NAME=VALUE", help = "sets the given variable in the application's " "environment") defaults["environment"] = [] self.add_option("--browser-arg", action = "append", type = "string", dest = "browserArgs", metavar = "ARG", help = "provides an argument to the test application") defaults["browserArgs"] = [] self.add_option("--leak-threshold", action = "store", type = "int", dest = "leakThreshold", metavar = "THRESHOLD", help = "fail if the number of bytes leaked through " "refcounted objects (or bytes in classes with " "MOZ_COUNT_CTOR and MOZ_COUNT_DTOR) is greater " "than the given number") defaults["leakThreshold"] = 0 self.add_option("--fatal-assertions", action = "store_true", dest = "fatalAssertions", help = "abort testing whenever an assertion is hit " "(requires a debug build to be effective)") defaults["fatalAssertions"] = False self.add_option("--extra-profile-file", action = "append", dest = "extraProfileFiles", help = "copy specified files/dirs to testing profile") defaults["extraProfileFiles"] = [] self.add_option("--debugger", action = "store", dest = "debugger", help = "use the given debugger to launch the application") self.add_option("--debugger-args", action = "store", dest = "debuggerArgs", help = "pass the given args to the debugger _before_ " "the application on the command line") self.add_option("--debugger-interactive", action = "store_true", dest = "debuggerInteractive", help = "prevents the test harness from redirecting stdout " "and stderr for interactive debuggers") # -h, --help are automatically handled by OptionParser self.set_defaults(**defaults) usage = """\ Usage instructions for runtests.py. All arguments are optional. If --chrome is specified, chrome tests will be run instead of web content tests. If --browser-chrome is specified, browser-chrome tests will be run instead of web content tests. See for details on the logging levels.""" self.set_usage(usage) ####################### # HTTP SERVER SUPPORT # ####################### class MochitestServer: "Web server used to serve Mochitests, for closer fidelity to the real web." def __init__(self, options): self._closeWhenDone = options.closeWhenDone self._utilityPath = options.utilityPath self._xrePath = options.xrePath def start(self): "Run the Mochitest server, returning the process ID of the server." env = dict(os.environ) if automation.UNIXISH: env["LD_LIBRARY_PATH"] = self._xrePath env["MOZILLA_FIVE_HOME"] = self._xrePath env["XPCOM_DEBUG_BREAK"] = "warn" elif automation.IS_MAC: env["DYLD_LIBRARY_PATH"] = self._xrePath elif automation.IS_WIN32: env["PATH"] = env["PATH"] + ";" + self._xrePath args = ["-g", self._xrePath, "-v", "170", "-f", "./" + "httpd.js", "-f", "./" + "server.js"] xpcshell = os.path.join(self._utilityPath, "xpcshell" + automation.BIN_SUFFIX) self._process = automation.Process([xpcshell] + args, env = env) pid = self._process.pid if pid < 0: print "Error starting server." sys.exit(2) automation.log.info("INFO | runtests.py | Server pid: %d", pid) def ensureReady(self, timeout): assert timeout >= 0 aliveFile = os.path.join(PROFILE_DIRECTORY, "server_alive.txt") i = 0 while i < timeout: if os.path.exists(aliveFile): break time.sleep(1) i += 1 else: print "Timed out while waiting for server startup." self.stop() sys.exit(1) def stop(self): try: c = urllib2.urlopen(SERVER_SHUTDOWN_URL) c.read() c.close() self._process.wait() except: self._process.kill() def getFullPath(path): "Get an absolute path relative to oldcwd." return os.path.normpath(os.path.join(oldcwd, os.path.expanduser(path))) def searchPath(path): "Go one step beyond getFullPath and try the various folders in PATH" # Try looking in the current working directory first. newpath = getFullPath(path) if os.path.exists(newpath): return newpath # At this point we have to fail if a directory was given (to prevent cases # like './gdb' from matching '/usr/bin/./gdb'). if not os.path.dirname(path): for dir in os.environ['PATH'].split(os.pathsep): newpath = os.path.join(dir, path) if os.path.exists(newpath): return newpath return None ################# # MAIN FUNCTION # ################# def main(): parser = MochitestOptions() options, args = parser.parse_args() if options.xrePath is None: # default xrePath to the app path if not provided # but only if an app path was explicitly provided if options.app != parser.defaults['app']: options.xrePath = os.path.dirname(options.app) else: # otherwise default to dist/bin options.xrePath = automation.DIST_BIN # allow relative paths options.xrePath = getFullPath(options.xrePath) options.app = getFullPath(options.app) if not os.path.exists(options.app): msg = """\ Error: Path %(app)s doesn't exist. Are you executing $objdir/_tests/testing/mochitest/runtests.py?""" print msg % {"app": options.app} sys.exit(1) options.utilityPath = getFullPath(options.utilityPath) options.certPath = getFullPath(options.certPath) debuggerInfo = None if options.debugger: debuggerPath = searchPath(options.debugger) if not debuggerPath: print "Error: Path %s doesn't exist." % options.debugger sys.exit(1) debuggerName = os.path.basename(debuggerPath).lower() def getDebuggerInfo(type, default): if debuggerName in DEBUGGER_INFO and type in DEBUGGER_INFO[debuggerName]: return DEBUGGER_INFO[debuggerName][type] return default debuggerInfo = { "path": debuggerPath, "interactive" : getDebuggerInfo("interactive", False), "args": getDebuggerInfo("args", "").split() } if options.debuggerArgs: debuggerInfo["args"] = options.debuggerArgs.split() if options.debuggerInteractive: debuggerInfo["interactive"] = options.debuggerInteractive # browser environment browserEnv = dict(os.environ) # These variables are necessary for correct application startup; change # via the commandline at your own risk. browserEnv["NO_EM_RESTART"] = "1" browserEnv["XPCOM_DEBUG_BREAK"] = "warn" appDir = os.path.dirname(options.app) if automation.UNIXISH: browserEnv["LD_LIBRARY_PATH"] = appDir browserEnv["MOZILLA_FIVE_HOME"] = appDir browserEnv["GNOME_DISABLE_CRASH_DIALOG"] = "1" for v in options.environment: ix = v.find("=") if ix <= 0: print "Error: syntax error in --setenv=" + v sys.exit(1) browserEnv[v[:ix]] = v[ix + 1:] automation.initializeProfile(PROFILE_DIRECTORY) manifest = addChromeToProfile(options) copyExtraFilesToProfile(options) server = MochitestServer(options) server.start() # If we're lucky, the server has fully started by now, and all paths are # ready, etc. However, xpcshell cold start times suck, at least for debug # builds. We'll try to connect to the server for awhile, and if we fail, # we'll try to kill the server and exit with an error. server.ensureReady(SERVER_STARTUP_TIMEOUT) # URL parameters to test URL: # # autorun -- kick off tests automatically # closeWhenDone -- runs quit.js after tests # logFile -- logs test run to an absolute path # # consoleLevel, fileLevel: set the logging level of the console and # file logs, if activated. # testURL = TESTS_URL + options.testPath urlOpts = [] if options.chrome: testURL = CHROMETESTS_URL if options.testPath: urlOpts.append("testPath=" + encodeURIComponent(options.testPath)) elif options.a11y: testURL = A11YTESTS_URL if options.testPath: urlOpts.append("testPath=" + encodeURIComponent(options.testPath)) # Force leak threshold. (bug 472773) if automation.IS_MAC and options.leakThreshold == 0: options.leakThreshold = 68 automation.log.info("INFO | runtests.py | setting leak threshold to %d", options.leakThreshold) elif options.browserChrome: testURL = "about:blank" # allow relative paths for logFile if options.logFile: options.logFile = getFullPath(options.logFile) if options.browserChrome: makeTestConfig(options) else: if options.autorun: urlOpts.append("autorun=1") if options.closeWhenDone: urlOpts.append("closeWhenDone=1") if options.logFile: urlOpts.append("logFile=" + encodeURIComponent(options.logFile)) urlOpts.append("fileLevel=" + encodeURIComponent(options.fileLevel)) if options.consoleLevel: urlOpts.append("consoleLevel=" + encodeURIComponent(options.consoleLevel)) if len(urlOpts) > 0: testURL += "?" + "&".join(urlOpts) browserEnv["XPCOM_MEM_BLOAT_LOG"] = LEAK_REPORT_FILE if options.fatalAssertions: browserEnv["XPCOM_DEBUG_BREAK"] = "stack-and-abort" status = automation.runApp(testURL, browserEnv, options.app, PROFILE_DIRECTORY, options.browserArgs, runSSLTunnel = True, utilityPath = options.utilityPath, xrePath = options.xrePath, certPath=options.certPath, debuggerInfo=debuggerInfo, symbolsPath=options.symbolsPath) # Server's no longer needed, and perhaps more importantly, anything it might # spew to console shouldn't disrupt the leak information table we print next. server.stop() automation.processLeakLog(LEAK_REPORT_FILE, options.leakThreshold) # delete the profile and manifest os.remove(manifest) # hanging due to non-halting threads is no fun; assume we hit the errors we # were going to hit already and exit. sys.exit(status) ####################### # CONFIGURATION SETUP # ####################### def makeTestConfig(options): "Creates a test configuration file for customizing test execution." def boolString(b): if b: return "true" return "false" logFile = options.logFile.replace("\\", "\\\\") testPath = options.testPath.replace("\\", "\\\\") content = """\ ({ autoRun: %(autorun)s, closeWhenDone: %(closeWhenDone)s, logPath: "%(logPath)s", testPath: "%(testPath)s" })""" % {"autorun": boolString(options.autorun), "closeWhenDone": boolString(options.closeWhenDone), "logPath": logFile, "testPath": testPath} config = open(os.path.join(PROFILE_DIRECTORY, "testConfig.js"), "w") config.write(content) config.close() def addChromeToProfile(options): "Adds MochiKit chrome tests to the profile." chromedir = os.path.join(PROFILE_DIRECTORY, "chrome") os.mkdir(chromedir) chrome = [] part = """ @namespace url("http://www.mozilla.org/keymaster/gatekeeper/there.is.only.xul"); /* set default namespace to XUL */ toolbar, toolbarpalette { background-color: rgb(235, 235, 235) !important; } toolbar#nav-bar { background-image: none !important; } """ chrome.append(part) # write userChrome.css chromeFile = open(os.path.join(PROFILE_DIRECTORY, "userChrome.css"), "a") chromeFile.write("".join(chrome)) chromeFile.close() # register our chrome dir chrometestDir = os.path.abspath(".") + "/" if automation.IS_WIN32: chrometestDir = "file:///" + chrometestDir.replace("\\", "/") (path, leaf) = os.path.split(options.app) manifest = os.path.join(path, "chrome", "mochikit.manifest") manifestFile = open(manifest, "w") manifestFile.write("content mochikit " + chrometestDir + " contentaccessible=yes\n") if options.browserChrome: overlayLine = "overlay " + BROWSER_CHROME_URL + " " \ "chrome://mochikit/content/browser-test-overlay.xul\n" manifestFile.write(overlayLine) manifestFile.close() return manifest def copyExtraFilesToProfile(options): "Copy extra files or dirs specified on the command line to the testing profile." for f in options.extraProfileFiles: abspath = getFullPath(f) dest = os.path.join(PROFILE_DIRECTORY, os.path.basename(abspath)) if os.path.isdir(abspath): shutil.copytree(abspath, dest) else: shutil.copy(abspath, dest) ######### # DO IT # ######### if __name__ == "__main__": main()