# # ***** BEGIN LICENSE BLOCK ***** # Version: MPL 1.1/GPL 2.0/LGPL 2.1 # # The contents of this file are subject to the Mozilla Public License Version # 1.1 (the "License"); you may not use this file except in compliance with # the License. You may obtain a copy of the License at # http://www.mozilla.org/MPL/ # # Software distributed under the License is distributed on an "AS IS" basis, # WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License # for the specific language governing rights and limitations under the # License. # # The Original Code is mozilla.org code. # # The Initial Developer of the Original Code is # Mozilla Foundation. # Portions created by the Initial Developer are Copyright (C) 1998 # the Initial Developer. All Rights Reserved. # # Contributor(s): # Robert Sayre # Jeff Walden # Serge Gautherie # # Alternatively, the contents of this file may be used under the terms of # either the GNU General Public License Version 2 or later (the "GPL"), or # the GNU Lesser General Public License Version 2.1 or later (the "LGPL"), # in which case the provisions of the GPL or the LGPL are applicable instead # of those above. If you wish to allow use of your version of this file only # under the terms of either the GPL or the LGPL, and not to allow others to # use your version of this file under the terms of the MPL, indicate your # decision by deleting the provisions above and replace them with the notice # and other provisions required by the GPL or the LGPL. If you do not delete # the provisions above, a recipient may use your version of this file under # the terms of any one of the MPL, the GPL or the LGPL. # # ***** END LICENSE BLOCK ***** """ Runs the Mochitest test harness. """ from datetime import datetime import optparse import os import os.path import sys import time import shutil from urllib import quote_plus as encodeURIComponent import urllib2 import commands import automation from automationutils import * # Path to the test script on the server TEST_SERVER_HOST = "localhost:8888" TEST_PATH = "/tests/" CHROME_PATH = "/redirect.html"; A11Y_PATH = "/redirect-a11y.html" TESTS_URL = "http://" + TEST_SERVER_HOST + TEST_PATH CHROMETESTS_URL = "http://" + TEST_SERVER_HOST + CHROME_PATH A11YTESTS_URL = "http://" + TEST_SERVER_HOST + A11Y_PATH SERVER_SHUTDOWN_URL = "http://" + TEST_SERVER_HOST + "/server/shutdown" # main browser chrome URL, same as browser.chromeURL pref #ifdef MOZ_SUITE BROWSER_CHROME_URL = "chrome://navigator/content/navigator.xul" #else BROWSER_CHROME_URL = "chrome://browser/content/browser.xul" #endif # Max time in seconds to wait for server startup before tests will fail -- if # this seems big, it's mostly for debug machines where cold startup # (particularly after a build) takes forever. if automation.IS_DEBUG_BUILD: SERVER_STARTUP_TIMEOUT = 180 else: SERVER_STARTUP_TIMEOUT = 90 oldcwd = os.getcwd() SCRIPT_DIRECTORY = os.path.abspath(os.path.realpath(os.path.dirname(sys.argv[0]))) os.chdir(SCRIPT_DIRECTORY) PROFILE_DIRECTORY = os.path.abspath("./mochitesttestingprofile") LEAK_REPORT_FILE = os.path.join(PROFILE_DIRECTORY, "runtests_leaks.log") ####################### # COMMANDLINE OPTIONS # ####################### class MochitestOptions(optparse.OptionParser): """Parses Mochitest commandline options.""" def __init__(self, **kwargs): optparse.OptionParser.__init__(self, **kwargs) defaults = {} # we want to pass down everything from automation.__all__ addCommonOptions(self, defaults=dict(zip(automation.__all__, [getattr(automation, x) for x in automation.__all__]))) automation.addExtraCommonOptions(self) self.add_option("--close-when-done", action = "store_true", dest = "closeWhenDone", help = "close the application when tests are done running") defaults["closeWhenDone"] = False self.add_option("--appname", action = "store", type = "string", dest = "app", help = "absolute path to application, overriding default") defaults["app"] = os.path.join(SCRIPT_DIRECTORY, automation.DEFAULT_APP) self.add_option("--utility-path", action = "store", type = "string", dest = "utilityPath", help = "absolute path to directory containing utility programs (xpcshell, ssltunnel, certutil)") defaults["utilityPath"] = automation.DIST_BIN self.add_option("--certificate-path", action = "store", type = "string", dest = "certPath", help = "absolute path to directory containing certificate store to use testing profile") defaults["certPath"] = automation.CERTS_SRC_DIR self.add_option("--log-file", action = "store", type = "string", dest = "logFile", metavar = "FILE", help = "file to which logging occurs") defaults["logFile"] = "" self.add_option("--autorun", action = "store_true", dest = "autorun", help = "start running tests when the application starts") defaults["autorun"] = False self.add_option("--timeout", type = "int", dest = "timeout", help = "per-test timeout in seconds") defaults["timeout"] = None self.add_option("--total-chunks", type = "int", dest = "totalChunks", help = "how many chunks to split the tests up into") defaults["totalChunks"] = None self.add_option("--this-chunk", type = "int", dest = "thisChunk", help = "which chunk to run") defaults["thisChunk"] = None self.add_option("--chunk-by-dir", type = "int", dest = "chunkByDir", help = "group tests together in the same chunk that are in the same top chunkByDir directories") defaults["chunkByDir"] = 0 self.add_option("--shuffle", dest = "shuffle", action = "store_true", help = "randomize test order") defaults["shuffle"] = False LOG_LEVELS = ("DEBUG", "INFO", "WARNING", "ERROR", "FATAL") LEVEL_STRING = ", ".join(LOG_LEVELS) self.add_option("--console-level", action = "store", type = "choice", dest = "consoleLevel", choices = LOG_LEVELS, metavar = "LEVEL", help = "one of %s to determine the level of console " "logging" % LEVEL_STRING) defaults["consoleLevel"] = None self.add_option("--file-level", action = "store", type = "choice", dest = "fileLevel", choices = LOG_LEVELS, metavar = "LEVEL", help = "one of %s to determine the level of file " "logging if a file has been specified, defaulting " "to INFO" % LEVEL_STRING) defaults["fileLevel"] = "INFO" self.add_option("--chrome", action = "store_true", dest = "chrome", help = "run chrome Mochitests") defaults["chrome"] = False self.add_option("--test-path", action = "store", type = "string", dest = "testPath", help = "start in the given directory's tests") defaults["testPath"] = "" self.add_option("--browser-chrome", action = "store_true", dest = "browserChrome", help = "run browser chrome Mochitests") defaults["browserChrome"] = False self.add_option("--a11y", action = "store_true", dest = "a11y", help = "run accessibility Mochitests"); self.add_option("--setenv", action = "append", type = "string", dest = "environment", metavar = "NAME=VALUE", help = "sets the given variable in the application's " "environment") defaults["environment"] = [] self.add_option("--browser-arg", action = "append", type = "string", dest = "browserArgs", metavar = "ARG", help = "provides an argument to the test application") defaults["browserArgs"] = [] self.add_option("--leak-threshold", action = "store", type = "int", dest = "leakThreshold", metavar = "THRESHOLD", help = "fail if the number of bytes leaked through " "refcounted objects (or bytes in classes with " "MOZ_COUNT_CTOR and MOZ_COUNT_DTOR) is greater " "than the given number") defaults["leakThreshold"] = 0 self.add_option("--fatal-assertions", action = "store_true", dest = "fatalAssertions", help = "abort testing whenever an assertion is hit " "(requires a debug build to be effective)") defaults["fatalAssertions"] = False self.add_option("--extra-profile-file", action = "append", dest = "extraProfileFiles", help = "copy specified files/dirs to testing profile") defaults["extraProfileFiles"] = [] # -h, --help are automatically handled by OptionParser self.set_defaults(**defaults) usage = """\ Usage instructions for runtests.py. All arguments are optional. If --chrome is specified, chrome tests will be run instead of web content tests. If --browser-chrome is specified, browser-chrome tests will be run instead of web content tests. See for details on the logging levels.""" self.set_usage(usage) ####################### # HTTP SERVER SUPPORT # ####################### class MochitestServer: "Web server used to serve Mochitests, for closer fidelity to the real web." def __init__(self, options): self._closeWhenDone = options.closeWhenDone self._utilityPath = options.utilityPath self._xrePath = options.xrePath def start(self): "Run the Mochitest server, returning the process ID of the server." env = automation.environment(xrePath = self._xrePath) env["XPCOM_DEBUG_BREAK"] = "warn" if automation.IS_WIN32: env["PATH"] = env["PATH"] + ";" + self._xrePath args = ["-g", self._xrePath, "-v", "170", "-f", "./" + "httpd.js", "-f", "./" + "server.js"] xpcshell = os.path.join(self._utilityPath, "xpcshell" + automation.BIN_SUFFIX) self._process = automation.Process([xpcshell] + args, env = env) pid = self._process.pid if pid < 0: print "Error starting server." sys.exit(2) automation.log.info("INFO | runtests.py | Server pid: %d", pid) def ensureReady(self, timeout): assert timeout >= 0 aliveFile = os.path.join(PROFILE_DIRECTORY, "server_alive.txt") i = 0 while i < timeout: if os.path.exists(aliveFile): break time.sleep(1) i += 1 else: print "Timed out while waiting for server startup." self.stop() sys.exit(1) def stop(self): try: c = urllib2.urlopen(SERVER_SHUTDOWN_URL) c.read() c.close() self._process.wait() except: self._process.kill() def getFullPath(path): "Get an absolute path relative to oldcwd." return os.path.normpath(os.path.join(oldcwd, os.path.expanduser(path))) ################# # MAIN FUNCTION # ################# def main(): parser = MochitestOptions() options, args = parser.parse_args() if options.totalChunks is not None and options.thisChunk is None: parser.error("thisChunk must be specified when totalChunks is specified") if options.totalChunks: if not 1 <= options.thisChunk <= options.totalChunks: parser.error("thisChunk must be between 1 and totalChunks") if options.xrePath is None: # default xrePath to the app path if not provided # but only if an app path was explicitly provided if options.app != parser.defaults['app']: options.xrePath = os.path.dirname(options.app) else: # otherwise default to dist/bin options.xrePath = automation.DIST_BIN # allow relative paths options.xrePath = getFullPath(options.xrePath) options.app = getFullPath(options.app) if not os.path.exists(options.app): msg = """\ Error: Path %(app)s doesn't exist. Are you executing $objdir/_tests/testing/mochitest/runtests.py?""" print msg % {"app": options.app} sys.exit(1) options.utilityPath = getFullPath(options.utilityPath) options.certPath = getFullPath(options.certPath) if options.symbolsPath: options.symbolsPath = getFullPath(options.symbolsPath) debuggerInfo = getDebuggerInfo(oldcwd, options.debugger, options.debuggerArgs, options.debuggerInteractive); # browser environment browserEnv = automation.environment(xrePath = options.xrePath) # These variables are necessary for correct application startup; change # via the commandline at your own risk. browserEnv["XPCOM_DEBUG_BREAK"] = "stack" for v in options.environment: ix = v.find("=") if ix <= 0: print "Error: syntax error in --setenv=" + v sys.exit(1) browserEnv[v[:ix]] = v[ix + 1:] automation.initializeProfile(PROFILE_DIRECTORY, options.extraPrefs) manifest = addChromeToProfile(options) copyExtraFilesToProfile(options) server = MochitestServer(options) server.start() # If we're lucky, the server has fully started by now, and all paths are # ready, etc. However, xpcshell cold start times suck, at least for debug # builds. We'll try to connect to the server for awhile, and if we fail, # we'll try to kill the server and exit with an error. server.ensureReady(SERVER_STARTUP_TIMEOUT) # URL parameters to test URL: # # autorun -- kick off tests automatically # closeWhenDone -- runs quit.js after tests # logFile -- logs test run to an absolute path # totalChunks -- how many chunks to split tests into # thisChunk -- which chunk to run # timeout -- per-test timeout in seconds # # consoleLevel, fileLevel: set the logging level of the console and # file logs, if activated. # testURL = TESTS_URL + options.testPath urlOpts = [] if options.chrome: testURL = CHROMETESTS_URL if options.testPath: urlOpts.append("testPath=" + encodeURIComponent(options.testPath)) elif options.a11y: testURL = A11YTESTS_URL if options.testPath: urlOpts.append("testPath=" + encodeURIComponent(options.testPath)) elif options.browserChrome: testURL = "about:blank" # allow relative paths for logFile if options.logFile: options.logFile = getFullPath(options.logFile) if options.browserChrome: makeTestConfig(options) else: if options.autorun: urlOpts.append("autorun=1") if options.timeout: urlOpts.append("timeout=%d" % options.timeout) if options.closeWhenDone: urlOpts.append("closeWhenDone=1") if options.logFile: urlOpts.append("logFile=" + encodeURIComponent(options.logFile)) urlOpts.append("fileLevel=" + encodeURIComponent(options.fileLevel)) if options.consoleLevel: urlOpts.append("consoleLevel=" + encodeURIComponent(options.consoleLevel)) if options.totalChunks: urlOpts.append("totalChunks=%d" % options.totalChunks) urlOpts.append("thisChunk=%d" % options.thisChunk) if options.chunkByDir: urlOpts.append("chunkByDir=%d" % options.chunkByDir) if options.shuffle: urlOpts.append("shuffle=1") if len(urlOpts) > 0: testURL += "?" + "&".join(urlOpts) browserEnv["XPCOM_MEM_BLOAT_LOG"] = LEAK_REPORT_FILE if options.fatalAssertions: browserEnv["XPCOM_DEBUG_BREAK"] = "stack-and-abort" # run once with -silent to let the extension manager do its thing # and then exit the app automation.log.info("INFO | runtests.py | Performing extension manager registration: start.\n") # Don't care about this |status|: |runApp()| reporting it should be enough. status = automation.runApp(None, browserEnv, options.app, PROFILE_DIRECTORY, ["-silent"], utilityPath = options.utilityPath, xrePath = options.xrePath, symbolsPath=options.symbolsPath) # We don't care to call |processLeakLog()| for this step. automation.log.info("\nINFO | runtests.py | Performing extension manager registration: end.") # Remove the leak detection file so it can't "leak" to the tests run. # The file is not there if leak logging was not enabled in the application build. if os.path.exists(LEAK_REPORT_FILE): os.remove(LEAK_REPORT_FILE) # then again to actually run mochitest if options.timeout: timeout = options.timeout + 30 elif options.autorun: timeout = None else: timeout = 330.0 # default JS harness timeout is 300 seconds automation.log.info("INFO | runtests.py | Running tests: start.\n") status = automation.runApp(testURL, browserEnv, options.app, PROFILE_DIRECTORY, options.browserArgs, runSSLTunnel = True, utilityPath = options.utilityPath, xrePath = options.xrePath, certPath=options.certPath, debuggerInfo=debuggerInfo, symbolsPath=options.symbolsPath, timeout = timeout) # Server's no longer needed, and perhaps more importantly, anything it might # spew to console shouldn't disrupt the leak information table we print next. server.stop() processLeakLog(LEAK_REPORT_FILE, options.leakThreshold) automation.log.info("\nINFO | runtests.py | Running tests: end.") # delete the profile and manifest os.remove(manifest) # hanging due to non-halting threads is no fun; assume we hit the errors we # were going to hit already and exit. sys.exit(status) ####################### # CONFIGURATION SETUP # ####################### def makeTestConfig(options): "Creates a test configuration file for customizing test execution." def boolString(b): if b: return "true" return "false" logFile = options.logFile.replace("\\", "\\\\") testPath = options.testPath.replace("\\", "\\\\") content = """\ ({ autoRun: %(autorun)s, closeWhenDone: %(closeWhenDone)s, logPath: "%(logPath)s", testPath: "%(testPath)s" })""" % {"autorun": boolString(options.autorun), "closeWhenDone": boolString(options.closeWhenDone), "logPath": logFile, "testPath": testPath} config = open(os.path.join(PROFILE_DIRECTORY, "testConfig.js"), "w") config.write(content) config.close() def addChromeToProfile(options): "Adds MochiKit chrome tests to the profile." chromedir = os.path.join(PROFILE_DIRECTORY, "chrome") os.mkdir(chromedir) chrome = [] part = """ @namespace url("http://www.mozilla.org/keymaster/gatekeeper/there.is.only.xul"); /* set default namespace to XUL */ toolbar, toolbarpalette { background-color: rgb(235, 235, 235) !important; } toolbar#nav-bar { background-image: none !important; } """ chrome.append(part) # write userChrome.css chromeFile = open(os.path.join(PROFILE_DIRECTORY, "userChrome.css"), "a") chromeFile.write("".join(chrome)) chromeFile.close() # register our chrome dir chrometestDir = os.path.abspath(".") + "/" if automation.IS_WIN32: chrometestDir = "file:///" + chrometestDir.replace("\\", "/") (path, leaf) = os.path.split(options.app) manifest = os.path.join(path, "chrome", "mochikit.manifest") manifestFile = open(manifest, "w") manifestFile.write("content mochikit " + chrometestDir + " contentaccessible=yes\n") if options.browserChrome: overlayLine = "overlay " + BROWSER_CHROME_URL + " " \ "chrome://mochikit/content/browser-test-overlay.xul\n" manifestFile.write(overlayLine) manifestFile.close() return manifest def copyExtraFilesToProfile(options): "Copy extra files or dirs specified on the command line to the testing profile." for f in options.extraProfileFiles: abspath = getFullPath(f) dest = os.path.join(PROFILE_DIRECTORY, os.path.basename(abspath)) if os.path.isdir(abspath): shutil.copytree(abspath, dest) else: shutil.copy(abspath, dest) ######### # DO IT # ######### if __name__ == "__main__": main()