# # ***** BEGIN LICENSE BLOCK ***** # Version: MPL 1.1/GPL 2.0/LGPL 2.1 # # The contents of this file are subject to the Mozilla Public License Version # 1.1 (the "License"); you may not use this file except in compliance with # the License. You may obtain a copy of the License at # http://www.mozilla.org/MPL/ # # Software distributed under the License is distributed on an "AS IS" basis, # WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License # for the specific language governing rights and limitations under the # License. # # The Original Code is mozilla.org code. # # The Initial Developer of the Original Code is # Mozilla Foundation. # Portions created by the Initial Developer are Copyright (C) 1998 # the Initial Developer. All Rights Reserved. # # Contributor(s): # Robert Sayre # Jeff Walden # # Alternatively, the contents of this file may be used under the terms of # either the GNU General Public License Version 2 or later (the "GPL"), or # the GNU Lesser General Public License Version 2.1 or later (the "LGPL"), # in which case the provisions of the GPL or the LGPL are applicable instead # of those above. If you wish to allow use of your version of this file only # under the terms of either the GPL or the LGPL, and not to allow others to # use your version of this file under the terms of the MPL, indicate your # decision by deleting the provisions above and replace them with the notice # and other provisions required by the GPL or the LGPL. If you do not delete # the provisions above, a recipient may use your version of this file under # the terms of any one of the MPL, the GPL or the LGPL. # # ***** END LICENSE BLOCK ***** """ Runs the Mochitest test harness. """ from datetime import datetime import logging import optparse import os import os.path import re import sys import time import shutil from urllib import quote_plus as encodeURIComponent import urllib2 import commands import automation # Path to the test script on the server TEST_SERVER_HOST = "localhost:8888" TEST_PATH = "/tests/" CHROME_PATH = "/redirect.html"; A11Y_PATH = "/redirect-a11y.html" TESTS_URL = "http://" + TEST_SERVER_HOST + TEST_PATH CHROMETESTS_URL = "http://" + TEST_SERVER_HOST + CHROME_PATH A11YTESTS_URL = "http://" + TEST_SERVER_HOST + A11Y_PATH SERVER_SHUTDOWN_URL = "http://" + TEST_SERVER_HOST + "/server/shutdown" # main browser chrome URL, same as browser.chromeURL pref #ifdef MOZ_SUITE BROWSER_CHROME_URL = "chrome://navigator/content/navigator.xul" #else BROWSER_CHROME_URL = "chrome://browser/content/browser.xul" #endif # Max time in seconds to wait for server startup before tests will fail -- if # this seems big, it's mostly for debug machines where cold startup # (particularly after a build) takes forever. SERVER_STARTUP_TIMEOUT = 45 INFINITY = 1.0e3000 oldcwd = os.getcwd() SCRIPT_DIRECTORY = os.path.abspath(os.path.realpath(os.path.dirname(sys.argv[0]))) os.chdir(SCRIPT_DIRECTORY) PROFILE_DIRECTORY = os.path.abspath("./mochitesttestingprofile") LEAK_REPORT_FILE = PROFILE_DIRECTORY + "/" + "leaks-report.log" log = logging.getLogger() ####################### # COMMANDLINE OPTIONS # ####################### class MochitestOptions(optparse.OptionParser): """Parses Mochitest commandline options.""" def __init__(self, **kwargs): optparse.OptionParser.__init__(self, **kwargs) defaults = {} self.add_option("--close-when-done", action = "store_true", dest = "closeWhenDone", help = "close the application when tests are done running") defaults["closeWhenDone"] = False self.add_option("--appname", action = "store", type = "string", dest = "app", help = "absolute path to application, overriding default") defaults["app"] = os.path.join(SCRIPT_DIRECTORY, automation.DEFAULT_APP) self.add_option("--xre-path", action = "store", type = "string", dest = "xrePath", help = "absolute path to directory containing XRE (probably xulrunner)") # we'll default this to the directory of app below defaults["xrePath"] = None self.add_option("--utility-path", action = "store", type = "string", dest = "utilityPath", help = "absolute path to directory containing utility programs (xpcshell, ssltunnel, certutil)") defaults["utilityPath"] = automation.DIST_BIN self.add_option("--certificate-path", action = "store", type = "string", dest = "certPath", help = "absolute path to directory containing certificate store to use testing profile") defaults["certPath"] = automation.CERTS_SRC_DIR self.add_option("--log-file", action = "store", type = "string", dest = "logFile", metavar = "FILE", help = "file to which logging occurs") defaults["logFile"] = "" self.add_option("--autorun", action = "store_true", dest = "autorun", help = "start running tests when the application starts") defaults["autorun"] = False LOG_LEVELS = ("DEBUG", "INFO", "WARNING", "ERROR", "FATAL") LEVEL_STRING = ", ".join(LOG_LEVELS) self.add_option("--console-level", action = "store", type = "choice", dest = "consoleLevel", choices = LOG_LEVELS, metavar = "LEVEL", help = "one of %s to determine the level of console " "logging" % LEVEL_STRING) defaults["consoleLevel"] = None self.add_option("--file-level", action = "store", type = "choice", dest = "fileLevel", choices = LOG_LEVELS, metavar = "LEVEL", help = "one of %s to determine the level of file " "logging if a file has been specified, defaulting " "to INFO" % LEVEL_STRING) defaults["fileLevel"] = "INFO" self.add_option("--chrome", action = "store_true", dest = "chrome", help = "run chrome Mochitests") defaults["chrome"] = False self.add_option("--test-path", action = "store", type = "string", dest = "testPath", help = "start in the given directory's tests") defaults["testPath"] = "" self.add_option("--browser-chrome", action = "store_true", dest = "browserChrome", help = "run browser chrome Mochitests") defaults["browserChrome"] = False self.add_option("--a11y", action = "store_true", dest = "a11y", help = "run accessibility Mochitests"); self.add_option("--setenv", action = "append", type = "string", dest = "environment", metavar = "NAME=VALUE", help = "sets the given variable in the application's " "environment") defaults["environment"] = [] self.add_option("--browser-arg", action = "append", type = "string", dest = "browserArgs", metavar = "ARG", help = "provides an argument to the test application") defaults["browserArgs"] = [] self.add_option("--leak-threshold", action = "store", type = "int", dest = "leakThreshold", metavar = "THRESHOLD", help = "fail if the number of bytes leaked through " "refcounted objects (or bytes in classes with " "MOZ_COUNT_CTOR and MOZ_COUNT_DTOR) is greater " "than the given number") defaults["leakThreshold"] = 0 self.add_option("--fatal-assertions", action = "store_true", dest = "fatalAssertions", help = "abort testing whenever an assertion is hit " "(requires a debug build to be effective)") defaults["fatalAssertions"] = False self.add_option("--extra-profile-file", action = "append", dest = "extraProfileFiles", help = "copy specified files/dirs to testing profile") defaults["extraProfileFiles"] = [] # -h, --help are automatically handled by OptionParser self.set_defaults(**defaults) usage = """\ Usage instructions for runtests.py. All arguments are optional. If --chrome is specified, chrome tests will be run instead of web content tests. If --browser-chrome is specified, browser-chrome tests will be run instead of web content tests. See for details on the logging levels.""" self.set_usage(usage) ####################### # HTTP SERVER SUPPORT # ####################### class MochitestServer: "Web server used to serve Mochitests, for closer fidelity to the real web." def __init__(self, options): self._closeWhenDone = options.closeWhenDone self._utilityPath = options.utilityPath self._xrePath = options.xrePath def start(self): "Run the Mochitest server, returning the process ID of the server." env = dict(os.environ) if automation.UNIXISH: env["LD_LIBRARY_PATH"] = self._xrePath env["MOZILLA_FIVE_HOME"] = self._xrePath env["XPCOM_DEBUG_BREAK"] = "warn" if automation.IS_MAC: env["DYLD_LIBRARY_PATH"] = self._xrePath elif automation.IS_WIN32: env["PATH"] = env["PATH"] + ";" + self._xrePath args = ["-g", self._xrePath, "-v", "170", "-f", "./" + "httpd.js", "-f", "./" + "server.js"] xpcshell = os.path.join(self._utilityPath, "xpcshell" + automation.BIN_SUFFIX) self._process = automation.Process([xpcshell] + args, env = env) pid = self._process.pid if pid < 0: print "Error starting server." sys.exit(2) log.info("Server pid: %d", pid) def ensureReady(self, timeout): assert timeout >= 0 aliveFile = os.path.join(PROFILE_DIRECTORY, "server_alive.txt") i = 0 while i < timeout: if os.path.exists(aliveFile): break time.sleep(1) i += 1 else: print "Timed out while waiting for server startup." self.stop() sys.exit(1) def stop(self): try: c = urllib2.urlopen(SERVER_SHUTDOWN_URL) c.read() c.close() self._process.wait() except: self._process.kill() def getFullPath(path): "Get an absolute path relative to oldcwd." return os.path.normpath(os.path.join(oldcwd, os.path.expanduser(path))) ################# # MAIN FUNCTION # ################# def main(): parser = MochitestOptions() options, args = parser.parse_args() options.app = getFullPath(options.app) if not os.path.exists(options.app): msg = """\ Error: Path %(app)s doesn't exist. Are you executing $objdir/_tests/testing/mochitest/runtests.py?""" print msg % {"app": options.app} sys.exit(1) # default xrePath to the app path if not provided if options.xrePath is None: options.xrePath = os.path.dirname(options.app) else: # allow relative paths options.xrePath = getFullPath(options.xrePath) options.utilityPath = getFullPath(options.utilityPath) options.certPath = getFullPath(options.certPath) # browser environment browserEnv = dict(os.environ) # These variables are necessary for correct application startup; change # via the commandline at your own risk. browserEnv["NO_EM_RESTART"] = "1" browserEnv["XPCOM_DEBUG_BREAK"] = "warn" appDir = os.path.dirname(options.app) if automation.UNIXISH: browserEnv["LD_LIBRARY_PATH"] = appDir browserEnv["MOZILLA_FIVE_HOME"] = appDir browserEnv["GNOME_DISABLE_CRASH_DIALOG"] = "1" for v in options.environment: ix = v.find("=") if ix <= 0: print "Error: syntax error in --setenv=" + v sys.exit(1) browserEnv[v[:ix]] = v[ix + 1:] automation.initializeProfile(PROFILE_DIRECTORY) manifest = addChromeToProfile(options) copyExtraFilesToProfile(options) server = MochitestServer(options) server.start() # If we're lucky, the server has fully started by now, and all paths are # ready, etc. However, xpcshell cold start times suck, at least for debug # builds. We'll try to connect to the server for awhile, and if we fail, # we'll try to kill the server and exit with an error. server.ensureReady(SERVER_STARTUP_TIMEOUT) # URL parameters to test URL: # # autorun -- kick off tests automatically # closeWhenDone -- runs quit.js after tests # logFile -- logs test run to an absolute path # # consoleLevel, fileLevel: set the logging level of the console and # file logs, if activated. # testURL = TESTS_URL + options.testPath urlOpts = [] if options.chrome: testURL = CHROMETESTS_URL if options.testPath: urlOpts.append("testPath=" + encodeURIComponent(options.testPath)) elif options.a11y: testURL = A11YTESTS_URL if options.testPath: urlOpts.append("testPath=" + encodeURIComponent(options.testPath)) elif options.browserChrome: testURL = "about:blank" # allow relative paths for logFile if options.logFile: options.logFile = getFullPath(options.logFile) if options.browserChrome: makeTestConfig(options) else: if options.autorun: urlOpts.append("autorun=1") if options.closeWhenDone: urlOpts.append("closeWhenDone=1") if options.logFile: urlOpts.append("logFile=" + encodeURIComponent(options.logFile)) urlOpts.append("fileLevel=" + encodeURIComponent(options.fileLevel)) if options.consoleLevel: urlOpts.append("consoleLevel=" + encodeURIComponent(options.consoleLevel)) if len(urlOpts) > 0: testURL += "?" + "&".join(urlOpts) browserEnv["XPCOM_MEM_BLOAT_LOG"] = LEAK_REPORT_FILE if options.fatalAssertions: browserEnv["XPCOM_DEBUG_BREAK"] = "stack-and-abort" (status, start) = automation.runApp(testURL, browserEnv, options.app, PROFILE_DIRECTORY, options.browserArgs, runSSLTunnel = True, utilityPath=options.utilityPath, xrePath=options.xrePath, certPath=options.certPath) # Server's no longer needed, and perhaps more importantly, anything it might # spew to console shouldn't disrupt the leak information table we print next. server.stop() if not os.path.exists(LEAK_REPORT_FILE): log.info("WARNING refcount logging is off, so leaks can't be detected!") else: # Per-Inst Leaked Total Rem ... # 0 TOTAL 17 192 419115886 2 ... # 833 nsTimerImpl 60 120 24726 2 ... lineRe = re.compile(r"^\s*\d+\s+(?P\S+)\s+" r"(?P-?\d+)\s+(?P-?\d+)\s+" r"\d+\s+(?P-?\d+)") leaks = open(LEAK_REPORT_FILE, "r") for line in leaks: matches = lineRe.match(line) if (matches and int(matches.group("numLeaked")) == 0 and matches.group("name") != "TOTAL"): continue log.info(line.rstrip()) leaks.close() threshold = options.leakThreshold leaks = open(LEAK_REPORT_FILE, "r") seenTotal = False prefix = "TEST-PASS" for line in leaks: matches = lineRe.match(line) if not matches: continue name = matches.group("name") size = int(matches.group("size")) bytesLeaked = int(matches.group("bytesLeaked")) numLeaked = int(matches.group("numLeaked")) if size < 0 or bytesLeaked < 0 or numLeaked < 0: log.info("TEST-UNEXPECTED-FAIL | runtests-leaks | negative leaks caught!") if "TOTAL" == name: seenTotal = True # Check for leaks. if bytesLeaked < 0 or bytesLeaked > threshold: prefix = "TEST-UNEXPECTED-FAIL" leakLog = "TEST-UNEXPECTED-FAIL | runtests-leaks | leaked" \ " %d bytes during test execution" % bytesLeaked elif bytesLeaked > 0: leakLog = "TEST-PASS | runtests-leaks | WARNING leaked" \ " %d bytes during test execution" % bytesLeaked else: leakLog = "TEST-PASS | runtests-leaks | no leaks detected!" # Remind the threshold if it is not 0, which is the default/goal. if threshold != 0: leakLog += " (threshold set at %d bytes)" % threshold # Log the information. log.info(leakLog) else: if numLeaked != 0: if abs(numLeaked) > 1: instance = "instances" rest = " each (%s bytes total)" % matches.group("bytesLeaked") else: instance = "instance" rest = "" log.info("%(prefix)s | runtests-leaks | leaked %(numLeaked)d %(instance)s of %(name)s " "with size %(size)s bytes%(rest)s" % { "prefix": prefix, "numLeaked": numLeaked, "instance": instance, "name": name, "size": matches.group("size"), "rest": rest }) if not seenTotal: log.info("TEST-UNEXPECTED-FAIL | runtests-leaks | missing output line for total leaks!") leaks.close() # print test run times finish = datetime.now() log.info(" started: %s", str(start)) log.info("finished: %s", str(finish)) # delete the profile and manifest os.remove(manifest) # hanging due to non-halting threads is no fun; assume we hit the errors we # were going to hit already and exit with a success code sys.exit(0) ####################### # CONFIGURATION SETUP # ####################### def makeTestConfig(options): "Creates a test configuration file for customizing test execution." def boolString(b): if b: return "true" return "false" logFile = options.logFile.replace("\\", "\\\\") testPath = options.testPath.replace("\\", "\\\\") content = """\ ({ autoRun: %(autorun)s, closeWhenDone: %(closeWhenDone)s, logPath: "%(logPath)s", testPath: "%(testPath)s" })""" % {"autorun": boolString(options.autorun), "closeWhenDone": boolString(options.closeWhenDone), "logPath": logFile, "testPath": testPath} config = open(os.path.join(PROFILE_DIRECTORY, "testConfig.js"), "w") config.write(content) config.close() def addChromeToProfile(options): "Adds MochiKit chrome tests to the profile." chromedir = os.path.join(PROFILE_DIRECTORY, "chrome") os.mkdir(chromedir) chrome = [] part = """ @namespace url("http://www.mozilla.org/keymaster/gatekeeper/there.is.only.xul"); /* set default namespace to XUL */ toolbar, toolbarpalette { background-color: rgb(235, 235, 235) !important; } toolbar#nav-bar { background-image: none !important; } """ chrome.append(part) # write userChrome.css chromeFile = open(os.path.join(PROFILE_DIRECTORY, "userChrome.css"), "a") chromeFile.write("".join(chrome)) chromeFile.close() # register our chrome dir chrometestDir = os.path.abspath(".") + "/" if automation.IS_WIN32: chrometestDir = "file:///" + chrometestDir.replace("\\", "/") (path, leaf) = os.path.split(options.app) manifest = os.path.join(path, "chrome", "mochikit.manifest") manifestFile = open(manifest, "w") manifestFile.write("content mochikit " + chrometestDir + " contentaccessible=yes\n") if options.browserChrome: overlayLine = "overlay " + BROWSER_CHROME_URL + " " \ "chrome://mochikit/content/browser-test-overlay.xul\n" manifestFile.write(overlayLine) manifestFile.close() return manifest def copyExtraFilesToProfile(options): "Copy extra files or dirs specified on the command line to the testing profile." for f in options.extraProfileFiles: abspath = getFullPath(f) dest = os.path.join(PROFILE_DIRECTORY, os.path.basename(abspath)) if os.path.isdir(abspath): shutil.copytree(abspath, dest) else: shutil.copy(abspath, dest) ######### # DO IT # ######### if __name__ == "__main__": main()