# # ***** BEGIN LICENSE BLOCK ***** # Version: MPL 1.1/GPL 2.0/LGPL 2.1 # # The contents of this file are subject to the Mozilla Public License Version # 1.1 (the "License"); you may not use this file except in compliance with # the License. You may obtain a copy of the License at # http://www.mozilla.org/MPL/ # # Software distributed under the License is distributed on an "AS IS" basis, # WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License # for the specific language governing rights and limitations under the # License. # # The Original Code is mozilla.org code. # # The Initial Developer of the Original Code is # Mozilla Foundation. # Portions created by the Initial Developer are Copyright (C) 1998 # the Initial Developer. All Rights Reserved. # # Contributor(s): # Robert Sayre # Jeff Walden # # Alternatively, the contents of this file may be used under the terms of # either the GNU General Public License Version 2 or later (the "GPL"), or # the GNU Lesser General Public License Version 2.1 or later (the "LGPL"), # in which case the provisions of the GPL or the LGPL are applicable instead # of those above. If you wish to allow use of your version of this file only # under the terms of either the GPL or the LGPL, and not to allow others to # use your version of this file under the terms of the MPL, indicate your # decision by deleting the provisions above and replace them with the notice # and other provisions required by the GPL or the LGPL. If you do not delete # the provisions above, a recipient may use your version of this file under # the terms of any one of the MPL, the GPL or the LGPL. # # ***** END LICENSE BLOCK ***** """ Runs the Mochitest test harness. """ from datetime import datetime import optparse import os import os.path import re import signal import sys import time from urllib import quote_plus as encodeURIComponent import urllib2 import commands import automation # Path to the test script on the server TEST_SERVER_HOST = "localhost:8888" TEST_PATH = "/tests/" CHROME_PATH = "/redirect.html"; A11Y_PATH = "/redirect-a11y.html" TESTS_URL = "http://" + TEST_SERVER_HOST + TEST_PATH CHROMETESTS_URL = "http://" + TEST_SERVER_HOST + CHROME_PATH A11YTESTS_URL = "http://" + TEST_SERVER_HOST + A11Y_PATH SERVER_SHUTDOWN_URL = "http://" + TEST_SERVER_HOST + "/server/shutdown" # main browser chrome URL, same as browser.chromeURL pref #ifdef MOZ_SUITE BROWSER_CHROME_URL = "chrome://navigator/content/navigator.xul" #else BROWSER_CHROME_URL = "chrome://browser/content/browser.xul" #endif # Max time in seconds to wait for server startup before tests will fail -- if # this seems big, it's mostly for debug machines where cold startup # (particularly after a build) takes forever. SERVER_STARTUP_TIMEOUT = 45 SCRIPT_DIRECTORY = os.path.abspath(os.path.realpath(os.path.dirname(sys.argv[0]))) os.chdir(SCRIPT_DIRECTORY) PROFILE_DIRECTORY = os.path.abspath("./mochitesttestingprofile") LEAK_REPORT_FILE = PROFILE_DIRECTORY + "/" + "leaks-report.log" ####################### # COMMANDLINE OPTIONS # ####################### class MochitestOptions(optparse.OptionParser): """Parses Mochitest commandline options.""" def __init__(self, **kwargs): optparse.OptionParser.__init__(self, **kwargs) defaults = {} self.add_option("--close-when-done", action = "store_true", dest = "closeWhenDone", help = "close the application when tests are done running") defaults["closeWhenDone"] = False self.add_option("--appname", action = "store", type = "string", dest = "app", help = "absolute path to application, overriding default") defaults["app"] = automation.DEFAULT_APP self.add_option("--log-file", action = "store", type = "string", dest = "logFile", help = "file to which logging occurs") defaults["logFile"] = "" self.add_option("--autorun", action = "store_true", dest = "autorun", help = "start running tests when the application starts") defaults["autorun"] = False LOG_LEVELS = ("DEBUG", "INFO", "ERROR", "FATAL", "WARNING") self.add_option("--console-level", action = "store", type = "choice", dest = "consoleLevel", choices = LOG_LEVELS, help = "logging level of console logging") defaults["consoleLevel"] = None self.add_option("--file-level", action = "store", type = "choice", dest = "fileLevel", choices = LOG_LEVELS, help = "logging level of file logging") defaults["fileLevel"] = None self.add_option("--chrome", action = "store_true", dest = "chrome", help = "run chrome Mochitests") defaults["chrome"] = False self.add_option("--test-path", action = "store", type = "string", dest = "testPath", help = "start in the given directory's tests") defaults["testPath"] = "" self.add_option("--browser-chrome", action = "store_true", dest = "browserChrome", help = "run browser chrome Mochitests") defaults["browserChrome"] = False self.add_option("--a11y", action = "store_true", dest = "a11y", help = "run accessibility Mochitests"); self.add_option("--setenv", action = "append", type = "string", dest = "environment", help = "given a VAR=value pair, sets that in the " "browser's environment") defaults["environment"] = [] self.add_option("--browser-arg", action = "append", type = "string", dest = "browserArgs", help = "provides an argument to the test application") defaults["browserArgs"] = [] self.add_option("--leak-threshold", action = "store", type = "int", dest = "leakThreshold", help = "fail if the number of bytes leaked through " "refcounted objects (or bytes in classes with " "MOZ_COUNT_CTOR and MOZ_COUNT_DTOR) is greater " "than the given number") defaults["leakThreshold"] = 1.0e3000 # -h, --help are automatically handled by OptionParser self.set_defaults(**defaults) usage = """\ Usage instructions for runtests.py. All arguments are optional. If --log-file is specified, --file-level must be specified as well. If --chrome is specified, chrome tests will be run instead of web content tests. If --browser-chrome is specified, browser-chrome tests will be run instead of web content tests. Logging levels are one of %s. For further details on the logging levels, see .""" % ", ".join(LOG_LEVELS) self.set_usage(usage) ####################### # HTTP SERVER SUPPORT # ####################### class MochitestServer: "Web server used to serve Mochitests, for closer fidelity to the real web." def __init__(self, options): self._closeWhenDone = options.closeWhenDone def start(self): "Run the Mochitest server, returning the process ID of the server." env = dict(os.environ) if automation.UNIXISH: env["LD_LIBRARY_PATH"] = automation.DIST_BIN env["MOZILLA_FIVE_HOME"] = automation.DIST_BIN env["XPCOM_DEBUG_BREAK"] = "warn" args = ["-v", "170", "-f", "./" + "httpd.js", "-f", "./" + "server.js"] xpcshell = automation.DIST_BIN + "/" + "xpcshell"; self._process = automation.Process(xpcshell, args, env = env) pid = self._process.pid if pid < 0: print "Error starting server." sys.exit(2) print "Server pid: " + str(pid) def ensureReady(self, timeout): assert timeout >= 0 aliveFile = PROFILE_DIRECTORY + "/" + "server_alive.txt" i = 0 while i < timeout: if os.path.exists(aliveFile): break time.sleep(1) i += 1 else: print "Timed out while waiting for server startup." self.stop() sys.exit(1) def stop(self): pid = self._process.pid try: c = urllib2.urlopen(SERVER_SHUTDOWN_URL) c.read() c.close() os.waitpid(pid, 0) except: if automation.IS_WIN32: pass # XXX do something here! else: os.kill(pid, signal.SIGKILL) ################# # MAIN FUNCTION # ################# def main(): parser = MochitestOptions() options, args = parser.parse_args() if not os.path.exists(options.app): msg = """\ Error: Path %(app)s doesn't exist. Are you executing $objdir/_tests/testing/mochitest/runtests.py?""" print msg % {"app": options.app} sys.exit(1) # browser environment browserEnv = dict(os.environ) # These variables are necessary for correct application startup; change # via the commandline at your own risk. browserEnv["NO_EM_RESTART"] = "1" browserEnv["XPCOM_DEBUG_BREAK"] = "warn" if automation.UNIXISH: browserEnv["LD_LIBRARY_PATH"] = automation.DIST_BIN browserEnv["MOZILLA_FIVE_HOME"] = automation.DIST_BIN browserEnv["GNOME_DISABLE_CRASH_DIALOG"] = "1" for v in options.environment: ix = v.find("=") if ix <= 0: print "Error: syntax error in --setenv=" + v sys.exit(1) browserEnv[v[:ix]] = v[ix + 1:] automation.initializeProfile(PROFILE_DIRECTORY) manifest = addChromeToProfile(options) server = MochitestServer(options) server.start() # If we're lucky, the server has fully started by now, and all paths are # ready, etc. However, xpcshell cold start times suck, at least for debug # builds. We'll try to connect to the server for awhile, and if we fail, # we'll try to kill the server and exit with an error. server.ensureReady(SERVER_STARTUP_TIMEOUT) # URL parameters to test URL: # # autorun -- kick off tests automatically # closeWhenDone -- runs quit.js after tests # logFile -- logs test run to an absolute path # # consoleLevel, fileLevel: set the logging level of the console and # file logs, if activated. # testURL = TESTS_URL + options.testPath urlOpts = [] if options.chrome: testURL = CHROMETESTS_URL if options.testPath: urlOpts.append("testPath=" + encodeURIComponent(options.testPath)) elif options.a11y: testURL = A11YTESTS_URL if options.testPath: urlOpts.append("testPath=" + encodeURIComponent(options.testPath)) elif options.browserChrome: testURL = "about:blank" if options.browserChrome: makeTestConfig(options) else: if options.autorun: urlOpts.append("autorun=1") if options.closeWhenDone: urlOpts.append("closeWhenDone=1") if options.logFile: urlOpts.append("logFile=" + encodeURIComponent(options.logFile)) if options.fileLevel: urlOpts.append("fileLevel=" + encodeURIComponent(options.fileLevel)) if options.consoleLevel: urlOpts.append("consoleLevel=" + encodeURIComponent(options.consoleLevel)) if len(urlOpts) > 0: testURL += "?" + "&".join(urlOpts) browserEnv["XPCOM_MEM_BLOAT_LOG"] = LEAK_REPORT_FILE start = automation.runApp(testURL, browserEnv, options.app, PROFILE_DIRECTORY, options.browserArgs) server.stop() if not os.path.exists(LEAK_REPORT_FILE): print "WARNING refcount logging is off, so leaks can't be detected!" else: leaks = open(LEAK_REPORT_FILE, "r") for line in leaks: print line, leaks.close() threshold = options.leakThreshold leaks = open(LEAK_REPORT_FILE, "r") # Per-Inst Leaked Total Rem ... # 0 TOTAL 17 192 419115886 2 ... # 833 nsTimerImpl 60 120 24726 2 ... lineRe = re.compile(r"^\s*\d+\s+(?P\S+)\s+" r"(?P\d+)\s+(?P\d+)\s+" r"\d+\s+(?P\d+)") thresholdExceeded = False seenTotal = False prefix = "WARNING" for line in leaks: matches = lineRe.match(line) if not matches: continue name = matches.group("name") if "TOTAL" == name: seenTotal = True bytesLeaked = int(matches.group("bytesLeaked")) if bytesLeaked > threshold: thresholdExceeded = True prefix = "ERROR FAIL" print ("ERROR FAIL leaked %(actual)d bytes during test " "execution (should have leaked no more than " "%(expect)d bytes)") % { "actual": bytesLeaked, "expect": threshold } elif bytesLeaked > 0: print "WARNING leaked %d bytes during test execution" % bytesLeaked else: numLeaked = int(matches.group("numLeaked")) if numLeaked != 0: if numLeaked > 1: instance = "instances" rest = " each (%s bytes total)" % matches.group("bytesLeaked") else: instance = "instance" rest = "" vars = { "prefix": prefix, "numLeaked": numLeaked, "instance": instance, "name": name, "size": matches.group("size"), "rest": rest } print ("%(prefix)s leaked %(numLeaked)d %(instance)s of %(name)s " "with size %(size)s bytes%(rest)s") % vars if not seenTotal: print "ERROR FAIL missing output line for total leaks!" leaks.close() # print test run times finish = datetime.now() print " started: " + str(start) print "finished: " + str(finish) # delete the profile and manifest os.remove(manifest) ####################### # CONFIGURATION SETUP # ####################### def makeTestConfig(options): "Creates a test configuration file for customizing test execution." def boolString(b): if b: return "true" return "false" logFile = options.logFile.replace("\\", "\\\\") testPath = options.testPath.replace("\\", "\\\\") content = """\ ({ autoRun: %(autorun)s, closeWhenDone: %(closeWhenDone)s, logPath: "%(logPath)s", testPath: "%(testPath)s" })""" % {"autorun": boolString(options.autorun), "closeWhenDone": boolString(options.closeWhenDone), "logPath": logFile, "testPath": testPath} config = open(PROFILE_DIRECTORY + "/" + "testConfig.js", "w") config.write(content) config.close() def addChromeToProfile(options): "Adds MochiKit chrome tests to the profile." chromedir = PROFILE_DIRECTORY + "/" + "chrome" os.mkdir(chromedir) chrome = [] part = """ @namespace url("http://www.mozilla.org/keymaster/gatekeeper/there.is.only.xul"); /* set default namespace to XUL */ toolbar, toolbarpalette { background-color: rgb(235, 235, 235) !important; } toolbar#nav-bar { background-image: none !important; } """ chrome.append(part) # write userChrome.css chromeFile = open(PROFILE_DIRECTORY + "/" + "userChrome.css", "a") chromeFile.write("".join(chrome)) chromeFile.close() # register our chrome dir chrometestDir = os.path.abspath(".") + "/" if automation.IS_WIN32: chrometestDir = "file:///" + chrometestDir.replace("\\", "/") (path, leaf) = os.path.split(options.app) manifest = path + "/" + "chrome/mochikit.manifest" manifestFile = open(manifest, "w") manifestFile.write("content mochikit " + chrometestDir + "\n") if options.browserChrome: overlayLine = "overlay " + BROWSER_CHROME_URL + " " \ "chrome://mochikit/content/browser-test-overlay.xul\n" manifestFile.write(overlayLine) manifestFile.close() return manifest ######### # DO IT # ######### if __name__ == "__main__": main()