gecko/testing/mochitest/runtests.py.in

589 lines
20 KiB
Python

#
# ***** BEGIN LICENSE BLOCK *****
# Version: MPL 1.1/GPL 2.0/LGPL 2.1
#
# The contents of this file are subject to the Mozilla Public License Version
# 1.1 (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
# http://www.mozilla.org/MPL/
#
# Software distributed under the License is distributed on an "AS IS" basis,
# WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
# for the specific language governing rights and limitations under the
# License.
#
# The Original Code is mozilla.org code.
#
# The Initial Developer of the Original Code is
# Mozilla Foundation.
# Portions created by the Initial Developer are Copyright (C) 1998
# the Initial Developer. All Rights Reserved.
#
# Contributor(s):
# Robert Sayre <sayrer@gmail.com>
# Jeff Walden <jwalden+bmo@mit.edu>
#
# Alternatively, the contents of this file may be used under the terms of
# either the GNU General Public License Version 2 or later (the "GPL"), or
# the GNU Lesser General Public License Version 2.1 or later (the "LGPL"),
# in which case the provisions of the GPL or the LGPL are applicable instead
# of those above. If you wish to allow use of your version of this file only
# under the terms of either the GPL or the LGPL, and not to allow others to
# use your version of this file under the terms of the MPL, indicate your
# decision by deleting the provisions above and replace them with the notice
# and other provisions required by the GPL or the LGPL. If you do not delete
# the provisions above, a recipient may use your version of this file under
# the terms of any one of the MPL, the GPL or the LGPL.
#
# ***** END LICENSE BLOCK *****
"""
Runs the Mochitest test harness.
"""
from datetime import datetime
import logging
import optparse
import os
import os.path
import sys
import time
import shutil
from urllib import quote_plus as encodeURIComponent
import urllib2
import commands
import automation
# Path to the test script on the server
TEST_SERVER_HOST = "localhost:8888"
TEST_PATH = "/tests/"
CHROME_PATH = "/redirect.html";
A11Y_PATH = "/redirect-a11y.html"
TESTS_URL = "http://" + TEST_SERVER_HOST + TEST_PATH
CHROMETESTS_URL = "http://" + TEST_SERVER_HOST + CHROME_PATH
A11YTESTS_URL = "http://" + TEST_SERVER_HOST + A11Y_PATH
SERVER_SHUTDOWN_URL = "http://" + TEST_SERVER_HOST + "/server/shutdown"
# main browser chrome URL, same as browser.chromeURL pref
#ifdef MOZ_SUITE
BROWSER_CHROME_URL = "chrome://navigator/content/navigator.xul"
#else
BROWSER_CHROME_URL = "chrome://browser/content/browser.xul"
#endif
# Max time in seconds to wait for server startup before tests will fail -- if
# this seems big, it's mostly for debug machines where cold startup
# (particularly after a build) takes forever.
SERVER_STARTUP_TIMEOUT = 45
INFINITY = 1.0e3000
oldcwd = os.getcwd()
SCRIPT_DIRECTORY = os.path.abspath(os.path.realpath(os.path.dirname(sys.argv[0])))
os.chdir(SCRIPT_DIRECTORY)
PROFILE_DIRECTORY = os.path.abspath("./mochitesttestingprofile")
LEAK_REPORT_FILE = PROFILE_DIRECTORY + "/" + "leaks-report.log"
log = logging.getLogger()
# Map of debugging programs to information about them, like default arguments
# and whether or not they are interactive.
DEBUGGER_INFO = {
# gdb requires that you supply the '--args' flag in order to pass arguments
# after the executable name to the executable.
"gdb": {
"interactive": True,
"args": "-q --args"
},
# valgrind doesn't explain much about leaks unless you set the
# '--leak-check=full' flag.
"valgrind": {
"interactive": False,
"args": "--leak-check=full"
}
}
#######################
# COMMANDLINE OPTIONS #
#######################
class MochitestOptions(optparse.OptionParser):
"""Parses Mochitest commandline options."""
def __init__(self, **kwargs):
optparse.OptionParser.__init__(self, **kwargs)
defaults = {}
self.add_option("--close-when-done",
action = "store_true", dest = "closeWhenDone",
help = "close the application when tests are done running")
defaults["closeWhenDone"] = False
self.add_option("--appname",
action = "store", type = "string", dest = "app",
help = "absolute path to application, overriding default")
defaults["app"] = os.path.join(SCRIPT_DIRECTORY, automation.DEFAULT_APP)
self.add_option("--xre-path",
action = "store", type = "string", dest = "xrePath",
help = "absolute path to directory containing XRE (probably xulrunner)")
# we'll default this to the directory of app below
defaults["xrePath"] = None
self.add_option("--utility-path",
action = "store", type = "string", dest = "utilityPath",
help = "absolute path to directory containing utility programs (xpcshell, ssltunnel, certutil)")
defaults["utilityPath"] = automation.DIST_BIN
self.add_option("--symbols-path",
action = "store", type = "string", dest = "symbolsPath",
help = "absolute path to directory containing breakpad symbols")
defaults["symbolsPath"] = automation.SYMBOLS_PATH
self.add_option("--certificate-path",
action = "store", type = "string", dest = "certPath",
help = "absolute path to directory containing certificate store to use testing profile")
defaults["certPath"] = automation.CERTS_SRC_DIR
self.add_option("--log-file",
action = "store", type = "string",
dest = "logFile", metavar = "FILE",
help = "file to which logging occurs")
defaults["logFile"] = ""
self.add_option("--autorun",
action = "store_true", dest = "autorun",
help = "start running tests when the application starts")
defaults["autorun"] = False
LOG_LEVELS = ("DEBUG", "INFO", "WARNING", "ERROR", "FATAL")
LEVEL_STRING = ", ".join(LOG_LEVELS)
self.add_option("--console-level",
action = "store", type = "choice", dest = "consoleLevel",
choices = LOG_LEVELS, metavar = "LEVEL",
help = "one of %s to determine the level of console "
"logging" % LEVEL_STRING)
defaults["consoleLevel"] = None
self.add_option("--file-level",
action = "store", type = "choice", dest = "fileLevel",
choices = LOG_LEVELS, metavar = "LEVEL",
help = "one of %s to determine the level of file "
"logging if a file has been specified, defaulting "
"to INFO" % LEVEL_STRING)
defaults["fileLevel"] = "INFO"
self.add_option("--chrome",
action = "store_true", dest = "chrome",
help = "run chrome Mochitests")
defaults["chrome"] = False
self.add_option("--test-path",
action = "store", type = "string", dest = "testPath",
help = "start in the given directory's tests")
defaults["testPath"] = ""
self.add_option("--browser-chrome",
action = "store_true", dest = "browserChrome",
help = "run browser chrome Mochitests")
defaults["browserChrome"] = False
self.add_option("--a11y",
action = "store_true", dest = "a11y",
help = "run accessibility Mochitests");
self.add_option("--setenv",
action = "append", type = "string",
dest = "environment", metavar = "NAME=VALUE",
help = "sets the given variable in the application's "
"environment")
defaults["environment"] = []
self.add_option("--browser-arg",
action = "append", type = "string",
dest = "browserArgs", metavar = "ARG",
help = "provides an argument to the test application")
defaults["browserArgs"] = []
self.add_option("--leak-threshold",
action = "store", type = "int",
dest = "leakThreshold", metavar = "THRESHOLD",
help = "fail if the number of bytes leaked through "
"refcounted objects (or bytes in classes with "
"MOZ_COUNT_CTOR and MOZ_COUNT_DTOR) is greater "
"than the given number")
defaults["leakThreshold"] = 0
self.add_option("--fatal-assertions",
action = "store_true", dest = "fatalAssertions",
help = "abort testing whenever an assertion is hit "
"(requires a debug build to be effective)")
defaults["fatalAssertions"] = False
self.add_option("--extra-profile-file",
action = "append", dest = "extraProfileFiles",
help = "copy specified files/dirs to testing profile")
defaults["extraProfileFiles"] = []
self.add_option("--debugger",
action = "store", dest = "debugger",
help = "use the given debugger to launch the application")
self.add_option("--debugger-args",
action = "store", dest = "debuggerArgs",
help = "pass the given args to the debugger _before_ "
"the application on the command line")
self.add_option("--debugger-interactive",
action = "store_true", dest = "debuggerInteractive",
help = "prevents the test harness from redirecting stdout "
"and stderr for interactive debuggers")
# -h, --help are automatically handled by OptionParser
self.set_defaults(**defaults)
usage = """\
Usage instructions for runtests.py.
All arguments are optional.
If --chrome is specified, chrome tests will be run instead of web content tests.
If --browser-chrome is specified, browser-chrome tests will be run instead of web content tests.
See <http://mochikit.com/doc/html/MochiKit/Logging.html> for details on the logging levels."""
self.set_usage(usage)
#######################
# HTTP SERVER SUPPORT #
#######################
class MochitestServer:
"Web server used to serve Mochitests, for closer fidelity to the real web."
def __init__(self, options):
self._closeWhenDone = options.closeWhenDone
self._utilityPath = options.utilityPath
self._xrePath = options.xrePath
def start(self):
"Run the Mochitest server, returning the process ID of the server."
env = dict(os.environ)
if automation.UNIXISH:
env["LD_LIBRARY_PATH"] = self._xrePath
env["MOZILLA_FIVE_HOME"] = self._xrePath
env["XPCOM_DEBUG_BREAK"] = "warn"
if automation.IS_MAC:
env["DYLD_LIBRARY_PATH"] = self._xrePath
elif automation.IS_WIN32:
env["PATH"] = env["PATH"] + ";" + self._xrePath
args = ["-g", self._xrePath,
"-v", "170",
"-f", "./" + "httpd.js",
"-f", "./" + "server.js"]
xpcshell = os.path.join(self._utilityPath,
"xpcshell" + automation.BIN_SUFFIX)
self._process = automation.Process([xpcshell] + args, env = env)
pid = self._process.pid
if pid < 0:
print "Error starting server."
sys.exit(2)
log.info("Server pid: %d", pid)
def ensureReady(self, timeout):
assert timeout >= 0
aliveFile = os.path.join(PROFILE_DIRECTORY, "server_alive.txt")
i = 0
while i < timeout:
if os.path.exists(aliveFile):
break
time.sleep(1)
i += 1
else:
print "Timed out while waiting for server startup."
self.stop()
sys.exit(1)
def stop(self):
try:
c = urllib2.urlopen(SERVER_SHUTDOWN_URL)
c.read()
c.close()
self._process.wait()
except:
self._process.kill()
def getFullPath(path):
"Get an absolute path relative to oldcwd."
return os.path.normpath(os.path.join(oldcwd, os.path.expanduser(path)))
def searchPath(path):
"Go one step beyond getFullPath and try the various folders in PATH"
# Try looking in the current working directory first.
newpath = getFullPath(path)
if os.path.exists(newpath):
return newpath
# At this point we have to fail if a directory was given (to prevent cases
# like './gdb' from matching '/usr/bin/./gdb').
if not os.path.dirname(path):
for dir in os.environ['PATH'].split(os.pathsep):
newpath = os.path.join(dir, path)
if os.path.exists(newpath):
return newpath
return None
#################
# MAIN FUNCTION #
#################
def main():
parser = MochitestOptions()
options, args = parser.parse_args()
options.app = getFullPath(options.app)
if not os.path.exists(options.app):
msg = """\
Error: Path %(app)s doesn't exist.
Are you executing $objdir/_tests/testing/mochitest/runtests.py?"""
print msg % {"app": options.app}
sys.exit(1)
# default xrePath to the app path if not provided
if options.xrePath is None:
options.xrePath = os.path.dirname(options.app)
else:
# allow relative paths
options.xrePath = getFullPath(options.xrePath)
options.utilityPath = getFullPath(options.utilityPath)
options.certPath = getFullPath(options.certPath)
debuggerInfo = None
if options.debugger:
debuggerPath = searchPath(options.debugger)
if not debuggerPath:
print "Error: Path %s doesn't exist." % options.debugger
sys.exit(1)
debuggerName = os.path.basename(debuggerPath).lower()
def getDebuggerInfo(type, default):
if debuggerName in DEBUGGER_INFO and type in DEBUGGER_INFO[debuggerName]:
return DEBUGGER_INFO[debuggerName][type]
return default
debuggerInfo = {
"path": debuggerPath,
"interactive" : getDebuggerInfo("interactive", False),
"args": getDebuggerInfo("args", "").split()
}
if options.debuggerArgs:
debuggerInfo["args"] = options.debuggerArgs.split()
if options.debuggerInteractive:
debuggerInfo["interactive"] = options.debuggerInteractive
# browser environment
browserEnv = dict(os.environ)
# These variables are necessary for correct application startup; change
# via the commandline at your own risk.
browserEnv["NO_EM_RESTART"] = "1"
browserEnv["XPCOM_DEBUG_BREAK"] = "warn"
appDir = os.path.dirname(options.app)
if automation.UNIXISH:
browserEnv["LD_LIBRARY_PATH"] = appDir
browserEnv["MOZILLA_FIVE_HOME"] = appDir
browserEnv["GNOME_DISABLE_CRASH_DIALOG"] = "1"
for v in options.environment:
ix = v.find("=")
if ix <= 0:
print "Error: syntax error in --setenv=" + v
sys.exit(1)
browserEnv[v[:ix]] = v[ix + 1:]
automation.initializeProfile(PROFILE_DIRECTORY)
manifest = addChromeToProfile(options)
copyExtraFilesToProfile(options)
server = MochitestServer(options)
server.start()
# If we're lucky, the server has fully started by now, and all paths are
# ready, etc. However, xpcshell cold start times suck, at least for debug
# builds. We'll try to connect to the server for awhile, and if we fail,
# we'll try to kill the server and exit with an error.
server.ensureReady(SERVER_STARTUP_TIMEOUT)
# URL parameters to test URL:
#
# autorun -- kick off tests automatically
# closeWhenDone -- runs quit.js after tests
# logFile -- logs test run to an absolute path
#
# consoleLevel, fileLevel: set the logging level of the console and
# file logs, if activated.
# <http://mochikit.com/doc/html/MochiKit/Logging.html>
testURL = TESTS_URL + options.testPath
urlOpts = []
if options.chrome:
testURL = CHROMETESTS_URL
if options.testPath:
urlOpts.append("testPath=" + encodeURIComponent(options.testPath))
elif options.a11y:
testURL = A11YTESTS_URL
if options.testPath:
urlOpts.append("testPath=" + encodeURIComponent(options.testPath))
elif options.browserChrome:
testURL = "about:blank"
# allow relative paths for logFile
if options.logFile:
options.logFile = getFullPath(options.logFile)
if options.browserChrome:
makeTestConfig(options)
else:
if options.autorun:
urlOpts.append("autorun=1")
if options.closeWhenDone:
urlOpts.append("closeWhenDone=1")
if options.logFile:
urlOpts.append("logFile=" + encodeURIComponent(options.logFile))
urlOpts.append("fileLevel=" + encodeURIComponent(options.fileLevel))
if options.consoleLevel:
urlOpts.append("consoleLevel=" + encodeURIComponent(options.consoleLevel))
if len(urlOpts) > 0:
testURL += "?" + "&".join(urlOpts)
browserEnv["XPCOM_MEM_BLOAT_LOG"] = LEAK_REPORT_FILE
if options.fatalAssertions:
browserEnv["XPCOM_DEBUG_BREAK"] = "stack-and-abort"
status = automation.runApp(testURL, browserEnv, options.app,
PROFILE_DIRECTORY, options.browserArgs,
runSSLTunnel = True,
utilityPath = options.utilityPath,
xrePath = options.xrePath,
certPath=options.certPath,
debuggerInfo=debuggerInfo,
symbolsPath=options.symbolsPath)
# Server's no longer needed, and perhaps more importantly, anything it might
# spew to console shouldn't disrupt the leak information table we print next.
server.stop()
automation.processLeakLog(LEAK_REPORT_FILE, options.leakThreshold)
# delete the profile and manifest
os.remove(manifest)
# hanging due to non-halting threads is no fun; assume we hit the errors we
# were going to hit already and exit with a success code
sys.exit(0)
#######################
# CONFIGURATION SETUP #
#######################
def makeTestConfig(options):
"Creates a test configuration file for customizing test execution."
def boolString(b):
if b:
return "true"
return "false"
logFile = options.logFile.replace("\\", "\\\\")
testPath = options.testPath.replace("\\", "\\\\")
content = """\
({
autoRun: %(autorun)s,
closeWhenDone: %(closeWhenDone)s,
logPath: "%(logPath)s",
testPath: "%(testPath)s"
})""" % {"autorun": boolString(options.autorun),
"closeWhenDone": boolString(options.closeWhenDone),
"logPath": logFile,
"testPath": testPath}
config = open(os.path.join(PROFILE_DIRECTORY, "testConfig.js"), "w")
config.write(content)
config.close()
def addChromeToProfile(options):
"Adds MochiKit chrome tests to the profile."
chromedir = os.path.join(PROFILE_DIRECTORY, "chrome")
os.mkdir(chromedir)
chrome = []
part = """
@namespace url("http://www.mozilla.org/keymaster/gatekeeper/there.is.only.xul"); /* set default namespace to XUL */
toolbar,
toolbarpalette {
background-color: rgb(235, 235, 235) !important;
}
toolbar#nav-bar {
background-image: none !important;
}
"""
chrome.append(part)
# write userChrome.css
chromeFile = open(os.path.join(PROFILE_DIRECTORY, "userChrome.css"), "a")
chromeFile.write("".join(chrome))
chromeFile.close()
# register our chrome dir
chrometestDir = os.path.abspath(".") + "/"
if automation.IS_WIN32:
chrometestDir = "file:///" + chrometestDir.replace("\\", "/")
(path, leaf) = os.path.split(options.app)
manifest = os.path.join(path, "chrome", "mochikit.manifest")
manifestFile = open(manifest, "w")
manifestFile.write("content mochikit " + chrometestDir + " contentaccessible=yes\n")
if options.browserChrome:
overlayLine = "overlay " + BROWSER_CHROME_URL + " " \
"chrome://mochikit/content/browser-test-overlay.xul\n"
manifestFile.write(overlayLine)
manifestFile.close()
return manifest
def copyExtraFilesToProfile(options):
"Copy extra files or dirs specified on the command line to the testing profile."
for f in options.extraProfileFiles:
abspath = getFullPath(f)
dest = os.path.join(PROFILE_DIRECTORY, os.path.basename(abspath))
if os.path.isdir(abspath):
shutil.copytree(abspath, dest)
else:
shutil.copy(abspath, dest)
#########
# DO IT #
#########
if __name__ == "__main__":
main()