Bug 493748 Refactor xpcshell test harness to enable it to work for remote devices r=ted (comment 14) p=jmaher Landing on a CLOSED TREE

This commit is contained in:
Joel Maher 2010-03-12 14:57:29 -08:00
parent 25c0579936
commit e2fbedec3f
2 changed files with 296 additions and 142 deletions

View File

@ -46,6 +46,7 @@ __all__ = [
"processLeakLog",
"getDebuggerInfo",
"DEBUGGER_INFO",
"replaceBackSlashes",
]
# Map of debugging programs to information about them, like default arguments
@ -323,3 +324,6 @@ def processLeakLog(leakLogFile, leakThreshold = 0):
processType = m.group(1)
processPID = m.group(2)
processSingleLeakFile(thisFile, processPID, processType, leakThreshold)
def replaceBackSlashes(input):
return input.replace('\\', '/')

View File

@ -52,14 +52,16 @@ class XPCShellTests(object):
oldcwd = os.getcwd()
def __init__(self):
# Init logging
""" Init logging """
handler = logging.StreamHandler(sys.stdout)
self.log.setLevel(logging.INFO)
self.log.addHandler(handler)
def readManifest(self, manifest):
"""Given a manifest file containing a list of test directories,
return a list of absolute paths to the directories contained within."""
"""
Given a manifest file containing a list of test directories,
return a list of absolute paths to the directories contained within.
"""
manifestdir = os.path.dirname(manifest)
testdirs = []
try:
@ -74,6 +76,240 @@ class XPCShellTests(object):
pass # just eat exceptions
return testdirs
def setAbsPath(self):
"""
Set the absolute path for xpcshell, httpdjspath and xrepath.
These 3 variables depend on input from the command line and we need to allow for absolute paths.
This function is overloaded for a remote solution as os.path* won't work remotely.
"""
self.testharnessdir = os.path.dirname(os.path.abspath(__file__))
self.xpcshell = os.path.abspath(self.xpcshell)
# we assume that httpd.js lives in components/ relative to xpcshell
self.httpdJSPath = os.path.join(os.path.dirname(self.xpcshell), 'components', 'httpd.js')
self.httpdJSPath = replaceBackSlashes(self.httpdJSPath)
if self.xrePath is None:
self.xrePath = os.path.dirname(self.xpcshell)
else:
self.xrePath = os.path.abspath(self.xrePath)
def buildEnvironment(self):
"""
Create and returns a dictionary of self.env to include all the appropriate env variables and values.
On a remote system, we overload this to set different values and are missing things like os.environ and PATH.
"""
self.env = dict(os.environ)
# Make assertions fatal
self.env["XPCOM_DEBUG_BREAK"] = "stack-and-abort"
# Don't launch the crash reporter client
self.env["MOZ_CRASHREPORTER_NO_REPORT"] = "1"
if sys.platform == 'win32':
self.env["PATH"] = self.env["PATH"] + ";" + self.xrePath
elif sys.platform in ('os2emx', 'os2knix'):
os.environ["BEGINLIBPATH"] = self.xrePath + ";" + self.env["BEGINLIBPATH"]
os.environ["LIBPATHSTRICT"] = "T"
elif sys.platform == 'osx':
self.env["DYLD_LIBRARY_PATH"] = self.xrePath
else: # unix or linux?
self.env["LD_LIBRARY_PATH"] = self.xrePath
return self.env
def buildXpcsRunArgs(self):
"""
Add arguments to run the test or make it interactive.
"""
if self.interactive:
self.xpcsRunArgs = [
'-e', 'print("To start the test, type |_execute_test();|.");',
'-i']
else:
self.xpcsRunArgs = ['-e', '_execute_test();']
def getPipes(self):
"""
Determine the value of the stdout and stderr for the test.
Return value is a list (pStdout, pStderr).
"""
if self.interactive:
pStdout = None
pStderr = None
else:
if (self.debuggerInfo and self.debuggerInfo["interactive"]):
pStdout = None
pStderr = None
else:
if sys.platform == 'os2emx':
pStdout = None
else:
pStdout = PIPE
pStderr = STDOUT
return pStdout, pStderr
def buildXpcsCmd(self, testdir):
"""
Load the root head.js file as the first file in our test path, before other head, test, and tail files.
On a remote system, we overload this to add additional command line arguments, so this gets overloaded.
"""
self.xpcsCmd = [self.xpcshell, '-g', self.xrePath, '-j', '-s'] + \
['-e', 'const _HTTPD_JS_PATH = "%s";' % self.httpdJSPath,
'-f', os.path.join(self.testharnessdir, 'head.js')]
if self.debuggerInfo:
self.xpcsCmd = [self.debuggerInfo["path"]] + self.debuggerInfo["args"] + self.xpcsCmd
def buildTestPath(self):
"""
If we specifiy a testpath, set the self.testPath variable to be the given directory or file.
|testPath| will be the optional path only, or |None|.
|singleFile| will be the optional test only, or |None|.
"""
self.singleFile = None
if self.testPath:
if self.testPath.endswith('.js'):
# Split into path and file.
if self.testPath.find('/') == -1:
# Test only.
self.singleFile = self.testPath
self.testPath = None
else:
# Both path and test.
# Reuse |testPath| temporarily.
self.testPath = self.testPath.rsplit('/', 1)
self.singleFile = self.testPath[1]
self.testPath = self.testPath[0]
else:
# Path only.
# Simply remove optional ending separator.
self.testPath = self.testPath.rstrip("/")
def getHeadFiles(self, testdir):
"""
Get the list of head files for a given test directory.
On a remote system, this is overloaded to list files in a remote directory structure.
"""
return [f for f in sorted(glob(os.path.join(testdir, "head_*.js"))) if os.path.isfile(f)]
def getTailFiles(self, testdir):
"""
Get the list of tail files for a given test directory.
Tails are executed in the reverse order, to "match" heads order,
as in "h1-h2-h3 then t3-t2-t1".
On a remote system, this is overloaded to list files in a remote directory structure.
"""
return [f for f in reversed(sorted(glob(os.path.join(testdir, "tail_*.js")))) if os.path.isfile(f)]
def getTestFiles(self, testdir):
"""
Ff a single test file was specified, we only want to execute that test,
otherwise return a list of all tests in a directory
On a remote system, this is overloaded to find files in the remote directory structure.
"""
testfiles = sorted(glob(os.path.join(testdir, "test_*.js")))
if self.singleFile:
if singleFile in [os.path.basename(x) for x in testfiles]:
testfiles = [os.path.join(testdir, singleFile)]
else: # not in this dir? skip it
return None
return testfiles
def setupProfileDir(self):
"""
Create a temporary folder for the profile and set appropriate environment variables.
On a remote system, we overload this to use a remote path structure.
"""
profileDir = mkdtemp()
self.env["XPCSHELL_TEST_PROFILE_DIR"] = profileDir
return profileDir
def setupLeakLogging(self):
"""
Enable leaks (only) detection to its own log file and set environment variables.
On a remote system, we overload this to use a remote filename and path structure
"""
filename = "runxpcshelltests_leaks.log"
leakLogFile = os.path.join(self.profileDir, filename)
self.env["XPCOM_MEM_LEAK_LOG"] = leakLogFile
return leakLogFile
def launchProcess(self, cmd, stdout, stderr, env, cwd):
"""
Simple wrapper to launch a process.
On a remote system, this is more complex and we need to overload this function.
"""
proc = Popen(cmd, stdout=stdout, stderr=stderr,
env=env, cwd=cwd)
return proc
def communicate(self, proc):
"""
Simple wrapper to communicate with a process.
On a remote system, this is overloaded to handle remote process communication.
"""
return proc.communicate()
def removeDir(self, dirname):
"""
Simple wrapper to remove (recursively) a given directory.
On a remote system, we need to overload this to work on the remote filesystem.
"""
shutil.rmtree(dirname)
def verifyDirPath(self, dirname):
"""
Simple wrapper to get the absolute path for a given directory name.
On a remote system, we need to overload this to work on the remote filesystem.
"""
return os.path.abspath(dirname)
def getReturnCode(self, proc):
"""
Simple wrapper to get the return code for a given process.
On a remote system we overload this to work with the remote process management.
"""
return proc.returncode
def createLogFile(self, test, stdout):
"""
For a given test and stdout buffer, create a log file. also log any found leaks.
On a remote system we have to fix the test name since it can contain directories.
"""
try:
f = open(test + ".log", "w")
f.write(stdout)
if os.path.exists(self.leakLogFile):
leaks = open(self.leakLogFile, "r")
f.write(leaks.read())
leaks.close()
finally:
if f:
f.close()
def buildCmdHead(self, headfiles, tailfiles, xpcscmd):
"""
Build the command line arguments for the head and tail files,
along with the address of the webserver which some tests require.
On a remote system, this is overloaded to resolve quoting issues over a secondary command line.
"""
cmdH = ", ".join(['"' + replaceBackSlashes(f) + '"'
for f in headfiles])
cmdT = ", ".join(['"' + replaceBackSlashes(f) + '"'
for f in tailfiles])
return xpcscmd + \
['-e', 'const _SERVER_ADDR = "localhost"',
'-e', 'const _HEAD_FILES = [%s];' % cmdH,
'-e', 'const _TAIL_FILES = [%s];' % cmdT]
def runTests(self, xpcshell, xrePath=None, symbolsPath=None,
manifest=None, testdirs=[], testPath=None,
interactive=False, logfiles=True,
@ -97,6 +333,16 @@ class XPCShellTests(object):
that will be used to launch xpcshell.
"""
self.xpcshell = xpcshell
self.xrePath = xrePath
self.symbolsPath = symbolsPath
self.manifest = manifest
self.testdirs = testdirs
self.testPath = testPath
self.interactive = interactive
self.logfiles = logfiles
self.debuggerInfo = debuggerInfo
if not testdirs and not manifest:
# nothing to test!
print >>sys.stderr, "Error: No test dirs or test manifest specified!"
@ -105,178 +351,78 @@ class XPCShellTests(object):
passCount = 0
failCount = 0
testharnessdir = os.path.dirname(os.path.abspath(__file__))
xpcshell = os.path.abspath(xpcshell)
# we assume that httpd.js lives in components/ relative to xpcshell
httpdJSPath = os.path.join(os.path.dirname(xpcshell), "components", "httpd.js").replace("\\", "/");
env = dict(os.environ)
# Make assertions fatal
env["XPCOM_DEBUG_BREAK"] = "stack-and-abort"
# Don't launch the crash reporter client
env["MOZ_CRASHREPORTER_NO_REPORT"] = "1"
if xrePath is None:
xrePath = os.path.dirname(xpcshell)
else:
xrePath = os.path.abspath(xrePath)
if sys.platform == 'win32':
env["PATH"] = env["PATH"] + ";" + xrePath
elif sys.platform in ('os2emx', 'os2knix'):
os.environ["BEGINLIBPATH"] = xrePath + ";" + env["BEGINLIBPATH"]
os.environ["LIBPATHSTRICT"] = "T"
elif sys.platform == 'osx':
env["DYLD_LIBRARY_PATH"] = xrePath
else: # unix or linux?
env["LD_LIBRARY_PATH"] = xrePath
# xpcsRunArgs: <head.js> function to call to run the test.
# pStdout, pStderr: Parameter values for later |Popen()| call.
if interactive:
xpcsRunArgs = [
'-e', 'print("To start the test, type |_execute_test();|.");',
'-i']
pStdout = None
pStderr = None
else:
xpcsRunArgs = ['-e', '_execute_test();']
if (debuggerInfo and debuggerInfo["interactive"]):
pStdout = None
pStderr = None
else:
if sys.platform == 'os2emx':
pStdout = None
else:
pStdout = PIPE
pStderr = STDOUT
# <head.js> has to be loaded by xpchell: it can't load itself.
xpcsCmd = [xpcshell, '-g', xrePath, '-j', '-s'] + \
['-e', 'const _HTTPD_JS_PATH = "%s";' % httpdJSPath,
'-f', os.path.join(testharnessdir, 'head.js')]
if debuggerInfo:
xpcsCmd = [debuggerInfo["path"]] + debuggerInfo["args"] + xpcsCmd
# |testPath| will be the optional path only, or |None|.
# |singleFile| will be the optional test only, or |None|.
singleFile = None
if testPath:
if testPath.endswith('.js'):
# Split into path and file.
if testPath.find('/') == -1:
# Test only.
singleFile = testPath
testPath = None
else:
# Both path and test.
# Reuse |testPath| temporarily.
testPath = testPath.rsplit('/', 1)
singleFile = testPath[1]
testPath = testPath[0]
else:
# Path only.
# Simply remove optional ending separator.
testPath = testPath.rstrip("/")
self.setAbsPath()
self.buildXpcsRunArgs()
self.buildEnvironment()
pStdout, pStderr = self.getPipes()
# Override testdirs.
if manifest is not None:
testdirs = self.readManifest(os.path.abspath(manifest))
self.buildTestPath()
# Process each test directory individually.
for testdir in testdirs:
self.buildXpcsCmd(testdir)
if testPath and not testdir.endswith(testPath):
continue
testdir = os.path.abspath(testdir)
# get the list of head and tail files from the directory
testHeadFiles = []
for f in sorted(glob(os.path.join(testdir, "head_*.js"))):
if os.path.isfile(f):
testHeadFiles += [f]
testTailFiles = []
# Tails are executed in the reverse order, to "match" heads order,
# as in "h1-h2-h3 then t3-t2-t1".
for f in reversed(sorted(glob(os.path.join(testdir, "tail_*.js")))):
if os.path.isfile(f):
testTailFiles += [f]
testHeadFiles = self.getHeadFiles(testdir)
testTailFiles = self.getTailFiles(testdir)
# if a single test file was specified, we only want to execute that test
testfiles = sorted(glob(os.path.join(testdir, "test_*.js")))
if singleFile:
if singleFile in [os.path.basename(x) for x in testfiles]:
testfiles = [os.path.join(testdir, singleFile)]
else: # not in this dir? skip it
continue
testfiles = self.getTestFiles(testdir)
if testfiles == None:
continue
cmdH = ", ".join(['"' + f.replace('\\', '/') + '"'
for f in testHeadFiles])
cmdT = ", ".join(['"' + f.replace('\\', '/') + '"'
for f in testTailFiles])
cmdH = xpcsCmd + \
['-e', 'const _SERVER_ADDR = "localhost";'] + \
['-e', 'const _HEAD_FILES = [%s];' % cmdH] + \
['-e', 'const _TAIL_FILES = [%s];' % cmdT]
cmdH = self.buildCmdHead(testHeadFiles, testTailFiles, self.xpcsCmd)
# Now execute each test individually.
for test in testfiles:
# create a temp dir that the JS harness can stick a profile in
self.profileDir = self.setupProfileDir()
self.leakLogFile = self.setupLeakLogging()
# The test file will have to be loaded after the head files.
cmdT = ['-e', 'const _TEST_FILE = ["%s"];' %
os.path.join(testdir, test).replace('\\', '/')]
replaceBackSlashes(os.path.join(testdir, test))]
# create a temp dir that the JS harness can stick a profile in
profileDir = None
try:
profileDir = mkdtemp()
env["XPCSHELL_TEST_PROFILE_DIR"] = profileDir
# Enable leaks (only) detection to its own log file.
leakLogFile = os.path.join(profileDir, "runxpcshelltests_leaks.log")
env["XPCOM_MEM_LEAK_LOG"] = leakLogFile
proc = Popen(cmdH + cmdT + xpcsRunArgs,
stdout=pStdout, stderr=pStderr, env=env, cwd=testdir)
proc = self.launchProcess(cmdH + cmdT + self.xpcsRunArgs,
stdout=pStdout, stderr=pStderr, env=self.env, cwd=testdir)
# allow user to kill hung subprocess with SIGINT w/o killing this script
# - don't move this line above Popen, or child will inherit the SIG_IGN
signal.signal(signal.SIGINT, signal.SIG_IGN)
# |stderr == None| as |pStderr| was either |None| or redirected to |stdout|.
stdout, stderr = proc.communicate()
stdout, stderr = self.communicate(proc)
signal.signal(signal.SIGINT, signal.SIG_DFL)
if interactive:
# Not sure what else to do here...
return True
if proc.returncode != 0 or (stdout and re.search("^TEST-UNEXPECTED-FAIL", stdout, re.MULTILINE)):
if (self.getReturnCode(proc) != 0) or (stdout and re.search("^TEST-UNEXPECTED-FAIL", stdout, re.MULTILINE)):
print """TEST-UNEXPECTED-FAIL | %s | test failed (with xpcshell return code: %d), see following log:
>>>>>>>
%s
<<<<<<<""" % (test, proc.returncode, stdout)
<<<<<<<""" % (test, self.getReturnCode(proc), stdout)
failCount += 1
else:
print "TEST-PASS | %s | test passed" % test
passCount += 1
checkForCrashes(testdir, symbolsPath, testName=test)
dumpLeakLog(leakLogFile, True)
checkForCrashes(testdir, self.symbolsPath, testName=test)
dumpLeakLog(self.leakLogFile, True)
if logfiles and stdout:
try:
f = open(test + ".log", "w")
f.write(stdout)
if os.path.exists(leakLogFile):
leaks = open(leakLogFile, "r")
f.write(leaks.read())
leaks.close()
finally:
if f:
f.close()
if self.logfiles and stdout:
self.createLogFile(test, stdout)
finally:
if profileDir:
shutil.rmtree(profileDir)
if self.profileDir:
self.removeDir(self.profileDir)
if passCount == 0 and failCount == 0:
print "TEST-UNEXPECTED-FAIL | runxpcshelltests.py | No tests run. Did you pass an invalid --test-path?"
@ -288,26 +434,30 @@ INFO | Failed: %d""" % (passCount, failCount)
return failCount == 0
def main():
"""Process command line arguments and call runTests() to do the real work."""
parser = OptionParser()
class XPCShellOptions(OptionParser):
def __init__(self):
"""Process command line arguments and call runTests() to do the real work."""
OptionParser.__init__(self)
addCommonOptions(parser)
parser.add_option("--interactive",
addCommonOptions(self)
self.add_option("--interactive",
action="store_true", dest="interactive", default=False,
help="don't automatically run tests, drop to an xpcshell prompt")
parser.add_option("--logfiles",
self.add_option("--logfiles",
action="store_true", dest="logfiles", default=True,
help="create log files (default, only used to override --no-logfiles)")
parser.add_option("--manifest",
self.add_option("--manifest",
type="string", dest="manifest", default=None,
help="Manifest of test directories to use")
parser.add_option("--no-logfiles",
self.add_option("--no-logfiles",
action="store_false", dest="logfiles",
help="don't create log files")
parser.add_option("--test-path",
self.add_option("--test-path",
type="string", dest="testPath", default=None,
help="single path and/or test filename to test")
def main():
parser = XPCShellOptions()
options, args = parser.parse_args()
if len(args) < 2 and options.manifest is None or \