wine/gecko
0
0
Fork 0
mirror of https://gitlab.winehq.org/wine/wine-gecko.git synced 2024-09-13 09:24:08 -07:00
Code Issues Packages Projects Releases Wiki Activity

Bug 1179981 - A new test harness for robocop; r=jmaher

Browse Source
This commit is contained in:
Geoff Brown 2015-07-09 12:30:01 -06:00
parent eacd6b8c75
commit 96d8502fb9
12 changed files with 620 additions and 441 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

29
build/mobile/remoteautomation.py
View File

@ -2,6 +2,7 @@
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
import datetime
import glob
import time
import re
@ -137,8 +138,11 @@ class RemoteAutomation(Automation):
if self._devicemanager.fileExists(traces):
try:
t = self._devicemanager.pullFile(traces)
print "Contents of %s:" % traces
print t
if t:
stripped = t.strip()
if len(stripped) > 0:
print "Contents of %s:" % traces
print t
# Once reported, delete traces
self.deleteANRs()
except DMError:
@ -281,7 +285,6 @@ class RemoteAutomation(Automation):
self.procName = cmd[0].split('/')[-1]
if cmd[0] == 'am' and cmd[1] in RemoteAutomation._specialAmCommands:
self.procName = app
print "Robocop process name: "+self.procName
# Setting timeout at 1 hour since on a remote device this takes much longer.
# Temporarily increased to 75 minutes because no more chunks can be created.
@ -360,19 +363,22 @@ class RemoteAutomation(Automation):
def wait(self, timeout = None, noOutputTimeout = None):
timer = 0
noOutputTimer = 0
interval = 20
interval = 10
if timeout == None:
timeout = self.timeout
status = 0
while (self.dm.getTopActivity() == self.procName):
# retrieve log updates every 60 seconds
if timer % 60 == 0:
top = self.procName
slowLog = False
while (top == self.procName):
# Get log updates on each interval, but if it is taking
# too long, only do it every 60 seconds
if (not slowLog) or (timer % 60 == 0):
startRead = datetime.datetime.now()
messages = self.read_stdout()
if (datetime.datetime.now() - startRead) > datetime.timedelta(seconds=5):
slowLog = True
if messages:
noOutputTimer = 0
time.sleep(interval)
timer += interval
noOutputTimer += interval
@ -382,10 +388,9 @@ class RemoteAutomation(Automation):
if (noOutputTimeout and noOutputTimer > noOutputTimeout):
status = 2
break
top = self.dm.getTopActivity()
# Flush anything added to stdout during the sleep
self.read_stdout()
return status
def kill(self, stagedShutdown = False):

2
mobile/android/tests/browser/robocop/BaseTest.java
View File

@ -134,7 +134,7 @@ abstract class BaseTest extends BaseRobocopTest {
} catch (Throwable t) {
// save screenshot -- written to /mnt/sdcard/Robotium-Screenshots
// as <filename>.jpg
mSolo.takeScreenshot("robocop-screenshot");
mSolo.takeScreenshot("robocop-screenshot-"+getClass().getName());
if (mAsserter != null) {
mAsserter.dumpLog("Exception caught during test!", t);
mAsserter.ok(false, "Exception caught", t.toString());

2
mobile/android/tests/browser/robocop/UITest.java
View File

@ -65,7 +65,7 @@ abstract class UITest extends BaseRobocopTest
} catch (Throwable t) {
// save screenshot -- written to /mnt/sdcard/Robotium-Screenshots
// as <filename>.jpg
mSolo.takeScreenshot("robocop-screenshot");
mSolo.takeScreenshot("robocop-screenshot-"+getClass().getName());
if (mAsserter != null) {
mAsserter.dumpLog("Exception caught during test!", t);
mAsserter.ok(false, "Exception caught", t.toString());

3
testing/config/mozharness/android_arm_4_3_config.py
View File

@ -28,7 +28,7 @@ config = {
],
},
"robocop": {
"run_filename": "runtestsremote.py",
"run_filename": "runrobocop.py",
"testsdir": "mochitest",
"options": ["--dm_trans=adb", "--app=%(app)s", "--remote-webserver=%(remote_webserver)s",
"--xre-path=%(xre_path)s", "--utility-path=%(utility_path)s",
@ -37,7 +37,6 @@ config = {
"--quiet", "--log-raw=%(raw_log_file)s",
"--total-chunks=4",
"--robocop-apk=../../robocop.apk",
"--robocop-ids=fennec_ids.txt",
"--robocop-ini=robocop.ini",
],
},

3
testing/config/mozharness/android_arm_config.py
View File

@ -30,7 +30,7 @@ config = {
],
},
"robocop": {
"run_filename": "runtestsremote.py",
"run_filename": "runrobocop.py",
"testsdir": "mochitest",
"options": ["--dm_trans=sut", "--app=%(app)s", "--remote-webserver=%(remote_webserver)s",
"--xre-path=%(xre_path)s", "--utility-path=%(utility_path)s",
@ -40,7 +40,6 @@ config = {
"--quiet", "--log-raw=%(raw_log_file)s",
"--total-chunks=4",
"--robocop-apk=../../robocop.apk",
"--robocop-ids=fennec_ids.txt",
"--robocop-ini=robocop.ini",
],
},

2
testing/config/mozharness/android_panda_config.py
View File

@ -115,7 +115,7 @@ config = {
"--symbols-path=%(symbols_path)s",
"--robocop-ini=mochitest/robocop.ini"
],
"run_filename": "runtestsremote.py",
"run_filename": "runrobocop.py",
"testsdir": "mochitest"
},
"xpcshell": {

22
testing/mochitest/mach_commands.py
View File

@ -374,6 +374,26 @@ class MochitestRunner(MozbuildObject):
return runtestsremote.run_test_harness(options)
def run_robocop_test(self, context, tests, suite=None, **kwargs):
host_ret = verify_host_bin()
if host_ret != 0:
return host_ret
import imp
path = os.path.join(self.mochitest_dir, 'runrobocop.py')
with open(path, 'r') as fh:
imp.load_module('runrobocop', fh, path,
('.py', 'r', imp.PY_SOURCE))
import runrobocop
options = Namespace(**kwargs)
from manifestparser import TestManifest
manifest = TestManifest()
manifest.tests.extend(tests)
options.manifestFile = manifest
return runrobocop.run_test_harness(options)
# parser
@ -604,7 +624,7 @@ class RobocopCommands(MachCommandBase):
flavor='instrumentation', subsuite='robocop'))
mochitest = self._spawn(MochitestRunner)
return mochitest.run_android_test(self._mach_context, tests, 'robocop', **kwargs)
return mochitest.run_robocop_test(self._mach_context, tests, 'robocop', **kwargs)
def REMOVED(cls):

13
testing/mochitest/mochitest_options.py
View File

@ -1016,12 +1016,6 @@ class AndroidArguments(ArgumentContainer):
"default": "",
"help": "name of the Robocop APK to use for ADB test running",
}],
[["--robocop-ids"],
{"dest": "robocopIds",
"default": "",
"help": "name of the file containing the view ID map \
(fennec_ids.txt)",
}],
[["--remoteTestRoot"],
{"dest": "remoteTestRoot",
"default": None,
@ -1130,13 +1124,6 @@ class AndroidArguments(ArgumentContainer):
options.robocopApk)
options.robocopApk = os.path.abspath(options.robocopApk)
if options.robocopIds != "":
if not os.path.exists(options.robocopIds):
parser.error(
"Unable to find specified robocop IDs file '%s'" %
options.robocopIds)
options.robocopIds = os.path.abspath(options.robocopIds)
# allow us to keep original application around for cleanup while
# running robocop via 'am'
options.remoteappname = options.app

1
testing/mochitest/moz.build
View File

@ -63,6 +63,7 @@ TEST_HARNESS_FILES.testing.mochitest += [
'nested_setup.js',
'pywebsocket_wrapper.py',
'redirect.html',
'runrobocop.py',
'runtests.py',
'runtestsb2g.py',
'runtestsremote.py',

554
testing/mochitest/runrobocop.py Normal file
View File

@ -0,0 +1,554 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
import json
import os
import shutil
import sys
import tempfile
import traceback
sys.path.insert(
0, os.path.abspath(
os.path.realpath(
os.path.dirname(__file__))))
from automation import Automation
from remoteautomation import RemoteAutomation, fennecLogcatFilters
from runtests import Mochitest, MessageLogger
from mochitest_options import MochitestArgumentParser
from manifestparser import TestManifest
from manifestparser.filters import chunk_by_slice
import devicemanager
import mozinfo
SCRIPT_DIR = os.path.abspath(os.path.realpath(os.path.dirname(__file__)))
class RobocopTestRunner(Mochitest):
"""
A test harness for Robocop. Robocop tests are UI tests for Firefox for Android,
based on the Robotium test framework. This harness leverages some functionality
from mochitest, for convenience.
"""
auto = None
dm = None
# Some robocop tests run for >60 seconds without generating any output.
NO_OUTPUT_TIMEOUT = 180
def __init__(self, automation, devmgr, options):
"""
Simple one-time initialization.
"""
Mochitest.__init__(self, options)
self.auto = automation
self.dm = devmgr
self.dm.default_timeout = 320
self.options = options
self.options.logFile = "robocop.log"
self.environment = self.auto.environment
self.deviceRoot = self.dm.getDeviceRoot()
self.remoteProfile = options.remoteTestRoot + "/profile"
self.remoteProfileCopy = options.remoteTestRoot + "/profile-copy"
self.auto.setRemoteProfile(self.remoteProfile)
self.remoteConfigFile = os.path.join(
self.deviceRoot, "robotium.config")
self.remoteLog = options.remoteLogFile
self.auto.setRemoteLog(self.remoteLog)
self.remoteScreenshots = "/mnt/sdcard/Robotium-Screenshots"
self.remoteNSPR = os.path.join(options.remoteTestRoot, "nspr")
self.auto.setServerInfo(
self.options.webServer, self.options.httpPort, self.options.sslPort)
self.localLog = options.logFile
self.localProfile = None
productPieces = self.options.remoteProductName.split('.')
if (productPieces is not None):
self.auto.setProduct(productPieces[0])
else:
self.auto.setProduct(self.options.remoteProductName)
self.auto.setAppName(self.options.remoteappname)
self.certdbNew = True
self.remoteCopyAvailable = True
self.passed = 0
self.failed = 0
self.todo = 0
def startup(self):
"""
Second-stage initialization: One-time initialization which may require cleanup.
"""
# Despite our efforts to clean up servers started by this script, in practice
# we still see infrequent cases where a process is orphaned and interferes
# with future tests, typically because the old server is keeping the port in use.
# Try to avoid those failures by checking for and killing orphan servers before
# trying to start new ones.
self.killNamedOrphans('ssltunnel')
self.killNamedOrphans('xpcshell')
self.auto.deleteANRs()
self.auto.deleteTombstones()
self.dm.killProcess(self.options.app.split('/')[-1])
self.dm.removeDir(self.remoteScreenshots)
self.dm.removeDir(self.remoteNSPR)
self.dm.mkDir(self.remoteNSPR)
self.dm.mkDir(os.path.dirname(self.options.remoteLogFile))
# Add Android version (SDK level) to mozinfo so that manifest entries
# can be conditional on android_version.
androidVersion = self.dm.shellCheckOutput(
['getprop', 'ro.build.version.sdk'])
self.log.info(
"Android sdk version '%s'; will use this to filter manifests" %
str(androidVersion))
mozinfo.info['android_version'] = androidVersion
if (self.options.dm_trans == 'adb' and self.options.robocopApk):
self.dm._checkCmd(["install", "-r", self.options.robocopApk])
self.log.debug("Robocop APK %s installed" %
self.options.robocopApk)
# Display remote diagnostics; if running in mach, keep output terse.
if self.options.log_mach is None:
self.printDeviceInfo()
self.setupLocalPaths()
self.buildProfile()
# ignoreSSLTunnelExts is a workaround for bug 1109310
self.startServers(
self.options,
debuggerInfo=None,
ignoreSSLTunnelExts=True)
self.log.debug("Servers started")
def cleanup(self):
"""
Cleanup at end of job run.
"""
self.log.debug("Cleaning up...")
self.stopServers()
self.dm.killProcess(self.options.app.split('/')[-1])
blobberUploadDir = os.environ.get('MOZ_UPLOAD_DIR', None)
if blobberUploadDir:
self.log.debug("Pulling any remote nspr logs and screenshots to %s." %
blobberUploadDir)
self.dm.getDirectory(self.remoteNSPR, blobberUploadDir)
self.dm.getDirectory(self.remoteScreenshots, blobberUploadDir)
Mochitest.cleanup(self, self.options)
if self.localProfile:
os.system("rm -Rf %s" % self.localProfile)
self.dm.removeDir(self.remoteProfile)
self.dm.removeDir(self.remoteProfileCopy)
self.dm.removeDir(self.remoteScreenshots)
self.dm.removeDir(self.remoteNSPR)
self.dm.removeFile(self.remoteConfigFile)
if self.dm.fileExists(self.remoteLog):
self.dm.removeFile(self.remoteLog)
self.log.debug("Cleanup complete.")
def findPath(self, paths, filename=None):
for path in paths:
p = path
if filename:
p = os.path.join(p, filename)
if os.path.exists(self.getFullPath(p)):
return path
return None
def makeLocalAutomation(self):
localAutomation = Automation()
localAutomation.IS_WIN32 = False
localAutomation.IS_LINUX = False
localAutomation.IS_MAC = False
localAutomation.UNIXISH = False
hostos = sys.platform
if (hostos == 'mac' or hostos == 'darwin'):
localAutomation.IS_MAC = True
elif (hostos == 'linux' or hostos == 'linux2'):
localAutomation.IS_LINUX = True
localAutomation.UNIXISH = True
elif (hostos == 'win32' or hostos == 'win64'):
localAutomation.BIN_SUFFIX = ".exe"
localAutomation.IS_WIN32 = True
return localAutomation
def setupLocalPaths(self):
"""
Setup xrePath and utilityPath and verify xpcshell.
This is similar to switchToLocalPaths in runtestsremote.py.
"""
localAutomation = self.makeLocalAutomation()
paths = [
self.options.xrePath,
localAutomation.DIST_BIN,
self.auto._product,
os.path.join('..', self.auto._product)
]
self.options.xrePath = self.findPath(paths)
if self.options.xrePath is None:
self.log.error(
"unable to find xulrunner path for %s, please specify with --xre-path" %
os.name)
sys.exit(1)
self.log.debug("using xre path %s" % self.options.xrePath)
xpcshell = "xpcshell"
if (os.name == "nt"):
xpcshell += ".exe"
if self.options.utilityPath:
paths = [self.options.utilityPath, self.options.xrePath]
else:
paths = [self.options.xrePath]
self.options.utilityPath = self.findPath(paths, xpcshell)
if self.options.utilityPath is None:
self.log.error(
"unable to find utility path for %s, please specify with --utility-path" %
os.name)
sys.exit(1)
self.log.debug("using utility path %s" % self.options.utilityPath)
xpcshell_path = os.path.join(self.options.utilityPath, xpcshell)
if localAutomation.elf_arm(xpcshell_path):
self.log.error('xpcshell at %s is an ARM binary; please use '
'the --utility-path argument to specify the path '
'to a desktop version.' % xpcshell_path)
sys.exit(1)
self.log.debug("xpcshell found at %s" % xpcshell_path)
def buildProfile(self):
"""
Build a profile locally, keep it locally for use by servers and
push a copy to the remote profile-copy directory.
This is similar to buildProfile in runtestsremote.py.
"""
self.options.extraPrefs.append('browser.search.suggest.enabled=true')
self.options.extraPrefs.append('browser.search.suggest.prompted=true')
self.options.extraPrefs.append('layout.css.devPixelsPerPx=1.0')
self.options.extraPrefs.append('browser.chrome.dynamictoolbar=false')
self.options.extraPrefs.append('browser.snippets.enabled=false')
self.options.extraPrefs.append('browser.casting.enabled=true')
self.options.extraPrefs.append('extensions.autoupdate.enabled=false')
manifest = Mochitest.buildProfile(self, self.options)
self.localProfile = self.options.profilePath
self.log.debug("Profile created at %s" % self.localProfile)
# some files are not needed for robocop; save time by not pushing
shutil.rmtree(os.path.join(self.localProfile, 'webapps'))
desktop_extensions = ['mochikit@mozilla.org', 'worker-test@mozilla.org', 'workerbootstrap-test@mozilla.org']
for ext in desktop_extensions:
shutil.rmtree(os.path.join(self.localProfile, 'extensions', 'staged', ext))
os.remove(os.path.join(self.localProfile, 'userChrome.css'))
try:
self.dm.pushDir(self.localProfile, self.remoteProfileCopy)
except devicemanager.DMError:
self.log.error(
"Automation Error: Unable to copy profile to device.")
raise
return manifest
def setupRemoteProfile(self):
"""
Remove any remote profile and re-create it.
"""
self.log.debug("Updating remote profile at %s" % self.remoteProfile)
self.dm.removeDir(self.remoteProfile)
if self.remoteCopyAvailable:
try:
self.dm.shellCheckOutput(
['cp', '-r', self.remoteProfileCopy, self.remoteProfile],
root=True, timeout=60)
except devicemanager.DMError:
# For instance, cp is not available on some older versions of
# Android.
self.log.info(
"Unable to copy remote profile; falling back to push.")
self.remoteCopyAvailable = False
if not self.remoteCopyAvailable:
self.dm.pushDir(self.localProfile, self.remoteProfile)
def parseLocalLog(self):
"""
Read and parse the local log file, noting any failures.
"""
with open(self.localLog) as currentLog:
data = currentLog.readlines()
os.unlink(self.localLog)
start_found = False
end_found = False
fail_found = False
for line in data:
try:
message = json.loads(line)
if not isinstance(message, dict) or 'action' not in message:
continue
except ValueError:
continue
if message['action'] == 'test_end':
end_found = True
start_found = False
break
if start_found and not end_found:
if 'status' in message:
if 'expected' in message:
self.failed += 1
elif message['status'] == 'PASS':
self.passed += 1
elif message['status'] == 'FAIL':
self.todo += 1
if message['action'] == 'test_start':
start_found = True
if 'expected' in message:
fail_found = True
result = 0
if fail_found:
result = 1
if not end_found:
self.log.info(
"PROCESS-CRASH | Automation Error: Missing end of test marker (process crashed?)")
result = 1
return result
def logTestSummary(self):
"""
Print a summary of all tests run to stdout, for treeherder parsing
(logging via self.log does not work here).
"""
print("0 INFO TEST-START | Shutdown")
print("1 INFO Passed: %s" % (self.passed))
print("2 INFO Failed: %s" % (self.failed))
print("3 INFO Todo: %s" % (self.todo))
print("4 INFO SimpleTest FINISHED")
if self.failed > 0:
return 1
return 0
def printDeviceInfo(self, printLogcat=False):
"""
Log remote device information and logcat (if requested).
This is similar to printDeviceInfo in runtestsremote.py
"""
try:
if printLogcat:
logcat = self.dm.getLogcat(
filterOutRegexps=fennecLogcatFilters)
self.log.info(
'\n' +
''.join(logcat).decode(
'utf-8',
'replace'))
self.log.info("Device info:")
devinfo = self.dm.getInfo()
for category in devinfo:
if type(devinfo[category]) is list:
self.log.info(" %s:" % category)
for item in devinfo[category]:
self.log.info(" %s" % item)
else:
self.log.info(" %s: %s" % (category, devinfo[category]))
self.log.info("Test root: %s" % self.dm.deviceRoot)
except devicemanager.DMError:
self.log.warning("Error getting device information")
def setupRobotiumConfig(self, browserEnv):
"""
Create robotium.config and push it to the device.
"""
fHandle = tempfile.NamedTemporaryFile(suffix='.config',
prefix='robotium-',
dir=os.getcwd(),
delete=False)
fHandle.write("profile=%s\n" % (self.remoteProfile))
fHandle.write("logfile=%s\n" % (self.options.remoteLogFile))
fHandle.write("host=http://mochi.test:8888/tests\n")
fHandle.write(
"rawhost=http://%s:%s/tests\n" %
(self.options.remoteWebServer, self.options.httpPort))
if browserEnv:
envstr = ""
delim = ""
for key, value in browserEnv.items():
try:
value.index(',')
self.log.error(
"setupRobotiumConfig: browserEnv - Found a ',' in our value, unable to process value. key=%s,value=%s" %
(key, value))
self.log.error("browserEnv=%s" % browserEnv)
except ValueError:
envstr += "%s%s=%s" % (delim, key, value)
delim = ","
fHandle.write("envvars=%s\n" % envstr)
fHandle.close()
self.dm.removeFile(self.remoteConfigFile)
self.dm.pushFile(fHandle.name, self.remoteConfigFile)
os.unlink(fHandle.name)
def buildBrowserEnv(self):
"""
Return an environment dictionary suitable for remote use.
This is similar to buildBrowserEnv in runtestsremote.py.
"""
browserEnv = self.environment(
xrePath=None,
debugger=None)
# remove desktop environment not used on device
if "MOZ_WIN_INHERIT_STD_HANDLES_PRE_VISTA" in browserEnv:
del browserEnv["MOZ_WIN_INHERIT_STD_HANDLES_PRE_VISTA"]
if "XPCOM_MEM_BLOAT_LOG" in browserEnv:
del browserEnv["XPCOM_MEM_BLOAT_LOG"]
browserEnv["NSPR_LOG_FILE"] = os.path.join(
self.remoteNSPR,
self.nsprLogName)
return browserEnv
def runSingleTest(self, test):
"""
Run the specified test.
"""
self.log.debug("Running test %s" % test['name'])
self.nsprLogName = "nspr-%s.log" % test['name']
browserEnv = self.buildBrowserEnv()
self.setupRobotiumConfig(browserEnv)
self.setupRemoteProfile()
self.options.app = "am"
if self.options.autorun:
# This launches a test (using "am instrument") and instructs
# Fennec to /quit/ the browser (using Robocop:Quit) and to
# /finish/ all opened activities.
browserArgs = [
"instrument",
"-w",
"-e", "quit_and_finish", "1",
"-e", "deviceroot", self.deviceRoot,
"-e", "class",
"org.mozilla.gecko.tests.%s" % test['name'].split('.java')[0],
"org.mozilla.roboexample.test/org.mozilla.gecko.FennecInstrumentationTestRunner"]
else:
# This does not launch a test at all. It launches an activity
# that starts Fennec and then waits indefinitely, since cat
# never returns.
browserArgs = ["start",
"-n", "org.mozilla.roboexample.test/org.mozilla.gecko.LaunchFennecWithConfigurationActivity",
"&&", "cat"]
self.dm.default_timeout = sys.maxint # Forever.
self.log.info("")
self.log.info("Serving mochi.test Robocop root at http://%s:%s/tests/robocop/" %
(self.options.remoteWebServer, self.options.httpPort))
self.log.info("")
result = -1
log_result = -1
try:
self.dm.recordLogcat()
result = self.auto.runApp(
None, browserEnv, "am", self.localProfile, browserArgs, timeout=self.NO_OUTPUT_TIMEOUT)
self.log.debug("runApp completes with status %d" % result)
if result != 0:
self.log.error("runApp() exited with code %s" % result)
if self.dm.fileExists(self.remoteLog):
self.dm.getFile(self.remoteLog, self.localLog)
self.dm.removeFile(self.remoteLog)
self.log.debug("Remote log %s retrieved to %s" %
(self.remoteLog, self.localLog))
else:
self.log.warning(
"Unable to retrieve log file (%s) from remote device" %
self.remoteLog)
log_result = self.parseLocalLog()
if result != 0 or log_result != 0:
# Display remote diagnostics; if running in mach, keep output
# terse.
if self.options.log_mach is None:
self.printDeviceInfo(printLogcat=True)
except:
self.log.error(
"Automation Error: Exception caught while running tests")
traceback.print_exc()
result = 1
self.log.debug("Test %s completes with status %d (log status %d)" %
(test['name'], int(result), int(log_result)))
return result
def runTests(self):
self.startup()
if isinstance(self.options.manifestFile, TestManifest):
mp = self.options.manifestFile
else:
mp = TestManifest(strict=False)
mp.read(self.options.robocopIni)
filters = []
if self.options.totalChunks:
filters.append(
chunk_by_slice(self.options.thisChunk, self.options.totalChunks))
robocop_tests = mp.active_tests(
exists=False, filters=filters, **mozinfo.info)
if not self.options.autorun:
# Force a single loop iteration. The iteration will start Fennec and
# the httpd server, but not actually run a test.
self.options.test_paths = [robocop_tests[0]['name']]
active_tests = []
for test in robocop_tests:
if self.options.test_paths and test['name'] not in self.options.test_paths:
continue
if 'disabled' in test:
self.log.info('TEST-INFO | skipping %s | %s' %
(test['name'], test['disabled']))
continue
active_tests.append(test)
self.log.suite_start([t['name'] for t in active_tests])
worstTestResult = None
for test in active_tests:
result = self.runSingleTest(test)
if worstTestResult is None or worstTestResult == 0:
worstTestResult = result
if worstTestResult is None:
self.log.warning(
"No tests run. Did you pass an invalid TEST_PATH?")
worstTestResult = 1
else:
print "INFO | runtests.py | Test summary: start."
logResult = self.logTestSummary()
print "INFO | runtests.py | Test summary: end."
if worstTestResult == 0:
worstTestResult = logResult
return worstTestResult
def run_test_harness(options):
if options is None:
raise ValueError(
"Invalid options specified, use --help for a list of valid options")
message_logger = MessageLogger(logger=None)
process_args = {'messageLogger': message_logger}
auto = RemoteAutomation(None, "fennec", processArgs=process_args)
auto.setDeviceManager(options.dm)
runResult = -1
robocop = RobocopTestRunner(auto, options.dm, options)
try:
message_logger.logger = robocop.log
message_logger.buffering = False
robocop.message_logger = message_logger
robocop.log.debug("options=%s" % vars(options))
runResult = robocop.runTests()
except KeyboardInterrupt:
robocop.log.info("runrobocop.py | Received keyboard interrupt")
runResult = -1
except:
traceback.print_exc()
robocop.log.error(
"runrobocop.py | Received unexpected exception while running tests")
runResult = 1
finally:
try:
robocop.cleanup()
except devicemanager.DMError:
# ignore device error while cleaning up
pass
message_logger.finish()
return runResult
def main(args=sys.argv[1:]):
parser = MochitestArgumentParser(app='android')
options = parser.parse_args(args)
return run_test_harness(options)
if __name__ == "__main__":
sys.exit(main())

427
testing/mochitest/runtestsremote.py
View File

@ -66,10 +66,6 @@ class MochiRemote(Mochitest):
self.remoteLog)
self._dm.removeDir(self.remoteProfile)
self._dm.removeDir(self.remoteChromeTestDir)
# Don't leave an old robotium.config hanging around; the
# profile it references was just deleted!
deviceRoot = self._dm.getDeviceRoot()
self._dm.removeFile(os.path.join(deviceRoot, "robotium.config"))
blobberUploadDir = os.environ.get('MOZ_UPLOAD_DIR', None)
if blobberUploadDir:
self._dm.getDirectory(self.remoteNSPR, blobberUploadDir)
@ -178,29 +174,6 @@ class MochiRemote(Mochitest):
self.localProfile = options.profilePath
self._dm.removeDir(self.remoteProfile)
# we do not need this for robotium based tests, lets save a LOT of time
if options.robocopIni:
shutil.rmtree(os.path.join(options.profilePath, 'webapps'))
shutil.rmtree(
os.path.join(
options.profilePath,
'extensions',
'staged',
'mochikit@mozilla.org'))
shutil.rmtree(
os.path.join(
options.profilePath,
'extensions',
'staged',
'worker-test@mozilla.org'))
shutil.rmtree(
os.path.join(
options.profilePath,
'extensions',
'staged',
'workerbootstrap-test@mozilla.org'))
os.remove(os.path.join(options.profilePath, 'userChrome.css'))
try:
self._dm.pushDir(options.profilePath, self.remoteProfile)
except devicemanager.DMError:
@ -220,38 +193,18 @@ class MochiRemote(Mochitest):
env["MOZ_HIDE_RESULTS_TABLE"] = "1"
retVal = Mochitest.buildURLOptions(self, options, env)
if not options.robocopIni:
# we really need testConfig.js (for browser chrome)
try:
self._dm.pushDir(options.profilePath, self.remoteProfile)
except devicemanager.DMError:
self.log.error(
"Automation Error: Unable to copy profile to device.")
raise
# we really need testConfig.js (for browser chrome)
try:
self._dm.pushDir(options.profilePath, self.remoteProfile)
except devicemanager.DMError:
self.log.error(
"Automation Error: Unable to copy profile to device.")
raise
options.profilePath = self.remoteProfile
options.logFile = self.localLog
return retVal
def getTestsToRun(self, options):
if options.robocopIni != "":
# Skip over tests filtering if we run robocop tests.
return None
else:
return super(MochiRemote, self).getTestsToRun(options)
def buildTestPath(self, options, testsToFilter=None):
if options.robocopIni != "":
# Skip over manifest building if we just want to run
# robocop tests.
return self.buildTestURL(options)
else:
return super(
MochiRemote,
self).buildTestPath(
options,
testsToFilter)
def getChromeTestDir(self, options):
local = super(MochiRemote, self).getChromeTestDir(options)
local = os.path.join(local, "chrome")
@ -264,105 +217,6 @@ class MochiRemote(Mochitest):
def getLogFilePath(self, logFile):
return logFile
# In the future we could use LogParser:
# http://hg.mozilla.org/automation/logparser/
def addLogData(self):
with open(self.localLog) as currentLog:
data = currentLog.readlines()
start_found = False
end_found = False
fail_found = False
for line in data:
try:
message = json.loads(line)
if not isinstance(message, dict) or 'action' not in message:
continue
except ValueError:
continue
if message['action'] == 'test_end':
end_found = True
start_found = False
break
if start_found and not end_found:
self.logMessages.append(message)
if message['action'] == 'test_start':
start_found = True
if 'expected' in message:
fail_found = True
result = 0
if fail_found:
result = 1
if not end_found:
self.log.info(
"PROCESS-CRASH | Automation Error: Missing end of test marker (process crashed?)")
result = 1
return result
def printLog(self):
passed = 0
failed = 0
todo = 0
incr = 1
logFile = []
logFile.append("0 INFO SimpleTest START")
for message in self.logMessages:
if 'status' not in message:
continue
if 'expected' in message:
failed += 1
elif message['status'] == 'PASS':
passed += 1
elif message['status'] == 'FAIL':
todo += 1
incr += 1
logFile.append("%s INFO TEST-START | Shutdown" % incr)
incr += 1
logFile.append("%s INFO Passed: %s" % (incr, passed))
incr += 1
logFile.append("%s INFO Failed: %s" % (incr, failed))
incr += 1
logFile.append("%s INFO Todo: %s" % (incr, todo))
incr += 1
logFile.append("%s INFO SimpleTest FINISHED" % incr)
# TODO: Consider not printing to stdout because we might be duplicating
# output
print '\n'.join(logFile)
with open(self.localLog, 'w') as localLog:
localLog.write('\n'.join(logFile))
if failed > 0:
return 1
return 0
def printScreenshots(self, screenShotDir):
# TODO: This can be re-written after completion of bug 749421
if not self._dm.dirExists(screenShotDir):
self.log.info(
"SCREENSHOT: No ScreenShots directory available: " +
screenShotDir)
return
printed = 0
for name in self._dm.listFiles(screenShotDir):
fullName = screenShotDir + "/" + name
self.log.info("SCREENSHOT: FOUND: [%s]" % fullName)
try:
image = self._dm.pullFile(fullName)
encoded = base64.b64encode(image)
self.log.info("SCREENSHOT: data:image/jpg;base64,%s" % encoded)
printed += 1
except:
self.log.info("SCREENSHOT: Could not be parsed")
pass
self.log.info("SCREENSHOT: TOTAL PRINTED: [%s]" % printed)
def printDeviceInfo(self, printLogcat=False):
try:
if printLogcat:
@ -386,44 +240,6 @@ class MochiRemote(Mochitest):
except devicemanager.DMError:
self.log.warning("Error getting device information")
def buildRobotiumConfig(self, options, browserEnv):
deviceRoot = self._dm.deviceRoot
fHandle = tempfile.NamedTemporaryFile(suffix='.config',
prefix='robotium-',
dir=os.getcwd(),
delete=False)
fHandle.write("profile=%s\n" % (self.remoteProfile))
fHandle.write("logfile=%s\n" % (options.remoteLogFile))
fHandle.write("host=http://mochi.test:8888/tests\n")
fHandle.write(
"rawhost=http://%s:%s/tests\n" %
(options.remoteWebServer, options.httpPort))
if browserEnv:
envstr = ""
delim = ""
for key, value in browserEnv.items():
try:
value.index(',')
self.log.error(
"buildRobotiumConfig: browserEnv - Found a ',' in our value, unable to process value. key=%s,value=%s" %
(key, value))
self.log.error("browserEnv=%s" % browserEnv)
except ValueError:
envstr += "%s%s=%s" % (delim, key, value)
delim = ","
fHandle.write("envvars=%s\n" % envstr)
fHandle.close()
self._dm.removeFile(os.path.join(deviceRoot, "robotium.config"))
self._dm.pushFile(
fHandle.name,
os.path.join(
deviceRoot,
"robotium.config"))
os.unlink(fHandle.name)
def getGMPPluginPath(self, options):
# TODO: bug 1149374
return None
@ -443,7 +259,6 @@ class MochiRemote(Mochitest):
browserEnv["NSPR_LOG_FILE"] = os.path.join(
self.remoteNSPR,
self.nsprLogName)
self.buildRobotiumConfig(options, browserEnv)
return browserEnv
def runApp(self, *args, **kwargs):
@ -511,222 +326,22 @@ def run_test_harness(options):
procName = options.app.split('/')[-1]
dm.killProcess(procName)
if options.robocopIni != "":
# turning buffering off as it's not used in robocop
message_logger.buffering = False
# sut may wait up to 300 s for a robocop am process before returning
dm.default_timeout = 320
if isinstance(options.manifestFile, TestManifest):
mp = options.manifestFile
else:
mp = TestManifest(strict=False)
mp.read(options.robocopIni)
filters = []
if options.totalChunks:
filters.append(
chunk_by_slice(options.thisChunk, options.totalChunks))
robocop_tests = mp.active_tests(exists=False, filters=filters, **mozinfo.info)
options.extraPrefs.append('browser.search.suggest.enabled=true')
options.extraPrefs.append('browser.search.suggest.prompted=true')
options.extraPrefs.append('layout.css.devPixelsPerPx=1.0')
options.extraPrefs.append('browser.chrome.dynamictoolbar=false')
options.extraPrefs.append('browser.snippets.enabled=false')
options.extraPrefs.append('browser.casting.enabled=true')
options.extraPrefs.append('extensions.autoupdate.enabled=false')
if (options.dm_trans == 'adb' and options.robocopApk):
dm._checkCmd(["install", "-r", options.robocopApk])
if not options.autorun:
# Force a single loop iteration. The iteration will start Fennec and
# the httpd server, but not actually run a test.
options.test_paths = [robocop_tests[0]['name']]
retVal = None
# Filtering tests
active_tests = []
for test in robocop_tests:
if options.test_paths and test['name'] not in options.test_paths:
continue
if 'disabled' in test:
log.info(
'TEST-INFO | skipping %s | %s' %
(test['name'], test['disabled']))
continue
active_tests.append(test)
log.suite_start([t['name'] for t in active_tests])
for test in active_tests:
# When running in a loop, we need to create a fresh profile for
# each cycle
if mochitest.localProfile:
options.profilePath = mochitest.localProfile
os.system("rm -Rf %s" % options.profilePath)
options.profilePath = None
mochitest.localProfile = options.profilePath
options.app = "am"
mochitest.nsprLogName = "nspr-%s.log" % test['name']
if options.autorun:
# This launches a test (using "am instrument") and instructs
# Fennec to /quit/ the browser (using Robocop:Quit) and to
# /finish/ all opened activities.
options.browserArgs = [
"instrument",
"-w",
"-e", "quit_and_finish", "1",
"-e", "deviceroot", deviceRoot,
"-e",
"class"]
options.browserArgs.append(
"org.mozilla.gecko.tests.%s" %
test['name'].split('.java')[0])
options.browserArgs.append(
"org.mozilla.roboexample.test/org.mozilla.gecko.FennecInstrumentationTestRunner")
else:
# This does not launch a test at all. It launches an activity
# that starts Fennec and then waits indefinitely, since cat
# never returns.
options.browserArgs = ["start",
"-n", "org.mozilla.roboexample.test/org.mozilla.gecko.LaunchFennecWithConfigurationActivity",
"&&", "cat"]
dm.default_timeout = sys.maxint # Forever.
mochitest.log.info("")
mochitest.log.info("Serving mochi.test Robocop root at http://%s:%s/tests/robocop/" %
(options.remoteWebServer, options.httpPort))
mochitest.log.info("")
# If the test is for checking the import from bookmarks then make
# sure there is data to import
if test['name'] == "testImportFromAndroid":
# Get the OS so we can run the insert in the apropriate
# database and following the correct table schema
osInfo = dm.getInfo("os")
devOS = " ".join(osInfo['os'])
if ("pandaboard" in devOS):
delete = [
'execsu',
'sqlite3',
"/data/data/com.android.browser/databases/browser2.db \'delete from bookmarks where _id > 14;\'"]
else:
delete = [
'execsu',
'sqlite3',
"/data/data/com.android.browser/databases/browser.db \'delete from bookmarks where _id > 14;\'"]
if (options.dm_trans == "sut"):
dm._runCmds([{"cmd": " ".join(delete)}])
# Insert the bookmarks
log.info(
"Insert bookmarks in the default android browser database")
for i in range(20):
if ("pandaboard" in devOS):
cmd = [
'execsu',
'sqlite3',
"/data/data/com.android.browser/databases/browser2.db 'insert or replace into bookmarks(_id,title,url,folder,parent,position) values (" +
str(
30 +
i) +
",\"Bookmark" +
str(i) +
"\",\"http://www.bookmark" +
str(i) +
".com\",0,1," +
str(
100 +
i) +
");'"]
else:
cmd = [
'execsu',
'sqlite3',
"/data/data/com.android.browser/databases/browser.db 'insert into bookmarks(title,url,bookmark) values (\"Bookmark" +
str(i) +
"\",\"http://www.bookmark" +
str(i) +
".com\",1);'"]
if (options.dm_trans == "sut"):
dm._runCmds([{"cmd": " ".join(cmd)}])
try:
screenShotDir = "/mnt/sdcard/Robotium-Screenshots"
dm.removeDir(screenShotDir)
dm.recordLogcat()
result = mochitest.runTests(options)
if result != 0:
log.error("runTests() exited with code %s" % result)
log_result = mochitest.addLogData()
if result != 0 or log_result != 0:
mochitest.printDeviceInfo(printLogcat=True)
mochitest.printScreenshots(screenShotDir)
# Ensure earlier failures aren't overwritten by success on this
# run
if retVal is None or retVal == 0:
retVal = result
except:
log.error(
"Automation Error: Exception caught while running tests")
traceback.print_exc()
mochitest.stopServers()
try:
mochitest.cleanup(options)
except devicemanager.DMError:
# device error cleaning up... oh well!
pass
retVal = 1
break
finally:
# Clean-up added bookmarks
if test['name'] == "testImportFromAndroid":
if ("pandaboard" in devOS):
cmd_del = [
'execsu',
'sqlite3',
"/data/data/com.android.browser/databases/browser2.db \'delete from bookmarks where _id > 14;\'"]
else:
cmd_del = [
'execsu',
'sqlite3',
"/data/data/com.android.browser/databases/browser.db \'delete from bookmarks where _id > 14;\'"]
if (options.dm_trans == "sut"):
dm._runCmds([{"cmd": " ".join(cmd_del)}])
if retVal is None:
log.warning("No tests run. Did you pass an invalid TEST_PATH?")
retVal = 1
else:
# if we didn't have some kind of error running the tests, make
# sure the tests actually passed
print "INFO | runtests.py | Test summary: start."
overallResult = mochitest.printLog()
print "INFO | runtests.py | Test summary: end."
if retVal == 0:
retVal = overallResult
else:
mochitest.nsprLogName = "nspr.log"
mochitest.nsprLogName = "nspr.log"
try:
dm.recordLogcat()
retVal = mochitest.runTests(options)
except:
log.error("Automation Error: Exception caught while running tests")
traceback.print_exc()
mochitest.stopServers()
try:
dm.recordLogcat()
retVal = mochitest.runTests(options)
except:
log.error("Automation Error: Exception caught while running tests")
traceback.print_exc()
mochitest.stopServers()
try:
mochitest.cleanup(options)
except devicemanager.DMError:
# device error cleaning up... oh well!
pass
retVal = 1
mochitest.cleanup(options)
except devicemanager.DMError:
# device error cleaning up... oh well!
pass
retVal = 1
mochitest.printDeviceInfo(printLogcat=True)
mochitest.printDeviceInfo(printLogcat=True)
message_logger.finish()

3
testing/testsuite-targets.mk
View File

@ -60,9 +60,8 @@ RUN_MOCHITEST_REMOTE = \
RUN_MOCHITEST_ROBOCOP = \
rm -f ./$@.log && \
$(PYTHON) _tests/testing/mochitest/runtestsremote.py \
$(PYTHON) _tests/testing/mochitest/runrobocop.py \
--robocop-apk=$(DEPTH)/build/mobile/robocop/robocop-debug.apk \
--robocop-ids=$(DEPTH)/mobile/android/base/fennec_ids.txt \
--robocop-ini=_tests/testing/mochitest/robocop.ini \
--log-tbpl=./$@.log $(DM_FLAGS) --dm_trans=$(DM_TRANS) \
--app=$(TEST_PACKAGE_NAME) --deviceIP=${TEST_DEVICE} --xre-path=${MOZ_HOST_BIN} \