Backout 8ec01f6f316f (bug 705175), a1dc0efef542 (bug 716077), b5686f25b258 (bug 710445), 32f8d3be2ad1 (bug 715309) for mochitest/reftest failures on Android & Windows

This commit is contained in:
Ed Morley 2012-01-07 14:57:58 +00:00
parent d57a89d468
commit 71e7d54a52
14 changed files with 216 additions and 202 deletions

View File

@ -208,14 +208,14 @@ class DeviceManager:
pieces = appname.split(' ')
parts = pieces[0].split('/')
app = parts[-1]
procre = re.compile('.*' + app + '.*')
procList = self.getProcessList()
if (procList == []):
return None
for proc in procList:
procName = proc[1].split('/')[-1]
if (procName == app):
if (procre.match(proc[1])):
pid = proc[0]
break
return pid
@ -346,7 +346,24 @@ class DeviceManager:
# success: path for app root
# failure: None
def getAppRoot(self):
assert 0 == 1
devroot = self.getDeviceRoot()
if (devroot == None):
return None
if (self.dirExists(devroot + '/fennec')):
return devroot + '/fennec'
elif (self.dirExists(devroot + '/firefox')):
return devroot + '/firefox'
elif (self.dirExsts('/data/data/org.mozilla.fennec')):
return 'org.mozilla.fennec'
elif (self.dirExists('/data/data/org.mozilla.firefox')):
return 'org.mozilla.firefox'
elif (self.dirExists('/data/data/org.mozilla.fennec_aurora')):
return 'org.mozilla.fennec_aurora'
elif (self.dirExists('/data/data/org.mozilla.firefox_beta')):
return 'org.mozilla.firefox_beta'
# Failure (either not installed or not a recognized platform)
return None
# Gets the directory location on the device for a specific test type

View File

@ -258,10 +258,6 @@ class DeviceManagerADB(DeviceManager):
# success: output filename
# failure: None
def launchProcess(self, cmd, outputFile = "process.txt", cwd = '', env = '', failIfRunning=False):
if cmd[0] == "am":
self.checkCmd(["shell"] + cmd)
return outputFile
acmd = ["shell", "am","start"]
cmd = ' '.join(cmd).strip()
i = cmd.find(" ")
@ -286,7 +282,7 @@ class DeviceManagerADB(DeviceManager):
acmd.append(''.join(['\'',uri, '\'']));
print acmd
self.checkCmd(acmd)
return outputFile
return outputFile;
# external function
# returns:
@ -426,14 +422,15 @@ class DeviceManagerADB(DeviceManager):
# returns:
# success: path for app root
# failure: None
def getAppRoot(self, packageName):
def getAppRoot(self):
devroot = self.getDeviceRoot()
if (devroot == None):
return None
if (packageName and self.dirExists('/data/data/' + packageName)):
self.packageName = packageName
return '/data/data/' + packageName
if (self.dirExists(devroot + '/fennec')):
return devroot + '/fennec'
elif (self.dirExists(devroot + '/firefox')):
return devroot + '/firefox'
elif (self.packageName and self.dirExists('/data/data/' + self.packageName)):
return '/data/data/' + self.packageName

View File

@ -552,14 +552,14 @@ class DeviceManagerSUT(DeviceManager):
pieces = appname.split(' ')
parts = pieces[0].split('/')
app = parts[-1]
procre = re.compile('.*' + app + '.*')
procList = self.getProcessList()
if (procList == []):
return None
for proc in procList:
procName = proc[1].split('/')[-1]
if (procName == app):
if (procre.match(proc[1])):
pid = proc[0]
break
return pid
@ -858,15 +858,6 @@ class DeviceManagerSUT(DeviceManager):
return deviceRoot
def getAppRoot(self, packageName):
try:
data = self.verifySendCMD(['getapproot '+packageName])
except:
return None
appRoot = self.stripPrompt(data).strip('\n')
return appRoot
# external function
# returns:
# success: output of unzip command

View File

@ -122,11 +122,6 @@ class RemoteAutomation(Automation):
if (self._remoteProfile):
profileDir = self._remoteProfile
# Hack for robocop, if app & testURL == None and extraArgs contains the rest of the stuff, lets
# assume extraArgs is all we need
if app == "am" and extraArgs[0] == "instrument":
return app, extraArgs
cmd, args = Automation.buildCommandLine(self, app, debuggerInfo, profileDir, testURL, extraArgs)
# Remove -foreground if it exists, if it doesn't this just returns
try:

View File

@ -42,9 +42,7 @@ package @ANDROID_PACKAGE_NAME@;
public interface Assert {
void dumpLog(String message);
void setLogFile(String filename);
void setTestName(String testName);
void finalize();
void ok(boolean condition, String name, String diag);
void is(Object a, Object b, String name);
void isnot(Object a, Object b, String name);

View File

@ -68,4 +68,14 @@ public interface Driver {
void startFrameRecording();
int stopFrameRecording();
void dumpLog(String message);
void setLogFile(String filename);
void ok(boolean condition, String name, String diag);
void is(Object a, Object b, String name);
void isnot(Object a, Object b, String name);
void todo(boolean condition, String name, String diag);
void todo_is(Object a, Object b, String name);
void todo_isnot(Object a, Object b, String name);
void info(String name, String message);
}

View File

@ -49,7 +49,6 @@ import java.util.ArrayList;
import java.util.LinkedList;
import java.util.HashMap;
import java.util.List;
import java.util.Date;
import java.lang.Class;
import java.lang.reflect.InvocationTargetException;
@ -78,22 +77,7 @@ public class FennecNativeAssert implements Assert {
// If waiting for an event.
private boolean asleep = false;
// Internal state variables to make logging match up with existing mochitests
private int lineNumber = 0;
private int passed = 0;
private int failed = 0;
private int todo = 0;
// Used to write the first line of the test file
private boolean logStarted = false;
// Used to write the test-start/test-end log lines
private String logTestName = "";
// Measure the time it takes to run test case
private long startTime = 0;
public FennecNativeAssert() {
public FennecNativeAssert(){
}
// Write information to a logfile and logcat
@ -126,34 +110,8 @@ public class FennecNativeAssert implements Assert {
public void setLogFile(String filename)
{
logFile = filename;
//TODO: these two messages are mochitest specific
String message;
if (!logStarted) {
message = Integer.toString(lineNumber++) + " INFO SimpleTest START";
dumpLog(message);
logStarted = true;
}
if (logTestName != "") {
message = Integer.toString(lineNumber++) + " INFO TEST-END | " + logTestName;
long diff = (new Date().getTime()) - startTime;
message += " | finished in " + diff + "ms";
dumpLog(message);
logTestName = "";
}
}
public void setTestName(String testName)
{
String[] nameParts = testName.split("\\.");
logTestName = nameParts[nameParts.length - 1];
startTime = new Date().getTime();
//TODO: this is mochitest specific
String message = Integer.toString(lineNumber++) + " INFO TEST-START | " + logTestName;
dumpLog(message);
}
class testInfo {
public boolean result;
@ -169,23 +127,24 @@ public class FennecNativeAssert implements Assert {
}
private void _logMochitestResult(testInfo test, String passString, String failString)
private void _logResult(testInfo test, String passString, String failString)
{
boolean isError = true;
String resultString = failString;
if (test.result || test.todo) {
if(test.result || test.todo){
isError = false;
}
if (test.result)
if(test.result)
{
resultString = passString;
}
String diag = test.name;
if (test.diag != null) diag += " - " + test.diag;
String diag= test.name;
if(test.diag!=null) diag+= " - " + test.diag;
String message = Integer.toString(lineNumber++) + " INFO " + resultString + " | " + logTestName + " | " + diag;
if (isError) {
if (logFile == null)
String message = resultString + " | " + "ROBOCOP" + " | " + diag;
if(isError) {
if(logFile == null)
{
assert(false);
}
@ -196,48 +155,11 @@ public class FennecNativeAssert implements Assert {
else {
dumpLog(message);
}
if (test.todo) {
todo++;
} else if (isError) {
failed++;
} else {
passed++;
}
}
public void finalize()
{
// It appears that we call finalize during cleanup, this might be an invalid assertion.
String message;
if (logTestName != "") {
long diff = (new Date().getTime()) - startTime;
//TODO: this is mochitest specific
message = Integer.toString(lineNumber++) + " INFO TEST-END | " + logTestName;
message += " | finished in " + diff + "ms";
dumpLog(message);
logTestName = "";
}
//TODO: this is all mochitest specific
message = Integer.toString(lineNumber++) + " INFO TEST-START | Shutdown";
dumpLog(message);
message = Integer.toString(lineNumber++) + " INFO Passed: " + Integer.toString(passed);
dumpLog(message);
message = Integer.toString(lineNumber++) + " INFO Failed: " + Integer.toString(failed);
dumpLog(message);
message = Integer.toString(lineNumber++) + " INFO Todo: " + Integer.toString(todo);
dumpLog(message);
message = Integer.toString(lineNumber++) + " INFO SimpleTest FINISHED";
dumpLog(message);
}
public void ok(boolean condition, String name, String diag) {
testInfo test = new testInfo(condition, name, diag, false);
_logMochitestResult(test, "TEST-PASS", "TEST-UNEXPECTED-FAIL");
_logResult(test, "TEST-PASS", "TEST-UNEXPECTED-FAIL");
testList.add(test);
}
@ -261,7 +183,7 @@ public class FennecNativeAssert implements Assert {
public void todo(boolean condition, String name, String diag) {
testInfo test = new testInfo(condition, name, diag, true);
_logMochitestResult(test, "TEST-UNEXPECTED-PASS", "TEST-KNOWN-FAIL");
_logResult(test, "TEST-UNEXPECTED-PASS", "TEST-KNOWN-FAIL");
testList.add(test);
}
@ -285,6 +207,6 @@ public class FennecNativeAssert implements Assert {
public void info(String name, String message) {
testInfo test = new testInfo(true, name, message, false);
_logMochitestResult(test, "TEST-INFO", "INFO FAILED?");
_logResult(test, "TEST-INFO", "INFO FAILED?");
}
}

View File

@ -70,6 +70,7 @@ public class FennecNativeDriver implements Driver {
private HashMap locators = null;
private Activity activity;
private Solo solo;
private String logFile;
// Objects for reflexive access of fennec classes.
private ClassLoader classLoader;
@ -82,6 +83,9 @@ public class FennecNativeDriver implements Driver {
private Method _startFrameRecording;
private Method _stopFrameRecording;
private LinkedList<testInfo> testList = new LinkedList<testInfo>();
public FennecNativeDriver(Activity activity, Solo robocop){
this.activity = activity;
this.solo = robocop;
@ -282,6 +286,38 @@ public class FennecNativeDriver implements Driver {
return text.toString();
}
// Write information to a logfile and logcat
public void dumpLog(String message)
{
File file = new File(logFile);
BufferedWriter bw = null;
try {
bw = new BufferedWriter(new FileWriter(logFile, true));
bw.write(message);
bw.newLine();
} catch(IOException e) {
e.printStackTrace();
} finally {
try {
if (bw != null) {
bw.flush();
bw.close();
}
} catch (IOException ex) {
ex.printStackTrace();
}
}
Log.i("Robocop", message);
}
// Set the filename used for dumpLog.
public void setLogFile(String filename)
{
logFile = filename;
}
// Takes a string of "key=value" pairs split by \n and creates a hash table.
public static HashMap convertTextToTable(String data)
{
@ -294,4 +330,95 @@ public class FennecNativeDriver implements Driver {
}
return retVal;
}
class testInfo {
public boolean result;
public String name;
public String diag;
public boolean todo;
public testInfo(boolean r, String n, String d, boolean t) {
result = r;
name = n;
diag = d;
todo = t;
}
}
private void _logResult(testInfo test, String passString, String failString)
{
boolean isError = true;
String resultString = failString;
if(test.result || test.todo){
isError = false;
}
if(test.result)
{
resultString = passString;
}
String diag= test.name;
if(test.diag!=null) diag+= " - " + test.diag;
String message = resultString + " | " + "ROBOCOP" + " | " + diag;
if(isError) {
dumpLog(message);
}
else {
dumpLog(message);
}
}
public void ok(boolean condition, String name, String diag) {
testInfo test = new testInfo(condition, name, diag, false);
_logResult(test, "TEST-PASS", "TEST-UNEXPECTED-FAIL");
testList.add(test);
}
public void is(Object a, Object b, String name) {
boolean pass = a.equals(b);
String diag = "got " + a.toString() + ", expected " + b.toString();
if(pass) {
diag = a.toString() + " should equal " + b.toString();
}
ok(pass, name, diag);
}
public void isnot(Object a, Object b, String name) {
boolean pass = !a.equals(b);
String diag = "didn't expect " + a.toString() + ", but got it";
if(pass) {
diag = a.toString() + " should not equal " + b.toString();
}
ok(pass, name, diag);
}
public void todo(boolean condition, String name, String diag) {
testInfo test = new testInfo(condition, name, diag, true);
_logResult(test, "TEST-UNEXPECTED-PASS", "TEST-KNOWN-FAIL");
testList.add(test);
}
public void todo_is(Object a, Object b, String name) {
boolean pass = a.equals(b);
String diag = "got " + a.toString() + ", expected " + b.toString();
if(pass) {
diag = a.toString() + " should equal " + b.toString();
}
todo(pass, name, diag);
}
public void todo_isnot(Object a, Object b, String name) {
boolean pass = !a.equals(b);
String diag = "didn't expect " + a.toString() + ", but got it";
if(pass) {
diag = a.toString() + " should not equal " + b.toString();
}
todo(pass, name, diag);
}
public void info(String name, String message) {
testInfo test = new testInfo(true, name, message, false);
_logResult(test, "TEST-INFO", "INFO FAILED?");
}
}

View File

@ -137,7 +137,7 @@ public class DoCommand {
String ffxProvider = "org.mozilla.ffxcp";
String fenProvider = "org.mozilla.fencp";
private final String prgVersion = "SUTAgentAndroid Version 1.04";
private final String prgVersion = "SUTAgentAndroid Version 1.03";
public enum Command
{
@ -1285,7 +1285,7 @@ private void CancelNotification()
try {
Context appCtx = ctx.createPackageContext(AppName, 0);
ContextWrapper appCtxW = new ContextWrapper(appCtx);
sRet = appCtxW.getApplicationInfo().dataDir;
sRet = appCtxW.getPackageResourcePath();
appCtxW = null;
appCtx = null;
ctx = null;

View File

@ -51,16 +51,15 @@ abstract class BaseTest extends ActivityInstrumentationTestCase2<Activity> {
mSolo = new Solo(getInstrumentation());
mDriver = new FennecNativeDriver(mActivity, mSolo);
mActions = new FennecNativeActions(mActivity, mSolo, getInstrumentation());
mDriver.setLogFile((String)config.get("logfile"));
mAsserter = new FennecNativeAssert();
mAsserter.setLogFile((String)config.get("logfile"));
mAsserter.setTestName(this.getClass().getName());
}
@Override
public void tearDown() throws Exception {
try {
mAsserter.finalize();
mSolo.finalize();
} catch (Throwable e) {
e.printStackTrace();

View File

@ -32,8 +32,8 @@ public class testPan extends BaseTest {
// asserter.ok(i < 1000, "Less than 1000", "Should take less than 1000 drags to get to bottom of the page.");
int frames = mDriver.stopFrameRecording();
mAsserter.dumpLog("__start_report" + Integer.toString(frames) + "__end_report");
mDriver.dumpLog("__start_report" + Integer.toString(frames) + "__end_report");
long msecs = System.currentTimeMillis();
mAsserter.dumpLog("__startTimestamp" + msecs + "__endTimestamp");
mDriver.dumpLog("__startTimestamp" + msecs + "__endTimestamp");
}
}

View File

@ -109,14 +109,9 @@ class RemoteOptions(MochitestOptions):
self.add_option("--robocop", action = "store",
type = "string", dest = "robocop",
help = "name of the .ini file containing the list of tests to run")
help = "use when running robotium tests on native UI")
defaults["robocop"] = ""
self.add_option("--robocop-path", action = "store",
type = "string", dest = "robocopPath",
help = "Path to the folder where robocop.apk is located at. Primarily used for ADB test running")
defaults["robocopPath"] = ""
defaults["remoteTestRoot"] = None
defaults["logFile"] = "mochitest.log"
defaults["autorun"] = True
@ -175,17 +170,6 @@ class RemoteOptions(MochitestOptions):
f.write("%s" % os.getpid())
f.close()
# Robocop specific options
if options.robocop and not os.path.exists(options.robocop):
print "ERROR: Unable to find specified manifest '%s'" % options.robocop
return None
options.robocop = os.path.abspath(options.robocop)
if options.robocopPath and not os.path.exists(os.path.join(options.robocopPath, 'robocop.apk')):
print "ERROR: Unable to find robocop.apk in path '%s'" % options.robocopPath
return None
options.robocopPath = os.path.abspath(options.robocopPath)
return options
def verifyOptions(self, options, mochitest):
@ -207,7 +191,6 @@ class MochiRemote(Mochitest):
_automation = None
_dm = None
localProfile = None
def __init__(self, automation, devmgr, options):
self._automation = automation
@ -295,8 +278,6 @@ class MochiRemote(Mochitest):
self.server.stop()
def buildProfile(self, options):
if self.localProfile:
options.profilePath = self.localProfile
manifest = Mochitest.buildProfile(self, options)
self.localProfile = options.profilePath
self._dm.removeDir(self.remoteProfile)
@ -338,7 +319,7 @@ def main():
auto = RemoteAutomation(dm_none, "fennec")
parser = RemoteOptions(auto, scriptdir)
options, args = parser.parse_args()
if (options.dm_trans == "adb"):
if (options.dm_trans == "adb" or options.robocop):
if (options.deviceIP):
dm = devicemanagerADB.DeviceManagerADB(options.deviceIP, options.devicePort)
else:
@ -370,45 +351,39 @@ def main():
procName = options.app.split('/')[-1]
if (dm.processExist(procName)):
dm.killProcess(procName)
dm.killProcess(procName)
if (options.robocop):
mp = manifestparser.TestManifest(strict=False)
# TODO: pull this in dynamically
mp.read(options.robocop)
robocop_tests = mp.active_tests(exists=False)
mp = manifestparser.TestManifest(strict=False)
# TODO: pull this in dynamically
mp.read('robocop.ini')
robocop_tests = mp.active_tests(exists=False)
fHandle = open("robotium.config", "w")
fHandle.write("profile=%s\n" % (mochitest.remoteProfile))
fHandle.write("logfile=%s\n" % (options.remoteLogFile))
fHandle.close()
deviceRoot = dm.getDeviceRoot()
fHandle = open("robotium.config", "w")
fHandle.write("profile=%s\n" % (mochitest.remoteProfile))
fHandle.write("logfile=%s\n" % (options.remoteLogFile))
fHandle.close()
deviceRoot = dm.getDeviceRoot()
# Note, we are pushing to /sdcard since we have this location hard coded in robocop
dm.pushFile("robotium.config", "/sdcard/robotium.config")
dm.pushFile(os.path.abspath(options.robocop + "/fennec_ids.txt"), "/sdcard/fennec_ids.txt")
options.extraPrefs.append('robocop.logfile="%s/robocop.log"' % deviceRoot)
# Note, we are pushing to /sdcard since we have this location hard coded in robocop
dm.pushFile("robotium.config", "/sdcard/robotium.config")
dm.pushFile(os.path.abspath(options.robocop + "/fennec_ids.txt"), "/sdcard/fennec_ids.txt")
options.extraPrefs.append('robocop.logfile="%s/robocop.log"' % deviceRoot)
if (options.dm_trans == 'adb' and options.robocopPath):
dm.checkCmd(["install", "-r", os.path.join(options.robocopPath, "robocop.apk")])
manifest = mochitest.buildProfile(options)
mochitest.startWebServer(options)
appname = options.app
if (options.dm_trans == 'adb'):
dm.checkCmd(["install", "-r", os.path.join(options.robocop, "robocop.apk")])
for test in robocop_tests:
if options.testPath and options.testPath != test['name']:
continue
options.app = "am"
options.browserArgs = ["instrument", "-w", "-e", "class"]
options.browserArgs.append("%s.tests.%s" % (appname, test['name']))
options.browserArgs.append("org.mozilla.roboexample.test/android.test.InstrumentationTestRunner")
try:
retVal = mochitest.runTests(options)
except:
print "TEST-UNEXPECTED-ERROR | %s | Exception caught while running robocop tests." % sys.exc_info()[1]
mochitest.stopWebServer(options)
mochitest.stopWebSocketServer(options)
sys.exit(1)
cmd = ["shell", "am", "instrument", "-w", "-e", "class"]
cmd.append("%s.tests.%s" % (options.app, test['name']))
cmd.append("org.mozilla.roboexample.test/android.test.InstrumentationTestRunner")
retVal = dm.checkCmd(cmd)
else:
# SUTAgent needs to install robocop and not crash when we launch robocop.
retVal = dm.launchProcess(["am", "instrument", "-w", "org.mozilla.roboexample.test/android.test.InstrumentationTestRunner"])
mochitest.stopWebServer(options)
else:
try:
retVal = mochitest.runTests(options)
@ -417,7 +392,7 @@ def main():
mochitest.stopWebServer(options)
mochitest.stopWebSocketServer(options)
sys.exit(1)
sys.exit(retVal)
if __name__ == "__main__":

View File

@ -76,10 +76,10 @@ RUN_MOCHITEST_REMOTE = \
RUN_MOCHITEST_ROBOTIUM = \
rm -f ./$@.log && \
$(PYTHON) _tests/testing/mochitest/runtestsremote.py --robocop-path build/mobile/robocop \
$(PYTHON) _tests/testing/mochitest/runtestsremote.py --robocop ../../../build/mobile/robocop \
--console-level=INFO --log-file=./$@.log --file-level=INFO $(DM_FLAGS) --dm_trans=adb \
--app=$(TEST_PACKAGE_NAME) --deviceIP=${TEST_DEVICE} --xre-path=${MOZ_HOST_BIN} \
--robocop build/mobile/robocop/robocop.ini $(SYMBOLS_PATH) $(TEST_PATH_ARG) $(EXTRA_TEST_ARGS)
$(SYMBOLS_PATH) $(TEST_PATH_ARG) $(EXTRA_TEST_ARGS)
ifndef NO_FAIL_ON_TEST_ERRORS
define CHECK_TEST_ERROR

View File

@ -37,7 +37,6 @@
# ***** END LICENSE BLOCK ***** */
import re, sys, os
import subprocess
import runxpcshelltests as xpcshell
from automationutils import *
import devicemanager, devicemanagerADB, devicemanagerSUT
@ -67,22 +66,6 @@ class XPCShellRemote(xpcshell.XPCShellTests, object):
self.remoteAPK = self.remoteJoin(self.remoteBinDir, os.path.basename(options.localAPK))
self.remoteDebugger = options.debugger
self.remoteDebuggerArgs = options.debuggerArgs
self.setAppRoot()
def setAppRoot(self):
# Determine the application root directory associated with the package
# name used by the Fennec APK.
self.appRoot = None
packageName = None
if self.options.localAPK:
try:
packageName = subprocess.check_output(["unzip", "-p", self.options.localAPK, "package-name.txt"])
if packageName:
self.appRoot = self.device.getAppRoot(packageName.strip())
except Exception as detail:
print "unable to determine app root: " + detail
pass
return None
def remoteJoin(self, path1, path2):
joined = os.path.join(path1, path2)
@ -220,9 +203,9 @@ class XPCShellRemote(xpcshell.XPCShellTests, object):
shellArgs = "cd "+self.remoteHere
shellArgs += "; LD_LIBRARY_PATH="+self.remoteBinDir
shellArgs += "; export CACHE_PATH="+self.remoteBinDir
if (self.appRoot):
if (self.device.getAppRoot()):
# xpcshell still runs without GRE_HOME; it may not be necessary
shellArgs += "; export GRE_HOME="+self.appRoot
shellArgs += "; export GRE_HOME="+self.device.getAppRoot()
shellArgs += "; export XPCSHELL_TEST_PROFILE_DIR="+self.profileDir
shellArgs += "; "+xpcshell+" "
shellArgs += " ".join(cmd[1:])