diff --git a/build/mobile/devicemanagerADB.py b/build/mobile/devicemanagerADB.py index cffcabc7307..4a89dc600cd 100644 --- a/build/mobile/devicemanagerADB.py +++ b/build/mobile/devicemanagerADB.py @@ -258,6 +258,10 @@ class DeviceManagerADB(DeviceManager): # success: output filename # failure: None def launchProcess(self, cmd, outputFile = "process.txt", cwd = '', env = '', failIfRunning=False): + if cmd[0] == "am": + self.checkCmd(["shell"] + cmd) + return outputFile + acmd = ["shell", "am","start"] cmd = ' '.join(cmd).strip() i = cmd.find(" ") @@ -282,7 +286,7 @@ class DeviceManagerADB(DeviceManager): acmd.append(''.join(['\'',uri, '\''])); print acmd self.checkCmd(acmd) - return outputFile; + return outputFile # external function # returns: diff --git a/build/mobile/remoteautomation.py b/build/mobile/remoteautomation.py index d1d321aa728..7702c295841 100644 --- a/build/mobile/remoteautomation.py +++ b/build/mobile/remoteautomation.py @@ -122,6 +122,11 @@ class RemoteAutomation(Automation): if (self._remoteProfile): profileDir = self._remoteProfile + # Hack for robocop, if app & testURL == None and extraArgs contains the rest of the stuff, lets + # assume extraArgs is all we need + if app == "am" and extraArgs[0] == "instrument": + return app, extraArgs + cmd, args = Automation.buildCommandLine(self, app, debuggerInfo, profileDir, testURL, extraArgs) # Remove -foreground if it exists, if it doesn't this just returns try: diff --git a/build/mobile/robocop/Assert.java.in b/build/mobile/robocop/Assert.java.in index b0d3527b165..287445d583c 100644 --- a/build/mobile/robocop/Assert.java.in +++ b/build/mobile/robocop/Assert.java.in @@ -42,7 +42,9 @@ package @ANDROID_PACKAGE_NAME@; public interface Assert { void dumpLog(String message); void setLogFile(String filename); + void setTestName(String testName); + void finalize(); void ok(boolean condition, String name, String diag); void is(Object a, Object b, String name); void isnot(Object a, Object b, String name); diff --git a/build/mobile/robocop/Driver.java.in b/build/mobile/robocop/Driver.java.in index dc66ca7e68a..d3416f889cc 100644 --- a/build/mobile/robocop/Driver.java.in +++ b/build/mobile/robocop/Driver.java.in @@ -68,14 +68,4 @@ public interface Driver { void startFrameRecording(); int stopFrameRecording(); - void dumpLog(String message); - void setLogFile(String filename); - - void ok(boolean condition, String name, String diag); - void is(Object a, Object b, String name); - void isnot(Object a, Object b, String name); - void todo(boolean condition, String name, String diag); - void todo_is(Object a, Object b, String name); - void todo_isnot(Object a, Object b, String name); - void info(String name, String message); } diff --git a/build/mobile/robocop/FennecNativeAssert.java.in b/build/mobile/robocop/FennecNativeAssert.java.in index f9d8b3a7f47..b4fc44dc782 100644 --- a/build/mobile/robocop/FennecNativeAssert.java.in +++ b/build/mobile/robocop/FennecNativeAssert.java.in @@ -49,6 +49,7 @@ import java.util.ArrayList; import java.util.LinkedList; import java.util.HashMap; import java.util.List; +import java.util.Date; import java.lang.Class; import java.lang.reflect.InvocationTargetException; @@ -77,7 +78,22 @@ public class FennecNativeAssert implements Assert { // If waiting for an event. private boolean asleep = false; - public FennecNativeAssert(){ + // Internal state variables to make logging match up with existing mochitests + private int lineNumber = 0; + private int passed = 0; + private int failed = 0; + private int todo = 0; + + // Used to write the first line of the test file + private boolean logStarted = false; + + // Used to write the test-start/test-end log lines + private String logTestName = ""; + + // Measure the time it takes to run test case + private long startTime = 0; + + public FennecNativeAssert() { } // Write information to a logfile and logcat @@ -110,8 +126,34 @@ public class FennecNativeAssert implements Assert { public void setLogFile(String filename) { logFile = filename; + + //TODO: these two messages are mochitest specific + String message; + if (!logStarted) { + message = Integer.toString(lineNumber++) + " INFO SimpleTest START"; + dumpLog(message); + logStarted = true; + } + + if (logTestName != "") { + message = Integer.toString(lineNumber++) + " INFO TEST-END | " + logTestName; + long diff = (new Date().getTime()) - startTime; + message += " | finished in " + diff + "ms"; + dumpLog(message); + logTestName = ""; + } } + public void setTestName(String testName) + { + String[] nameParts = testName.split("\\."); + logTestName = nameParts[nameParts.length - 1]; + startTime = new Date().getTime(); + + //TODO: this is mochitest specific + String message = Integer.toString(lineNumber++) + " INFO TEST-START | " + logTestName; + dumpLog(message); + } class testInfo { public boolean result; @@ -127,24 +169,23 @@ public class FennecNativeAssert implements Assert { } - - private void _logResult(testInfo test, String passString, String failString) + private void _logMochitestResult(testInfo test, String passString, String failString) { boolean isError = true; String resultString = failString; - if(test.result || test.todo){ + if (test.result || test.todo) { isError = false; } - if(test.result) + if (test.result) { resultString = passString; } - String diag= test.name; - if(test.diag!=null) diag+= " - " + test.diag; + String diag = test.name; + if (test.diag != null) diag += " - " + test.diag; - String message = resultString + " | " + "ROBOCOP" + " | " + diag; - if(isError) { - if(logFile == null) + String message = Integer.toString(lineNumber++) + " INFO " + resultString + " | " + logTestName + " | " + diag; + if (isError) { + if (logFile == null) { assert(false); } @@ -155,11 +196,48 @@ public class FennecNativeAssert implements Assert { else { dumpLog(message); } + + if (test.todo) { + todo++; + } else if (isError) { + failed++; + } else { + passed++; + } + } + + public void finalize() + { + // It appears that we call finalize during cleanup, this might be an invalid assertion. + String message; + + if (logTestName != "") { + long diff = (new Date().getTime()) - startTime; + + //TODO: this is mochitest specific + message = Integer.toString(lineNumber++) + " INFO TEST-END | " + logTestName; + message += " | finished in " + diff + "ms"; + dumpLog(message); + + logTestName = ""; + } + + //TODO: this is all mochitest specific + message = Integer.toString(lineNumber++) + " INFO TEST-START | Shutdown"; + dumpLog(message); + message = Integer.toString(lineNumber++) + " INFO Passed: " + Integer.toString(passed); + dumpLog(message); + message = Integer.toString(lineNumber++) + " INFO Failed: " + Integer.toString(failed); + dumpLog(message); + message = Integer.toString(lineNumber++) + " INFO Todo: " + Integer.toString(todo); + dumpLog(message); + message = Integer.toString(lineNumber++) + " INFO SimpleTest FINISHED"; + dumpLog(message); } public void ok(boolean condition, String name, String diag) { testInfo test = new testInfo(condition, name, diag, false); - _logResult(test, "TEST-PASS", "TEST-UNEXPECTED-FAIL"); + _logMochitestResult(test, "TEST-PASS", "TEST-UNEXPECTED-FAIL"); testList.add(test); } @@ -183,7 +261,7 @@ public class FennecNativeAssert implements Assert { public void todo(boolean condition, String name, String diag) { testInfo test = new testInfo(condition, name, diag, true); - _logResult(test, "TEST-UNEXPECTED-PASS", "TEST-KNOWN-FAIL"); + _logMochitestResult(test, "TEST-UNEXPECTED-PASS", "TEST-KNOWN-FAIL"); testList.add(test); } @@ -207,6 +285,6 @@ public class FennecNativeAssert implements Assert { public void info(String name, String message) { testInfo test = new testInfo(true, name, message, false); - _logResult(test, "TEST-INFO", "INFO FAILED?"); + _logMochitestResult(test, "TEST-INFO", "INFO FAILED?"); } } diff --git a/build/mobile/robocop/FennecNativeDriver.java.in b/build/mobile/robocop/FennecNativeDriver.java.in index dc193328f18..0c45b2e21bd 100644 --- a/build/mobile/robocop/FennecNativeDriver.java.in +++ b/build/mobile/robocop/FennecNativeDriver.java.in @@ -70,7 +70,6 @@ public class FennecNativeDriver implements Driver { private HashMap locators = null; private Activity activity; private Solo solo; - private String logFile; // Objects for reflexive access of fennec classes. private ClassLoader classLoader; @@ -83,9 +82,6 @@ public class FennecNativeDriver implements Driver { private Method _startFrameRecording; private Method _stopFrameRecording; - - private LinkedList testList = new LinkedList(); - public FennecNativeDriver(Activity activity, Solo robocop){ this.activity = activity; this.solo = robocop; @@ -286,38 +282,6 @@ public class FennecNativeDriver implements Driver { return text.toString(); } - // Write information to a logfile and logcat - public void dumpLog(String message) - { - File file = new File(logFile); - BufferedWriter bw = null; - - try { - bw = new BufferedWriter(new FileWriter(logFile, true)); - bw.write(message); - bw.newLine(); - } catch(IOException e) { - e.printStackTrace(); - } finally { - try { - if (bw != null) { - bw.flush(); - bw.close(); - } - } catch (IOException ex) { - ex.printStackTrace(); - } - } - - Log.i("Robocop", message); - } - - // Set the filename used for dumpLog. - public void setLogFile(String filename) - { - logFile = filename; - } - // Takes a string of "key=value" pairs split by \n and creates a hash table. public static HashMap convertTextToTable(String data) { @@ -330,95 +294,4 @@ public class FennecNativeDriver implements Driver { } return retVal; } - - class testInfo { - public boolean result; - public String name; - public String diag; - public boolean todo; - public testInfo(boolean r, String n, String d, boolean t) { - result = r; - name = n; - diag = d; - todo = t; - } - - } - - - private void _logResult(testInfo test, String passString, String failString) - { - boolean isError = true; - String resultString = failString; - if(test.result || test.todo){ - isError = false; - } - if(test.result) - { - resultString = passString; - } - String diag= test.name; - if(test.diag!=null) diag+= " - " + test.diag; - - String message = resultString + " | " + "ROBOCOP" + " | " + diag; - if(isError) { - dumpLog(message); - } - else { - dumpLog(message); - } - } - - public void ok(boolean condition, String name, String diag) { - testInfo test = new testInfo(condition, name, diag, false); - _logResult(test, "TEST-PASS", "TEST-UNEXPECTED-FAIL"); - testList.add(test); - } - - public void is(Object a, Object b, String name) { - boolean pass = a.equals(b); - String diag = "got " + a.toString() + ", expected " + b.toString(); - if(pass) { - diag = a.toString() + " should equal " + b.toString(); - } - ok(pass, name, diag); - } - - public void isnot(Object a, Object b, String name) { - boolean pass = !a.equals(b); - String diag = "didn't expect " + a.toString() + ", but got it"; - if(pass) { - diag = a.toString() + " should not equal " + b.toString(); - } - ok(pass, name, diag); - } - - public void todo(boolean condition, String name, String diag) { - testInfo test = new testInfo(condition, name, diag, true); - _logResult(test, "TEST-UNEXPECTED-PASS", "TEST-KNOWN-FAIL"); - testList.add(test); - } - - public void todo_is(Object a, Object b, String name) { - boolean pass = a.equals(b); - String diag = "got " + a.toString() + ", expected " + b.toString(); - if(pass) { - diag = a.toString() + " should equal " + b.toString(); - } - todo(pass, name, diag); - } - - public void todo_isnot(Object a, Object b, String name) { - boolean pass = !a.equals(b); - String diag = "didn't expect " + a.toString() + ", but got it"; - if(pass) { - diag = a.toString() + " should not equal " + b.toString(); - } - todo(pass, name, diag); - } - - public void info(String name, String message) { - testInfo test = new testInfo(true, name, message, false); - _logResult(test, "TEST-INFO", "INFO FAILED?"); - } } diff --git a/mobile/android/base/tests/BaseTest.java.in b/mobile/android/base/tests/BaseTest.java.in index 2ee3c9fb258..31f9a5efb99 100644 --- a/mobile/android/base/tests/BaseTest.java.in +++ b/mobile/android/base/tests/BaseTest.java.in @@ -51,15 +51,16 @@ abstract class BaseTest extends ActivityInstrumentationTestCase2 { mSolo = new Solo(getInstrumentation()); mDriver = new FennecNativeDriver(mActivity, mSolo); mActions = new FennecNativeActions(mActivity, mSolo, getInstrumentation()); - mDriver.setLogFile((String)config.get("logfile")); mAsserter = new FennecNativeAssert(); mAsserter.setLogFile((String)config.get("logfile")); + mAsserter.setTestName(this.getClass().getName()); } @Override public void tearDown() throws Exception { try { + mAsserter.finalize(); mSolo.finalize(); } catch (Throwable e) { e.printStackTrace(); diff --git a/mobile/android/base/tests/testPan.java.in b/mobile/android/base/tests/testPan.java.in index ebcea643b93..05f7f896bec 100644 --- a/mobile/android/base/tests/testPan.java.in +++ b/mobile/android/base/tests/testPan.java.in @@ -32,8 +32,8 @@ public class testPan extends BaseTest { // asserter.ok(i < 1000, "Less than 1000", "Should take less than 1000 drags to get to bottom of the page."); int frames = mDriver.stopFrameRecording(); - mDriver.dumpLog("__start_report" + Integer.toString(frames) + "__end_report"); + mAsserter.dumpLog("__start_report" + Integer.toString(frames) + "__end_report"); long msecs = System.currentTimeMillis(); - mDriver.dumpLog("__startTimestamp" + msecs + "__endTimestamp"); + mAsserter.dumpLog("__startTimestamp" + msecs + "__endTimestamp"); } } diff --git a/testing/mochitest/runtestsremote.py b/testing/mochitest/runtestsremote.py index 63ca42773e9..56bb70aa472 100644 --- a/testing/mochitest/runtestsremote.py +++ b/testing/mochitest/runtestsremote.py @@ -191,6 +191,7 @@ class MochiRemote(Mochitest): _automation = None _dm = None + localProfile = None def __init__(self, automation, devmgr, options): self._automation = automation @@ -278,6 +279,8 @@ class MochiRemote(Mochitest): self.server.stop() def buildProfile(self, options): + if self.localProfile: + options.profilePath = self.localProfile manifest = Mochitest.buildProfile(self, options) self.localProfile = options.profilePath self._dm.removeDir(self.remoteProfile) @@ -319,7 +322,7 @@ def main(): auto = RemoteAutomation(dm_none, "fennec") parser = RemoteOptions(auto, scriptdir) options, args = parser.parse_args() - if (options.dm_trans == "adb" or options.robocop): + if (options.dm_trans == "adb"): if (options.deviceIP): dm = devicemanagerADB.DeviceManagerADB(options.deviceIP, options.devicePort) else: @@ -351,39 +354,42 @@ def main(): procName = options.app.split('/')[-1] if (dm.processExist(procName)): - dm.killProcess(procName) + dm.killProcess(procName) if (options.robocop): - mp = manifestparser.TestManifest(strict=False) - # TODO: pull this in dynamically - mp.read('robocop.ini') - robocop_tests = mp.active_tests(exists=False) + mp = manifestparser.TestManifest(strict=False) + # TODO: pull this in dynamically + mp.read('robocop.ini') + robocop_tests = mp.active_tests(exists=False) - fHandle = open("robotium.config", "w") - fHandle.write("profile=%s\n" % (mochitest.remoteProfile)) - fHandle.write("logfile=%s\n" % (options.remoteLogFile)) - fHandle.close() - deviceRoot = dm.getDeviceRoot() + fHandle = open("robotium.config", "w") + fHandle.write("profile=%s\n" % (mochitest.remoteProfile)) + fHandle.write("logfile=%s\n" % (options.remoteLogFile)) + fHandle.close() + deviceRoot = dm.getDeviceRoot() - # Note, we are pushing to /sdcard since we have this location hard coded in robocop - dm.pushFile("robotium.config", "/sdcard/robotium.config") - dm.pushFile(os.path.abspath(options.robocop + "/fennec_ids.txt"), "/sdcard/fennec_ids.txt") - options.extraPrefs.append('robocop.logfile="%s/robocop.log"' % deviceRoot) + # Note, we are pushing to /sdcard since we have this location hard coded in robocop + dm.pushFile("robotium.config", "/sdcard/robotium.config") + dm.pushFile(os.path.abspath(options.robocop + "/fennec_ids.txt"), "/sdcard/fennec_ids.txt") + options.extraPrefs.append('robocop.logfile="%s/robocop.log"' % deviceRoot) - manifest = mochitest.buildProfile(options) - mochitest.startWebServer(options) + if (options.dm_trans == 'adb'): + dm.checkCmd(["install", "-r", os.path.join(options.robocop, "robocop.apk")]) - if (options.dm_trans == 'adb'): - dm.checkCmd(["install", "-r", os.path.join(options.robocop, "robocop.apk")]) + appname = options.app for test in robocop_tests: - cmd = ["shell", "am", "instrument", "-w", "-e", "class"] - cmd.append("%s.tests.%s" % (options.app, test['name'])) - cmd.append("org.mozilla.roboexample.test/android.test.InstrumentationTestRunner") - retVal = dm.checkCmd(cmd) - else: - # SUTAgent needs to install robocop and not crash when we launch robocop. - retVal = dm.launchProcess(["am", "instrument", "-w", "org.mozilla.roboexample.test/android.test.InstrumentationTestRunner"]) - mochitest.stopWebServer(options) + options.app = "am" + options.browserArgs = ["instrument", "-w", "-e", "class"] + options.browserArgs.append("%s.tests.%s" % (appname, test['name'])) + options.browserArgs.append("org.mozilla.roboexample.test/android.test.InstrumentationTestRunner") + + try: + retVal = mochitest.runTests(options) + except: + print "TEST-UNEXPECTED-ERROR | %s | Exception caught while running robocop tests." % sys.exc_info()[1] + mochitest.stopWebServer(options) + mochitest.stopWebSocketServer(options) + sys.exit(1) else: try: retVal = mochitest.runTests(options) @@ -392,7 +398,7 @@ def main(): mochitest.stopWebServer(options) mochitest.stopWebSocketServer(options) sys.exit(1) - + sys.exit(retVal) if __name__ == "__main__":