Bug 715309 - robocop needs to have a more uniform log format. r=gbrown

build/mobile/devicemanagerADB.py

@@ -258,6 +258,10 @@ class DeviceManagerADB(DeviceManager):
   #  success: output filename
   #  failure: None
   def launchProcess(self, cmd, outputFile = "process.txt", cwd = '', env = '', failIfRunning=False):
+    if cmd[0] == "am":
+      self.checkCmd(["shell"] + cmd)
+      return outputFile
+
     acmd = ["shell", "am","start"]
     cmd = ' '.join(cmd).strip()
     i = cmd.find(" ")
@@ -282,7 +286,7 @@ class DeviceManagerADB(DeviceManager):
       acmd.append(''.join(['\'',uri, '\'']));
     print acmd
     self.checkCmd(acmd)
-    return outputFile;
+    return outputFile
 
   # external function
   # returns:

build/mobile/remoteautomation.py

@@ -122,6 +122,11 @@ class RemoteAutomation(Automation):
         if (self._remoteProfile):
             profileDir = self._remoteProfile
 
+        # Hack for robocop, if app & testURL == None and extraArgs contains the rest of the stuff, lets 
+        # assume extraArgs is all we need
+        if app == "am" and extraArgs[0] == "instrument":
+            return app, extraArgs
+ 
         cmd, args = Automation.buildCommandLine(self, app, debuggerInfo, profileDir, testURL, extraArgs)
         # Remove -foreground if it exists, if it doesn't this just returns
         try:

build/mobile/robocop/Assert.java.in

@@ -42,7 +42,9 @@ package @ANDROID_PACKAGE_NAME@;
 public interface Assert {
   void dumpLog(String message);
   void setLogFile(String filename);
+  void setTestName(String testName);
 
+  void finalize();
   void ok(boolean condition, String name, String diag);
   void is(Object a, Object b, String name);
   void isnot(Object a, Object b, String name);

build/mobile/robocop/Driver.java.in

@@ -68,14 +68,4 @@ public interface Driver {
 
   void startFrameRecording();
   int stopFrameRecording();
-  void dumpLog(String message);
-  void setLogFile(String filename);
-
-  void ok(boolean condition, String name, String diag);
-  void is(Object a, Object b, String name);
-  void isnot(Object a, Object b, String name);
-  void todo(boolean condition, String name, String diag);
-  void todo_is(Object a, Object b, String name);
-  void todo_isnot(Object a, Object b, String name);
-  void info(String name, String message);
 }

build/mobile/robocop/FennecNativeAssert.java.in

@@ -49,6 +49,7 @@ import java.util.ArrayList;
 import java.util.LinkedList;
 import java.util.HashMap;
 import java.util.List;
+import java.util.Date;
 
 import java.lang.Class;
 import java.lang.reflect.InvocationTargetException;
@@ -77,7 +78,22 @@ public class FennecNativeAssert implements Assert {
   // If waiting for an event.
   private boolean asleep = false;
 
-  public FennecNativeAssert(){
+  // Internal state variables to make logging match up with existing mochitests
+  private int lineNumber = 0;
+  private int passed = 0;
+  private int failed = 0;
+  private int todo = 0;
+  
+  // Used to write the first line of the test file
+  private boolean logStarted = false;
+
+  // Used to write the test-start/test-end log lines
+  private String logTestName = "";
+
+  // Measure the time it takes to run test case
+  private long startTime = 0;
+
+  public FennecNativeAssert() {
   }
 
   // Write information to a logfile and logcat
@@ -110,8 +126,34 @@ public class FennecNativeAssert implements Assert {
   public void setLogFile(String filename)
   {
     logFile = filename;
+
+    //TODO: these two messages are mochitest specific
+    String message;
+    if (!logStarted) {
+      message = Integer.toString(lineNumber++) + " INFO SimpleTest START";
+      dumpLog(message);
+      logStarted = true;
+    }
+
+    if (logTestName != "") {
+      message = Integer.toString(lineNumber++) + " INFO TEST-END | " + logTestName;
+      long diff = (new Date().getTime()) - startTime;
+      message += " | finished in " + diff + "ms";
+      dumpLog(message);
+      logTestName = "";
+    }
   }
 
+  public void setTestName(String testName)
+  {
+    String[] nameParts = testName.split("\\.");
+    logTestName = nameParts[nameParts.length - 1];
+    startTime = new Date().getTime();
+
+    //TODO: this is mochitest specific
+    String message = Integer.toString(lineNumber++) + " INFO TEST-START | " + logTestName;
+    dumpLog(message);
+  }
 
   class testInfo {
     public boolean result;
@@ -127,24 +169,23 @@ public class FennecNativeAssert implements Assert {
 
   }
 
-
+  private void _logMochitestResult(testInfo test, String passString, String failString)
-  private void _logResult(testInfo test, String passString, String failString)
   {
     boolean isError = true;
     String resultString = failString;
-    if(test.result || test.todo){
+    if (test.result || test.todo) {
       isError = false;
     }
-    if(test.result)
+    if (test.result)
     {
       resultString = passString;
     }
-    String diag= test.name;
+    String diag = test.name;
-    if(test.diag!=null) diag+= " - " + test.diag;
+    if (test.diag != null) diag += " - " + test.diag;
 
-    String message = resultString + " | " + "ROBOCOP" + " | " + diag;
+    String message = Integer.toString(lineNumber++) + " INFO " + resultString + " | " + logTestName + " | " + diag;
-    if(isError) {
+    if (isError) {
-      if(logFile == null)
+      if (logFile == null)
       {
         assert(false);
       }
@@ -155,11 +196,48 @@ public class FennecNativeAssert implements Assert {
     else {
       dumpLog(message);
     }
+
+    if (test.todo) {
+      todo++;
+    } else if (isError) {
+      failed++;
+    } else {
+      passed++;
+    }
+  }
+
+  public void finalize()
+  {
+    // It appears that we call finalize during cleanup, this might be an invalid assertion.
+    String message;
+
+    if (logTestName != "") {
+      long diff = (new Date().getTime()) - startTime;
+
+      //TODO: this is mochitest specific
+      message = Integer.toString(lineNumber++) + " INFO TEST-END | " + logTestName;
+      message += " | finished in " + diff + "ms";
+      dumpLog(message);
+
+      logTestName = "";
+    }
+
+    //TODO: this is all mochitest specific
+    message = Integer.toString(lineNumber++) + " INFO TEST-START | Shutdown";
+    dumpLog(message);
+    message = Integer.toString(lineNumber++) + " INFO Passed: " + Integer.toString(passed);
+    dumpLog(message);
+    message = Integer.toString(lineNumber++) + " INFO Failed: " + Integer.toString(failed);
+    dumpLog(message);
+    message = Integer.toString(lineNumber++) + " INFO Todo: " + Integer.toString(todo);
+    dumpLog(message);
+    message = Integer.toString(lineNumber++) + " INFO SimpleTest FINISHED";
+    dumpLog(message);
   }
 
   public void ok(boolean condition, String name, String diag) {
     testInfo test = new testInfo(condition, name, diag, false);
-    _logResult(test, "TEST-PASS", "TEST-UNEXPECTED-FAIL");
+    _logMochitestResult(test, "TEST-PASS", "TEST-UNEXPECTED-FAIL");
     testList.add(test);
   }
 
@@ -183,7 +261,7 @@ public class FennecNativeAssert implements Assert {
 
   public void todo(boolean condition, String name, String diag) {
     testInfo test = new testInfo(condition, name, diag, true);
-    _logResult(test, "TEST-UNEXPECTED-PASS", "TEST-KNOWN-FAIL");
+    _logMochitestResult(test, "TEST-UNEXPECTED-PASS", "TEST-KNOWN-FAIL");
     testList.add(test);
   }
 
@@ -207,6 +285,6 @@ public class FennecNativeAssert implements Assert {
 
   public void info(String name, String message) {
     testInfo test = new testInfo(true, name, message, false);
-    _logResult(test, "TEST-INFO", "INFO FAILED?");
+    _logMochitestResult(test, "TEST-INFO", "INFO FAILED?");
   }
 }

build/mobile/robocop/FennecNativeDriver.java.in

@@ -70,7 +70,6 @@ public class FennecNativeDriver implements Driver {
   private HashMap locators = null;
   private Activity activity;
   private Solo solo;
-  private String logFile;
 
   // Objects for reflexive access of fennec classes.
   private ClassLoader classLoader;
@@ -83,9 +82,6 @@ public class FennecNativeDriver implements Driver {
   private Method _startFrameRecording;
   private Method _stopFrameRecording;
 
-
-  private LinkedList<testInfo> testList = new LinkedList<testInfo>();
-
   public FennecNativeDriver(Activity activity, Solo robocop){
     this.activity = activity;
     this.solo = robocop;
@@ -286,38 +282,6 @@ public class FennecNativeDriver implements Driver {
     return text.toString();  
   }
 
-  // Write information to a logfile and logcat
-  public void dumpLog(String message)
-  {
-    File file = new File(logFile);
-    BufferedWriter bw = null;
-
-    try {
-      bw = new BufferedWriter(new FileWriter(logFile, true));
-      bw.write(message);
-      bw.newLine();
-    } catch(IOException e) {
-      e.printStackTrace();
-    } finally {
-      try {
-        if (bw != null) {
-          bw.flush();
-          bw.close();
-        }
-      } catch (IOException ex) {
-        ex.printStackTrace();
-      }
-    }
-
-    Log.i("Robocop", message);
-  }
-
-  // Set the filename used for dumpLog.
-  public void setLogFile(String filename)
-  {
-    logFile = filename;
-  }
-
   // Takes a string of "key=value" pairs split by \n and creates a hash table.
   public static HashMap convertTextToTable(String data)
   {
@@ -330,95 +294,4 @@ public class FennecNativeDriver implements Driver {
     }
     return retVal;
   }
-
-  class testInfo {
-    public boolean result;
-    public String name;
-    public String diag;
-    public boolean todo;
-    public testInfo(boolean r, String n, String d, boolean t) {
-      result = r;
-      name = n;
-      diag = d;
-      todo = t;
-    }
-
-  }
-
-
-  private void _logResult(testInfo test, String passString, String failString)
-  {
-    boolean isError = true;
-    String resultString = failString;
-    if(test.result || test.todo){
-      isError = false;
-    }
-    if(test.result)
-    {
-      resultString = passString;
-    }
-    String diag= test.name;
-    if(test.diag!=null) diag+= " - " + test.diag;
-
-    String message = resultString + " | " + "ROBOCOP" + " | " + diag;
-    if(isError) {
-      dumpLog(message);
-    }
-    else {
-      dumpLog(message);
-    }
-  }
-
-  public void ok(boolean condition, String name, String diag) {
-    testInfo test = new testInfo(condition, name, diag, false);
-    _logResult(test, "TEST-PASS", "TEST-UNEXPECTED-FAIL");
-    testList.add(test);
-  }
-
-  public void is(Object a, Object b, String name) {
-    boolean pass = a.equals(b);
-    String diag = "got " + a.toString() + ", expected " + b.toString();
-    if(pass) {
-      diag = a.toString() + " should equal " + b.toString();
-    }
-    ok(pass, name, diag);
-  }
-  
-  public void isnot(Object a, Object b, String name) {
-    boolean pass = !a.equals(b);
-    String diag = "didn't expect " + a.toString() + ", but got it";
-    if(pass) {
-      diag = a.toString() + " should not equal " + b.toString();
-    }
-    ok(pass, name, diag);
-  }
-
-  public void todo(boolean condition, String name, String diag) {
-    testInfo test = new testInfo(condition, name, diag, true);
-    _logResult(test, "TEST-UNEXPECTED-PASS", "TEST-KNOWN-FAIL");
-    testList.add(test);
-  }
-
-  public void todo_is(Object a, Object b, String name) {
-    boolean pass = a.equals(b);
-    String diag = "got " + a.toString() + ", expected " + b.toString();
-    if(pass) {
-      diag = a.toString() + " should equal " + b.toString();
-    }
-    todo(pass, name, diag);
-  }
-  
-  public void todo_isnot(Object a, Object b, String name) {
-    boolean pass = !a.equals(b);
-    String diag = "didn't expect " + a.toString() + ", but got it";
-    if(pass) {
-      diag = a.toString() + " should not equal " + b.toString();
-    }
-    todo(pass, name, diag);
-  }
-
-  public void info(String name, String message) {
-    testInfo test = new testInfo(true, name, message, false);
-    _logResult(test, "TEST-INFO", "INFO FAILED?");
-  }
 }

mobile/android/base/tests/BaseTest.java.in

@@ -51,15 +51,16 @@ abstract class BaseTest extends ActivityInstrumentationTestCase2<Activity> {
         mSolo = new Solo(getInstrumentation());
         mDriver = new FennecNativeDriver(mActivity, mSolo);
         mActions = new FennecNativeActions(mActivity, mSolo, getInstrumentation());
-        mDriver.setLogFile((String)config.get("logfile"));
 
         mAsserter = new FennecNativeAssert();
         mAsserter.setLogFile((String)config.get("logfile"));
+        mAsserter.setTestName(this.getClass().getName());
     }
 
     @Override
     public void tearDown() throws Exception {
         try {
+            mAsserter.finalize();
             mSolo.finalize();
         } catch (Throwable e) {
             e.printStackTrace();

mobile/android/base/tests/testPan.java.in

@@ -32,8 +32,8 @@ public class testPan extends BaseTest {
         // asserter.ok(i < 1000, "Less than 1000", "Should take less than 1000 drags to get to bottom of the page.");
 
         int frames = mDriver.stopFrameRecording();
-        mDriver.dumpLog("__start_report" + Integer.toString(frames) + "__end_report");
+        mAsserter.dumpLog("__start_report" + Integer.toString(frames) + "__end_report");
         long msecs = System.currentTimeMillis();
-        mDriver.dumpLog("__startTimestamp" + msecs + "__endTimestamp");
+        mAsserter.dumpLog("__startTimestamp" + msecs + "__endTimestamp");
     }	
 }

testing/mochitest/runtestsremote.py

@@ -191,6 +191,7 @@ class MochiRemote(Mochitest):
 
     _automation = None
     _dm = None
+    localProfile = None
 
     def __init__(self, automation, devmgr, options):
         self._automation = automation
@@ -278,6 +279,8 @@ class MochiRemote(Mochitest):
         self.server.stop()
         
     def buildProfile(self, options):
+        if self.localProfile:
+            options.profilePath = self.localProfile
         manifest = Mochitest.buildProfile(self, options)
         self.localProfile = options.profilePath
         self._dm.removeDir(self.remoteProfile)
@@ -319,7 +322,7 @@ def main():
     auto = RemoteAutomation(dm_none, "fennec")
     parser = RemoteOptions(auto, scriptdir)
     options, args = parser.parse_args()
-    if (options.dm_trans == "adb" or options.robocop):
+    if (options.dm_trans == "adb"):
         if (options.deviceIP):
             dm = devicemanagerADB.DeviceManagerADB(options.deviceIP, options.devicePort)
         else:
@@ -351,39 +354,42 @@ def main():
 
     procName = options.app.split('/')[-1]
     if (dm.processExist(procName)):
-      dm.killProcess(procName)
+        dm.killProcess(procName)
     
     if (options.robocop):
-      mp = manifestparser.TestManifest(strict=False)
+        mp = manifestparser.TestManifest(strict=False)
-      # TODO: pull this in dynamically
+        # TODO: pull this in dynamically
-      mp.read('robocop.ini')
+        mp.read('robocop.ini')
-      robocop_tests = mp.active_tests(exists=False)
+        robocop_tests = mp.active_tests(exists=False)
 
-      fHandle = open("robotium.config", "w")
+        fHandle = open("robotium.config", "w")
-      fHandle.write("profile=%s\n" % (mochitest.remoteProfile))
+        fHandle.write("profile=%s\n" % (mochitest.remoteProfile))
-      fHandle.write("logfile=%s\n" % (options.remoteLogFile))
+        fHandle.write("logfile=%s\n" % (options.remoteLogFile))
-      fHandle.close()
+        fHandle.close()
-      deviceRoot = dm.getDeviceRoot()
+        deviceRoot = dm.getDeviceRoot()
       
-      # Note, we are pushing to /sdcard since we have this location hard coded in robocop
+        # Note, we are pushing to /sdcard since we have this location hard coded in robocop
-      dm.pushFile("robotium.config", "/sdcard/robotium.config")
+        dm.pushFile("robotium.config", "/sdcard/robotium.config")
-      dm.pushFile(os.path.abspath(options.robocop + "/fennec_ids.txt"), "/sdcard/fennec_ids.txt")
+        dm.pushFile(os.path.abspath(options.robocop + "/fennec_ids.txt"), "/sdcard/fennec_ids.txt")
-      options.extraPrefs.append('robocop.logfile="%s/robocop.log"' % deviceRoot)
+        options.extraPrefs.append('robocop.logfile="%s/robocop.log"' % deviceRoot)
 
-      manifest = mochitest.buildProfile(options)
+        if (options.dm_trans == 'adb'):
-      mochitest.startWebServer(options)
+          dm.checkCmd(["install", "-r", os.path.join(options.robocop, "robocop.apk")])
 
-      if (options.dm_trans == 'adb'):
+        appname = options.app
-        dm.checkCmd(["install", "-r", os.path.join(options.robocop, "robocop.apk")])
         for test in robocop_tests:
-          cmd = ["shell", "am", "instrument", "-w", "-e", "class"]
+            options.app = "am"
-          cmd.append("%s.tests.%s" % (options.app, test['name']))
+            options.browserArgs = ["instrument", "-w", "-e", "class"]
-          cmd.append("org.mozilla.roboexample.test/android.test.InstrumentationTestRunner")
+            options.browserArgs.append("%s.tests.%s" % (appname, test['name']))
-          retVal = dm.checkCmd(cmd)
+            options.browserArgs.append("org.mozilla.roboexample.test/android.test.InstrumentationTestRunner")
-      else:
+
-        # SUTAgent needs to install robocop and not crash when we launch robocop.
+            try:
-        retVal = dm.launchProcess(["am", "instrument", "-w", "org.mozilla.roboexample.test/android.test.InstrumentationTestRunner"])
+                retVal = mochitest.runTests(options)
-      mochitest.stopWebServer(options)
+            except:
+                print "TEST-UNEXPECTED-ERROR | %s | Exception caught while running robocop tests." % sys.exc_info()[1]
+                mochitest.stopWebServer(options)
+                mochitest.stopWebSocketServer(options)
+                sys.exit(1)
     else:
       try:
         retVal = mochitest.runTests(options)
@@ -392,7 +398,7 @@ def main():
         mochitest.stopWebServer(options)
         mochitest.stopWebSocketServer(options)
         sys.exit(1)
-      
+
     sys.exit(retVal)
         
 if __name__ == "__main__":