Bug 812395 - Make emulator fail gracefully during errors for install_gecko, r=ahal

layout/tools/reftest/runreftestb2g.py

@@ -480,7 +480,7 @@ def main(args=sys.argv[1:]):
         kwargs['port'] = int(port)
     if options.geckoPath:
         kwargs['gecko_path'] = options.geckoPath
-    marionette = Marionette(**kwargs)
+    marionette = Marionette.getMarionetteOrExit(**kwargs)
     auto.marionette = marionette
 
     # create the DeviceManager

testing/marionette/client/marionette/emulator.py

@@ -13,9 +13,11 @@ import platform
 import shutil
 import socket
 import subprocess
+import sys
 from telnetlib import Telnet
 import tempfile
 import time
+import traceback
 
 from emulator_battery import EmulatorBattery
 from emulator_geo import EmulatorGeo
@@ -387,33 +389,43 @@ waitFor(
         push_attempts = 10
 
         print 'installing gecko binaries...'
-        # need to remount so we can write to /system/b2g
-        self._run_adb(['remount'])
-        for root, dirs, files in os.walk(gecko_path):
-            for filename in files:
-                rel_path = os.path.relpath(os.path.join(root, filename), gecko_path)
-                system_b2g_file = os.path.join('/system/b2g', rel_path)
-                for retry in range(1, push_attempts+1):
-                    print 'pushing', system_b2g_file, '(attempt %s of %s)' % (retry, push_attempts)
-                    try:
-                        self.dm.pushFile(os.path.join(root, filename), system_b2g_file)
-                        break
-                    except DMError:
-                        if retry == push_attempts:
-                            raise
 
-        print 'restarting B2G'
-        # see bug 809437 for the path that lead to this madness
-        time.sleep(5)
-        self.dm.shellCheckOutput(['stop', 'b2g'])
-        time.sleep(10)
-        self.dm.shellCheckOutput(['start', 'b2g'])
-        time.sleep(5)
+        try:
+            # need to remount so we can write to /system/b2g
+            self._run_adb(['remount'])
+            for root, dirs, files in os.walk(gecko_path):
+                for filename in files:
+                    rel_path = os.path.relpath(os.path.join(root, filename), gecko_path)
+                    system_b2g_file = os.path.join('/system/b2g', rel_path)
+                    for retry in range(1, push_attempts+1):
+                        print 'pushing', system_b2g_file, '(attempt %s of %s)' % (retry, push_attempts)
+                        try:
+                            self.dm.pushFile(os.path.join(root, filename), system_b2g_file)
+                            break
+                        except DMError:
+                            if retry == push_attempts:
+                                raise
 
-        if not self.wait_for_port():
-            raise TimeoutException("Timeout waiting for marionette on port '%s'" % self.marionette_port)
-        self.wait_for_system_message(marionette)
+            print 'restarting B2G'
+            # see bug 809437 for the path that lead to this madness
+            self.dm.shellCheckOutput(['stop', 'b2g'])
+            time.sleep(10)
+            self.dm.shellCheckOutput(['start', 'b2g'])
 
+            if not self.wait_for_port():
+                raise TimeoutException("Timeout waiting for marionette on port '%s'" % self.marionette_port)
+            self.wait_for_system_message(marionette)
+
+        except (DMError, MarionetteException):
+            # Bug 812395 - raise a single exception type for these so we can
+            # explicitly catch them elsewhere.
+
+            # print exception, but hide from mozharness error detection
+            exc = traceback.format_exc()
+            exc = exc.replace('Traceback', '_traceback')
+            print exc
+
+            raise InstallGeckoError("unable to restart B2G after installing gecko")
 
     def rotate_log(self, srclog, index=1):
         """ Rotate a logfile, by recursively rotating logs further in the sequence,

testing/marionette/client/marionette/errors.py

@@ -17,6 +17,9 @@ class MarionetteException(Exception):
         else:
             return str(self.msg)
 
+class InstallGeckoError(MarionetteException):
+    pass
+
 class TimeoutException(MarionetteException):
     pass
 

testing/marionette/client/marionette/marionette.py

@@ -3,6 +3,8 @@
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 import socket
+import sys
+import traceback
 
 from client import MarionetteClient
 from application_cache import ApplicationCache
@@ -11,6 +13,7 @@ from errors import *
 from emulator import Emulator
 from geckoinstance import GeckoInstance
 
+
 class HTMLElement(object):
 
     CLASS = "class name"
@@ -74,7 +77,7 @@ class HTMLElement(object):
 
     def is_displayed(self):
         return self.marionette._send_message('isElementDisplayed', 'value', element=self.id)
-     
+
     @property
     def size(self):
         return self.marionette._send_message('getElementSize', 'value', element=self.id)
@@ -155,6 +158,26 @@ class Marionette(object):
         for qemu in self.extra_emulators:
             qemu.emulator.close()
 
+    @classmethod
+    def getMarionetteOrExit(cls, *args, **kwargs):
+        try:
+            m = cls(*args, **kwargs)
+            return m
+        except InstallGeckoError:
+            # Bug 812395 - the process of installing gecko into the emulator
+            # and then restarting B2G tickles some bug in the emulator/b2g
+            # that intermittently causes B2G to fail to restart.  To work
+            # around this in TBPL runs, we will fail gracefully from this
+            # error so that the mozharness script can try the run again.
+
+            # This string will get caught by mozharness and will cause it
+            # to retry the tests.
+            print "Error installing gecko!"
+
+            # Exit without a normal exception to prevent mozharness from
+            # flagging the error.
+            sys.exit()
+
     def _send_message(self, command, response_key, **kwargs):
         if not self.session and command not in ('newSession', 'getStatus'):
             raise MarionetteException(message="Please start a session")

testing/marionette/client/marionette/runtests.py

@@ -8,13 +8,11 @@ import inspect
 import logging
 from optparse import OptionParser
 import os
-import types
 import unittest
 import socket
 import sys
 import time
 import platform
-import weakref
 import xml.dom.minidom as dom
 
 from manifestparser import TestManifest
@@ -276,7 +274,8 @@ class MarionetteTestRunner(object):
         elif self.address:
             host, port = self.address.split(':')
             if self.emulator:
-                self.marionette = Marionette(host=host, port=int(port),
+                self.marionette = Marionette.getMarionetteOrExit(
+                                             host=host, port=int(port),
                                              connectToRunningEmulator=True,
                                              homedir=self.homedir,
                                              baseurl=self.baseurl,
@@ -287,7 +286,8 @@ class MarionetteTestRunner(object):
                                              port=int(port),
                                              baseurl=self.baseurl)
         elif self.emulator:
-            self.marionette = Marionette(emulator=self.emulator,
+            self.marionette = Marionette.getMarionetteOrExit(
+                                         emulator=self.emulator,
                                          emulatorBinary=self.emulatorBinary,
                                          emulatorImg=self.emulatorImg,
                                          emulator_res=self.emulator_res,

testing/mochitest/runtestsb2g.py

@@ -482,7 +482,8 @@ def main():
         host,port = options.marionette.split(':')
         kwargs['host'] = host
         kwargs['port'] = int(port)
-    marionette = Marionette(**kwargs)
+
+    marionette = Marionette.getMarionetteOrExit(**kwargs)
 
     auto.marionette = marionette