Bug 800110 - Mirror mozbase -> m-c for bug 800097 @ da60c88b8c ;r=ahal,r=wlach

layout/tools/reftest/Makefile.in

@@ -63,6 +63,7 @@ _HARNESS_FILES = \
   $(topsrcdir)/testing/mozbase/mozdevice/mozdevice/devicemanager.py \
   $(topsrcdir)/testing/mozbase/mozdevice/mozdevice/devicemanagerADB.py \
   $(topsrcdir)/testing/mozbase/mozdevice/mozdevice/devicemanagerSUT.py \
+  $(topsrcdir)/testing/mozbase/mozdevice/mozdevice/Zeroconf.py \
   $(topsrcdir)/build/mobile/b2gautomation.py \
   $(topsrcdir)/build/automationutils.py \
   $(topsrcdir)/build/mobile/remoteautomation.py \

testing/mochitest/Makefile.in

@@ -52,6 +52,7 @@ _SERV_FILES = 	\
 		$(topsrcdir)/testing/mozbase/mozdevice/mozdevice/devicemanager.py \
 		$(topsrcdir)/testing/mozbase/mozdevice/mozdevice/devicemanagerADB.py \
 		$(topsrcdir)/testing/mozbase/mozdevice/mozdevice/devicemanagerSUT.py \
+		$(topsrcdir)/testing/mozbase/mozdevice/mozdevice/Zeroconf.py \
 		$(topsrcdir)/build/automationutils.py \
 		$(topsrcdir)/build/manifestparser.py \
 		$(topsrcdir)/build/mobile/remoteautomation.py \

testing/mozbase/mozdevice/mozdevice/__init__.py

@@ -5,7 +5,6 @@
 from devicemanager import DMError
 from devicemanagerADB import DeviceManagerADB
 from devicemanagerSUT import DeviceManagerSUT
-from droid import DroidADB, DroidSUT
+from droid import DroidADB, DroidSUT, DroidConnectByHWID
 from emulator import Emulator
 from b2gemulator import B2GEmulator
-

testing/mozbase/mozdevice/mozdevice/devicemanager.py

@@ -10,6 +10,8 @@ import struct
 import StringIO
 import zlib
 
+from Zeroconf import Zeroconf, ServiceBrowser
+
 class DMError(Exception):
     "generic devicemanager exception."
 
@@ -233,16 +235,6 @@ class DeviceManager:
           failure: None
         """
 
-    @abstractmethod
-    def isDir(self, remotePath):
-        """
-        Checks if remotePath is a directory on the device
-
-        returns:
-          success: True
-          failure: False
-        """
-
     @abstractmethod
     def validateFile(self, remoteFile, localFile):
         """
@@ -614,12 +606,12 @@ class NetworkTools:
             ip = socket.gethostbyname(socket.gethostname())
         except socket.gaierror:
             ip = socket.gethostbyname(socket.gethostname() + ".local") # for Mac OS X
-        if ip.startswith("127.") and os.name != "nt":
+        if (ip is None or ip.startswith("127.")) and os.name != "nt":
             interfaces = ["eth0","eth1","eth2","wlan0","wlan1","wifi0","ath0","ath1","ppp0"]
             for ifname in interfaces:
                 try:
                     ip = self.getInterfaceIp(ifname)
-                    break;
+                    break
                 except IOError:
                     pass
         return ip
@@ -680,3 +672,34 @@ def _pop_last_line(file_obj):
         bytes_from_end += 1
 
     return None
+
+class ZeroconfListener(object):
+    def __init__(self, hwid, evt):
+        self.hwid = hwid
+        self.evt = evt
+
+    # Format is 'SUTAgent [hwid:015d2bc2825ff206] [ip:10_242_29_221]._sutagent._tcp.local.'
+    def addService(self, zeroconf, type, name):
+        #print "Found _sutagent service broadcast:", name
+        if not name.startswith("SUTAgent"):
+            return
+
+        sutname = name.split('.')[0]
+        m = re.search('\[hwid:([^\]]*)\]', sutname)
+        if m is None:
+            return
+
+        hwid = m.group(1)
+
+        m = re.search('\[ip:([0-9_]*)\]', sutname)
+        if m is None:
+            return
+
+        ip = m.group(1).replace("_", ".")
+        
+        if self.hwid == hwid:
+            self.ip = ip
+            self.evt.set()
+
+    def removeService(self, zeroconf, type, name):
+        pass

testing/mozbase/mozdevice/mozdevice/devicemanagerADB.py

@@ -12,7 +12,7 @@ import time
 class DeviceManagerADB(DeviceManager):
 
     def __init__(self, host=None, port=20701, retrylimit=5, packageName='fennec',
-                 adbPath='adb', deviceSerial=None, deviceRoot=None):
+                 adbPath='adb', deviceSerial=None, deviceRoot=None, **kwargs):
         self.host = host
         self.port = port
         self.retrylimit = retrylimit
@@ -499,8 +499,11 @@ class DeviceManagerADB(DeviceManager):
         # if self.deviceRoot is already set, create it if necessary, and use it
         if self.deviceRoot:
             if not self.dirExists(self.deviceRoot):
-                if not self.mkDir(self.deviceRoot):
-                    raise DMError("Unable to create device root %s" % self.deviceRoot)
+                try:
+                    self.mkDir(self.deviceRoot)
+                except:
+                    print "Unable to create device root %s" % self.deviceRoot
+                    raise
             return
 
         # /mnt/sdcard/tests is preferred to /data/local/tests, but this can be

testing/mozbase/mozdevice/mozdevice/devicemanagerSUT.py

@@ -26,11 +26,12 @@ class DeviceManagerSUT(DeviceManager):
     agentErrorRE = re.compile('^##AGENT-WARNING##\ ?(.*)')
     default_timeout = 300
 
-    def __init__(self, host, port = 20701, retrylimit = 5, deviceRoot = None):
+    def __init__(self, host, port = 20701, retrylimit = 5, deviceRoot = None, **kwargs):
         self.host = host
         self.port = port
         self.retrylimit = retrylimit
         self._sock = None
+        self._everConnected = False
         self.deviceRoot = deviceRoot
 
         # Initialize device root
@@ -150,7 +151,7 @@ class DeviceManagerSUT(DeviceManager):
 
         if not self._sock:
             try:
-                if self.debug >= 1:
+                if self.debug >= 1 and self._everConnected:
                     print "reconnecting socket"
                 self._sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
             except socket.error, msg:
@@ -164,6 +165,7 @@ class DeviceManagerSUT(DeviceManager):
                 else:
                     raise DMError("Remote Device Error: Timeout in connecting", fatal=True)
                     return False
+                self._everConnected = True
             except socket.error, msg:
                 self._sock.close()
                 self._sock = None

testing/mozbase/mozdevice/mozdevice/dmcli.py

@@ -3,21 +3,20 @@
 # You can obtain one at http://mozilla.org/MPL/2.0/.
 
 """
-Command-line client to control a device with the SUTAgent software installed
+Command-line client to control a device
 """
 
 import os
 import posixpath
 import StringIO
 import sys
+import textwrap
+import mozdevice
 from optparse import OptionParser
 
-from mozdevice import droid
-
-class SUTCli(object):
+class DMCli(object):
 
     def __init__(self, args=sys.argv[1:]):
-        usage = "usage: %prog [options] <command> [<args>]\n\ndevice commands:\n"
         self.commands = { 'install': { 'function': self.install,
                                        'min_args': 1,
                                        'max_args': 1,
@@ -89,11 +88,15 @@ class SUTCli(object):
 
                           }
 
-        for (commandname, command) in sorted(self.commands.iteritems()):
-            help_args = command['help_args']
-            usage += "  %s - %s\n" % (" ".join([ commandname,
-                                                 help_args ]).rstrip(),
-                                      command['help'])
+        usage = "usage: %prog [options] <command> [<args>]\n\ndevice commands:\n"
+        usage += "\n".join([textwrap.fill("%s %s - %s" %
+                                          (cmdname, cmd['help_args'],
+                                           cmd['help']),
+                                          initial_indent="  ",
+                                          subsequent_indent="      ")
+                            for (cmdname, cmd) in 
+                            sorted(self.commands.iteritems())])
+
         self.parser = OptionParser(usage)
         self.add_options(self.parser)
 
@@ -102,12 +105,10 @@ class SUTCli(object):
         if len(self.args) < 1:
             self.parser.error("must specify command")
 
-        if not self.options.deviceip:
-            if not os.environ.get('TEST_DEVICE'):
-                self.parser.error("Must specify device ip in TEST_DEVICE or "
-                                  "with --remoteDevice option")
-            else:
-                self.options.deviceip = os.environ.get('TEST_DEVICE')
+        if self.options.dmtype == "sut" and not self.options.host and \
+                not self.options.hwid:
+            self.parser.error("Must specify device ip in TEST_DEVICE or "
+                              "with --host option with SUT")
 
         (command_name, command_args) = (self.args[0], self.args[1:])
         if command_name not in self.commands:
@@ -119,24 +120,63 @@ class SUTCli(object):
                 command['max_args'] and len(command_args) > command['max_args']:
             self.parser.error("Wrong number of arguments")
 
-        self.dm = droid.DroidSUT(self.options.deviceip,
-                                 port=int(self.options.deviceport))
+        self.dm = self.getDevice(dmtype=self.options.dmtype,
+                                 hwid=self.options.hwid,
+                                 host=self.options.host,
+                                 port=self.options.port)
         command['function'](*command_args)
 
     def add_options(self, parser):
-        parser.add_option("-r", "--remoteDevice", action="store",
-                          type = "string", dest = "deviceip",
-                          help = "Device IP", default=None)
-        parser.add_option("-p", "--remotePort", action="store",
-                          type = "int", dest = "deviceport",
-                          help = "SUTAgent port (defaults to 20701)",
-                          default=20701)
+        parser.add_option("-v", "--verbose", action="store_true",
+                          dest="verbose",
+                          help="Verbose output from DeviceManager",
+                          default = False)
+        parser.add_option("--host", action="store",
+                          type = "string", dest = "host",
+                          help = "Device hostname (only if using TCP/IP)",
+                          default=os.environ.get('TEST_DEVICE'))
+        parser.add_option("-p", "--port", action="store",
+                          type = "int", dest = "port",
+                          help = "Custom device port (if using SUTAgent or "
+                          "adb-over-tcp)", default=None)
+        parser.add_option("-m", "--dmtype", action="store",
+                          type = "string", dest = "dmtype",
+                          help = "DeviceManager type (adb or sut, defaults " \
+                              "to adb)", default=os.environ.get('DM_TRANS',
+                                                                'adb'))
+        parser.add_option("-d", "--hwid", action="store",
+                          type="string", dest="hwid",
+                          help="HWID", default=None)
+
+    def getDevice(self, dmtype="adb", hwid=None, host=None, port=None):
+        '''
+        Returns a device with the specified parameters
+        '''
+        if self.options.verbose:
+            mozdevice.DroidSUT.debug = 4
+
+        if hwid:
+            return mozdevice.DroidConnectByHWID(hwid)
+
+        if dmtype == "adb":
+            if host and not port:
+                port = 5555
+            return mozdevice.DroidADB(packageName=None, host=host,
+                                      port=port)
+        elif dmtype == "sut":
+            if not host:
+                self.parser.error("Must specify host with SUT!")
+            if not port:
+                port = 20701
+            return mozdevice.DroidSUT(host=host, port=port)
+        else:
+            self.parser.error("Unknown device manager type: %s" % type)
 
     def push(self, src, dest):
         if os.path.isdir(src):
             self.dm.pushDir(src, dest)
         else:
-            dest_is_dir = dest[-1] == '/' or self.dm.isDir(dest)
+            dest_is_dir = dest[-1] == '/' or self.dm.dirExists(dest)
             dest = posixpath.normpath(dest)
             if dest_is_dir:
                 dest = posixpath.join(dest, os.path.basename(src))
@@ -148,17 +188,10 @@ class SUTCli(object):
             return
         if not dest:
             dest = posixpath.basename(src)
-        if self.dm.isDir(src):
-            result = self.dm.getDirectory(src, dest)
-            if result:
-                print '\n'.join([posixpath.join(dest, x) for x in result])
-                return
+        if self.dm.dirExists(src):
+            self.dm.getDirectory(src, dest)
         else:
-            result = self.dm.getFile(src, dest)
-            if result:
-                print dest
-                return
-        print 'Pull failed.'
+            self.dm.getFile(src, dest)
 
     def install(self, apkfile):
         basename = os.path.basename(apkfile)
@@ -198,7 +231,7 @@ class SUTCli(object):
     def processlist(self):
         pslist = self.dm.getProcessList()
         for ps in pslist:
-            print " ".join(ps)
+            print " ".join(str(i) for i in ps)
 
     def listfiles(self, dir):
         filelist = self.dm.listFiles(dir)
@@ -207,7 +240,7 @@ class SUTCli(object):
 
 def cli(args=sys.argv[1:]):
     # process the command line
-    cli = SUTCli(args)
+    cli = DMCli(args)
 
 if __name__ == '__main__':
     cli()

testing/mozbase/mozdevice/mozdevice/droid.py

@@ -2,9 +2,13 @@
 # License, v. 2.0. If a copy of the MPL was not distributed with this file,
 # You can obtain one at http://mozilla.org/MPL/2.0/.
 
+import StringIO
+import threading
+
+from Zeroconf import Zeroconf, ServiceBrowser
+from devicemanager import ZeroconfListener, NetworkTools
 from devicemanagerADB import DeviceManagerADB
 from devicemanagerSUT import DeviceManagerSUT
-import StringIO
 
 class DroidMixin(object):
     """Mixin to extend DeviceManager with Android-specific functionality"""
@@ -81,3 +85,33 @@ class DroidADB(DeviceManagerADB, DroidMixin):
 
 class DroidSUT(DeviceManagerSUT, DroidMixin):
     pass
+
+def DroidConnectByHWID(hwid, timeout=30, **kwargs):
+    """Try to connect to the given device by waiting for it to show up using mDNS with the given timeout."""
+    nt = NetworkTools()
+    local_ip = nt.getLanIp()
+
+    zc = Zeroconf(local_ip)
+
+    evt = threading.Event()
+    listener = ZeroconfListener(hwid, evt)
+    sb = ServiceBrowser(zc, "_sutagent._tcp.local.", listener)
+    foundIP = None
+    if evt.wait(timeout):
+        # we found the hwid 
+        foundIP = listener.ip
+    sb.cancel()
+    zc.close()
+
+    if foundIP is not None:
+        return DroidSUT(foundIP, **kwargs)
+    print "Connected via SUT to %s [at %s]" % (hwid, foundIP)
+
+    # try connecting via adb
+    try:
+        sut = DroidADB(deviceSerial=hwid, **kwargs)
+    except:
+        return None
+
+    print "Connected via ADB to %s" % (hwid)
+    return sut

testing/mozbase/mozdevice/setup.py

@@ -33,6 +33,6 @@ setup(name='mozdevice',
       entry_points="""
       # -*- Entry points: -*-
       [console_scripts]
-      sut = mozdevice.sutcli:cli
+      dm = mozdevice.dmcli:cli
       """,
       )

testing/mozbase/mozprocess/mozprocess/processhandler.py

@@ -237,7 +237,7 @@ class ProcessHandlerMixin(object):
                                                 0,    # job mem limit (ignored)
                                                 0,    # peak process limit (ignored)
                                                 0)    # peak job limit (ignored)
-                                                
+
                         winprocess.SetInformationJobObject(self._job,
                                                            JobObjectExtendedLimitInformation,
                                                            addressof(jeli),
@@ -605,13 +605,19 @@ falling back to not using job objects for managing child processes"""
         """
         self.didTimeout = False
         self.startTime = datetime.now()
-        self.proc = self.Process(self.cmd,
-                                 stdout=subprocess.PIPE,
-                                 stderr=subprocess.STDOUT,
-                                 cwd=self.cwd,
-                                 env=self.env,
-                                 ignore_children = self._ignore_children,
-                                 **self.keywordargs)
+
+        # default arguments
+        args = dict(stdout=subprocess.PIPE,
+                    stderr=subprocess.STDOUT,
+                    cwd=self.cwd,
+                    env=self.env,
+                    ignore_children=self._ignore_children)
+
+        # build process arguments
+        args.update(self.keywordargs)
+
+        # launch the process
+        self.proc = self.Process(self.cmd, **args)
 
         self.processOutput(timeout=timeout, outputTimeout=outputTimeout)