gecko/build/mobile/devicemanager.py

# ***** BEGIN LICENSE BLOCK *****
# Version: MPL 1.1/GPL 2.0/LGPL 2.1
#
# The contents of this file are subject to the Mozilla Public License Version
# 1.1 (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
# http://www.mozilla.org/MPL/
#
# Software distributed under the License is distributed on an "AS IS" basis,
# WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
# for the specific language governing rights and limitations under the
# License.   
#
# The Original Code is Test Automation Framework.
#
# The Initial Developer of the Original Code is Joel Maher.
#
# Portions created by the Initial Developer are Copyright (C) 2009
# the Initial Developer. All Rights Reserved.
#
# Contributor(s):
#   Joel Maher <joel.maher@gmail.com> (Original Developer)
#   Clint Talbert <cmtalbert@gmail.com>
#
# Alternatively, the contents of this file may be used under the terms of
# either the GNU General Public License Version 2 or later (the "GPL"), or
# the GNU Lesser General Public License Version 2.1 or later (the "LGPL"),
# in which case the provisions of the GPL or the LGPL are applicable instead
# of those above. If you wish to allow use of your version of this file only
# under the terms of either the GPL or the LGPL, and not to allow others to
# use your version of this file under the terms of the MPL, indicate your
# decision by deleting the provisions above and replace them with the notice
# and other provisions required by the GPL or the LGPL. If you do not delete
# the provisions above, a recipient may use your version of this file under
# the terms of any one of the MPL, the GPL or the LGPL.
#
# ***** END LICENSE BLOCK *****

import socket
import SocketServer
import time, datetime
import os
import re
import hashlib
import subprocess
from threading import Thread
import traceback
import sys

class FileError(Exception):
  " Signifies an error which occurs while doing a file operation."

  def __init__(self, msg = ''):
    self.msg = msg

  def __str__(self):
    return self.msg

class DeviceManager:
  host = ''
  port = 0
  debug = 2 
  _redo = False
  deviceRoot = None
  tempRoot = os.getcwd()
  base_prompt = '$>'
  base_prompt_re = '\$\>'
  prompt_sep = '\x00'
  prompt_regex = '.*(' + base_prompt_re + prompt_sep + ')'
  agentErrorRE = re.compile('^##AGENT-WARNING##.*')


  def __init__(self, host, port = 20701):
    self.host = host
    self.port = port
    self._sock = None
    self.getDeviceRoot()

  def cmdNeedsResponse(self, cmd):
    """ Not all commands need a response from the agent:
        * if the cmd matches the pushRE then it is the first half of push
          and therefore we want to wait until the second half before looking
          for a response
        * rebt obviously doesn't get a response
        * uninstall performs a reboot to ensure starting in a clean state and
          so also doesn't look for a response
    """
    noResponseCmds = [re.compile('^push .*$'),
                      re.compile('^rebt'),
                      re.compile('^uninst .*$'),
                      re.compile('^pull .*$')]

    for c in noResponseCmds:
      if (c.match(cmd)):
        return False
    
    # If the command is not in our list, then it gets a response
    return True

  def shouldCmdCloseSocket(self, cmd):
    """ Some commands need to close the socket after they are sent:
    * push
    * rebt
    * uninst
    * quit
    """
    
    socketClosingCmds = [re.compile('^push .*$'),
                         re.compile('^quit.*'),
                         re.compile('^rebt.*'),
                         re.compile('^uninst .*$')]

    for c in socketClosingCmds:
      if (c.match(cmd)):
        return True

    return False

  def sendCMD(self, cmdline, newline = True):
    promptre = re.compile(self.prompt_regex + '$')
    data = ""
    shouldCloseSocket = False
    recvGuard = 1000

    if (self._sock == None):
      try:
        if (self.debug >= 1):
          print "reconnecting socket"
        self._sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
      except:
        self._redo = True
        self._sock = None
        if (self.debug >= 2):
          print "unable to create socket"
        return None
      
      try:
        self._sock.connect((self.host, int(self.port)))
        self._sock.recv(1024)
      except:
        self._redo = True
        self._sock.close()
        self._sock = None
        if (self.debug >= 2):
          print "unable to connect socket"
        return None
    
    for cmd in cmdline:
      if newline: cmd += '\r\n'
      
      try:
        numbytes = self._sock.send(cmd)
        if (numbytes != len(cmd)):
          print "ERROR: our cmd was " + str(len(cmd)) + " bytes and we only sent " + str(numbytes)
          return None
        if (self.debug >= 4): print "send cmd: " + str(cmd)
      except:
        self._redo = True
        self._sock.close()
        self._sock = None
        return None
      
      # Check if the command should close the socket
      shouldCloseSocket = self.shouldCmdCloseSocket(cmd)

      # Handle responses from commands
      if (self.cmdNeedsResponse(cmd)):
        found = False
        loopguard = 0
        # TODO: We had an old sleep here but we don't need it

        while (found == False and (loopguard < recvGuard)):
          temp = ''
          if (self.debug >= 4): print "recv'ing..."

          # Get our response
          try:
            temp = self._sock.recv(1024)
            if (self.debug >= 4): print "response: " + str(temp)
          except:
            self._redo = True
            self._sock.close()
            self._sock = None
            return None

          # If something goes wrong in the agent it will send back a string that
          # starts with '##AGENT-ERROR##'
          if (self.agentErrorRE.match(temp)):
            data = temp
            break

          lines = temp.split('\n')

          for line in lines:
            if (promptre.match(line)):
              found = True
          data += temp

          # If we violently lose the connection to the device, this loop tends to spin,
          # this guard prevents that
          if (temp == ''):
            loopguard += 1
            
    # TODO: We had an old sleep here but we don't need it
    if (shouldCloseSocket == True):
      try:
        self._sock.close()
        self._sock = None
      except:
        self._redo = True
        self._sock = None
        return None

    return data
  
  
  # take a data blob and strip instances of the prompt '$>\x00'
  def stripPrompt(self, data):
    promptre = re.compile(self.prompt_regex + '.*')
    retVal = []
    lines = data.split('\n')
    for line in lines:
      try:
        while (promptre.match(line)):
          pieces = line.split(self.prompt_sep)
          index = pieces.index('$>')
          pieces.pop(index)
          line = self.prompt_sep.join(pieces)
      except(ValueError):
        pass
      retVal.append(line)

    return '\n'.join(retVal)
  

  def pushFile(self, localname, destname):
    if (self.debug >= 3): print "in push file with: " + localname + ", and: " + destname
    if (self.validateFile(destname, localname) == True):
      if (self.debug >= 3): print "files are validated"
      return ''

    if self.mkDirs(destname) == None:
      print "unable to make dirs: " + destname
      return None

    if (self.debug >= 3): print "sending: push " + destname
    
    filesize = os.path.getsize(localname)
    f = open(localname, 'rb')
    data = f.read()
    f.close()
    retVal = self.sendCMD(['push ' + destname + ' ' + str(filesize) + '\r\n', data], newline = False)
    
    if (self.debug >= 3): print "push returned: " + str(retVal)

    validated = False
    if (retVal):
      retline = self.stripPrompt(retVal).strip() 
      if (retline == None or self.agentErrorRE.match(retVal)):
        # Then we failed to get back a hash from agent, try manual validation
        validated = self.validateFile(destname, localname)
      else:
        # Then we obtained a hash from push
        localHash = self.getLocalHash(localname)
        if (str(localHash) == str(retline)):
          validated = True
    else:
      # We got nothing back from sendCMD, try manual validation
      validated = self.validateFile(destname, localname)

    if (validated):
      if (self.debug >= 3): print "Push File Validated!"
      return True
    else:
      if (self.debug >= 2): print "Push File Failed to Validate!"
      return None
  
  def mkDir(self, name):
    if (self.dirExists(name)):
      return name
    else:
      return self.sendCMD(['mkdr ' + name])
  
  # make directory structure on the device
  def mkDirs(self, filename):
    parts = filename.split('/')
    name = ""
    for part in parts:
      if (part == parts[-1]): break
      if (part != ""):
        name += '/' + part
        if (self.mkDir(name) == None):
          print "failed making directory: " + str(name)
          return None
    return ''

  # push localDir from host to remoteDir on the device
  def pushDir(self, localDir, remoteDir):
    if (self.debug >= 2): print "pushing directory: " + localDir + " to " + remoteDir
    for root, dirs, files in os.walk(localDir):
      parts = root.split(localDir)
      for file in files:
        remoteRoot = remoteDir + '/' + parts[1]
        remoteName = remoteRoot + '/' + file
        if (parts[1] == ""): remoteRoot = remoteDir
        if (self.pushFile(os.path.join(root, file), remoteName) == None):
          self.removeFile(remoteName)
          if (self.pushFile(os.path.join(root, file), remoteName) == None):
            return None
    return True

  def dirExists(self, dirname):
    match = ".*" + dirname + "$"
    dirre = re.compile(match)
    data = self.sendCMD(['cd ' + dirname, 'cwd'])
    # Because this is a compound command, cd can fail while cwd can succeed, 
    # we should check for agent error directly
    if (data == None or self.agentErrorRE.match(data) ):
      return None
    retVal = self.stripPrompt(data)
    data = retVal.split('\n')
    found = False
    for d in data:
      if (dirre.match(d)): 
        found = True

    return found

  # Because we always have / style paths we make this a lot easier with some
  # assumptions
  def fileExists(self, filepath):
    s = filepath.split('/')
    containingpath = '/'.join(s[:-1])
    listfiles = self.listFiles(containingpath)
    for f in listfiles:
      if (f == s[-1]):
        return True
    return False

  # list files on the device, requires cd to directory first
  def listFiles(self, rootdir):
    rootdir = rootdir.rstrip('/')
    if (self.dirExists(rootdir) == False):
      return []  
    data = self.sendCMD(['cd ' + rootdir, 'ls'])
    if (data == None):
      return None
    retVal = self.stripPrompt(data)
    files = filter(lambda x: x, retVal.split('\n'))
    if len(files) == 1 and files[0] == '<empty>':
      # special case on the agent: empty directories return just the string "<empty>"
      return []
    return files

  def removeFile(self, filename):
    if (self.debug>= 2): print "removing file: " + filename
    return self.sendCMD(['rm ' + filename])
    
  # does a recursive delete of directory on the device: rm -Rf remoteDir
  def removeDir(self, remoteDir):
    self.sendCMD(['rmdr ' + remoteDir])

  def getProcessList(self):
    data = self.sendCMD(['ps'])
    if (data == None):
      return None
      
    retVal = self.stripPrompt(data)
    lines = retVal.split('\n')
    files = []
    for line in lines:
      if (line.strip() != ''):
        pidproc = line.strip().split()
        if (len(pidproc) == 2):
          files += [[pidproc[0], pidproc[1]]]
        elif (len(pidproc) == 3):
          #android returns <userID> <procID> <procName>
          files += [[pidproc[1], pidproc[2], pidproc[0]]]     
    return files

  def getMemInfo(self):
    data = self.sendCMD(['mems'])
    if (data == None):
      return None
    retVal = self.stripPrompt(data)
    # TODO: this is hardcoded for now
    fhandle = open("memlog.txt", 'a')
    fhandle.write("\n")
    fhandle.write(retVal)
    fhandle.close()

  def fireProcess(self, appname):
    if (self.debug >= 2): print "FIRE PROC: '" + appname + "'"
    
    if (self.processExist(appname) != ''):
      print "WARNING: process %s appears to be running already\n" % appname
    
    self.sendCMD(['exec ' + appname])

    #NOTE: we sleep for 30 seconds to allow the application to startup
    time.sleep(30)

    self.process = self.processExist(appname)
    if (self.debug >= 4): print "got pid: " + str(self.process) + " for process: " + str(appname)

  def launchProcess(self, cmd, outputFile = "process.txt", cwd = '', env = ''):
    cmdline = subprocess.list2cmdline(cmd)
    if (outputFile == "process.txt" or outputFile == None):
      outputFile = self.getDeviceRoot() + '/' + "process.txt"
      cmdline += " > " + outputFile
    
    # Prepend our env to the command 
    cmdline = ('%s ' % self.formatEnvString(env)) + cmdline

    self.fireProcess(cmdline)
    return outputFile
  
  #hardcoded: sleep interval of 5 seconds, timeout of 10 minutes
  def communicate(self, process, timeout = 600):
    interval = 5
    timed_out = True
    if (timeout > 0):
      total_time = 0
      while total_time < timeout:
        time.sleep(interval)
        if (not self.poll(process)):
          timed_out = False
          break
        total_time += interval

    if (timed_out == True):
      return None

    return [self.getFile(process, "temp.txt"), None]


  def poll(self, process):
    try:
      if (self.processExist(process) == ''):
        return None
      return 1
    except:
      return None
    return 1
  
  # iterates process list and returns pid if exists, otherwise ''
  def processExist(self, appname):
    pid = ''

    #remove the environment variables in the cli if they exist
    parts = appname.split(' ')
    for p in parts:
      if (p is ''):
        parts.remove(p)

    if len(parts[0].strip('"').split('=')) > 1:
      envvars = parts[0].strip('"').split(',')
      for e in envvars:
        env = e.split('=')
        if (len(env) > 1):
          os.environ[env[0]] = str(env[1])
      appname = ' '.join(parts[1:])

  
    pieces = appname.split(' ')
    parts = pieces[0].split('/')
    app = parts[-1]
    procre = re.compile('.*' + app + '.*')

    procList = self.getProcessList()
    if (procList == None):
      return None
      
    for proc in procList:
      if (procre.match(proc[1])):
        pid = proc[0]
        break
    return pid

  def killProcess(self, appname):
    if (self.sendCMD(['kill ' + appname]) == None):
      return None

    return True

  def getTempDir(self):
    retVal = ''
    data = self.sendCMD(['tmpd'])
    if (data == None):
      return None
    return self.stripPrompt(data).strip('\n')

  def catFile(self, remoteFile):
    data = self.sendCMD(['cat ' + remoteFile])
    if data == None:
        return None
    return self.stripPrompt(data)
  
  def pullFile(self, remoteFile):
    """Returns contents of remoteFile using the "pull" command.
    The "pull" command is different from other commands in that DeviceManager
    has to read a certain number of bytes instead of just reading to the
    next prompt.  This is more robust than the "cat" command, which will be
    confused if the prompt string exists within the file being catted.
    However it means we can't use the response-handling logic in sendCMD().
    """
    
    def err(error_msg):
        err_str = 'error returned from pull: %s' % error_msg
        print err_str
        self._sock = None
        raise FileError(err_str) 

    # FIXME: We could possibly move these socket-reading functions up to
    # the class level if we wanted to refactor sendCMD().  For now they are
    # only used to pull files.
    
    def uread(to_recv, error_msg):
      """ unbuffered read """
      data = self._sock.recv(to_recv)
      if not data:
        err(error_msg)
        return None
      return data

    def read_until_char(c, buffer, error_msg):
      """ read until 'c' is found; buffer rest """
      while not '\n' in buffer:
        data = uread(1024, error_msg)
        if data == None:
          err(error_msg)
          return ('', '', '')
        buffer += data
      return buffer.partition(c)

    def read_exact(total_to_recv, buffer, error_msg):
      """ read exact number of 'total_to_recv' bytes """
      while len(buffer) < total_to_recv:
        to_recv = min(total_to_recv - len(buffer), 1024)
        data = uread(to_recv, error_msg)
        if data == None:
          return None
        buffer += data
      return buffer

    prompt = self.base_prompt + self.prompt_sep
    buffer = ''
    
    # expected return value:
    # <filename>,<filesize>\n<filedata>
    # or, if error,
    # <filename>,-1\n<error message>
    self.sendCMD(['pull ' + remoteFile])
    # read metadata; buffer the rest
    metadata, sep, buffer = read_until_char('\n', buffer, 'could not find metadata')
    if not metadata:
      return None
    if self.debug >= 3:
      print 'metadata: %s' % metadata

    filename, sep, filesizestr = metadata.partition(',')
    if sep == '':
      err('could not find file size in returned metadata')
      return None
    try:
        filesize = int(filesizestr)
    except ValueError:
      err('invalid file size in returned metadata')
      return None

    if filesize == -1:
      # read error message
      error_str, sep, buffer = read_until_char('\n', buffer, 'could not find error message')
      if not error_str:
        return None
      # prompt should follow
      read_exact(len(prompt), buffer, 'could not find prompt')
      print 'DeviceManager: error pulling file: %s' % error_str
      return None

    # read file data
    total_to_recv = filesize + len(prompt)
    buffer = read_exact(total_to_recv, buffer, 'could not get all file data')
    if buffer == None:
      return None
    if buffer[-len(prompt):] != prompt:
      err('no prompt found after file data--DeviceManager may be out of sync with agent')
      return buffer
    return buffer[:-len(prompt)]

  # copy file from device (remoteFile) to host (localFile)
  def getFile(self, remoteFile, localFile = ''):
    if localFile == '':
      localFile = os.path.join(self.tempRoot, "temp.txt")
  
    retVal = self.pullFile(remoteFile)
    if retVal == None:
      return None
    fhandle = open(localFile, 'wb')
    fhandle.write(retVal)
    fhandle.close()
    if not self.validateFile(remoteFile, localFile):
      print 'failed to validate file when downloading %s!' % remoteFile
      return None
    return retVal
    
  # copy directory structure from device (remoteDir) to host (localDir)
  def getDirectory(self, remoteDir, localDir):
    if (self.debug >= 2): print "getting files in '" + remoteDir + "'"
    filelist = self.listFiles(remoteDir)
    if (filelist == None):
      return None
    if (self.debug >= 3): print filelist
    if not os.path.exists(localDir):
      os.makedirs(localDir)
   
    for f in filelist:
      if f == '.' or f == '..':
        continue
      remotePath = remoteDir + '/' + f
      localPath = os.path.join(localDir, f)
      try:
        is_dir = self.isDir(remotePath)
      except FileError:
        print 'isdir failed on file "%s"; continuing anyway...' % remotePath
        continue
      if is_dir:
        if (self.getDirectory(remotePath, localPath) == None):
          print 'failed to get directory "%s"' % remotePath
          return None
      else:
        # It's sometimes acceptable to have getFile() return None, such as
        # when the agent encounters broken symlinks.
        # FIXME: This should be improved so we know when a file transfer really
        # failed.
        if self.getFile(remotePath, localPath) == None:
          print 'failed to get file "%s"; continuing anyway...' % remotePath 
    return filelist

  def isDir(self, remotePath):
    data = self.sendCMD(['isdir ' + remotePath])
    retVal = self.stripPrompt(data).strip()
    if not retVal:
      raise FileError('isdir returned null')
    return retVal == 'TRUE'

  # true/false check if the two files have the same md5 sum
  def validateFile(self, remoteFile, localFile):
    remoteHash = self.getRemoteHash(remoteFile)
    localHash = self.getLocalHash(localFile)

    if (remoteHash == localHash):
      return True

    return False
  
  # return the md5 sum of a remote file
  def getRemoteHash(self, filename):
    data = self.sendCMD(['hash ' + filename])
    if (data == None):
        return ''
    retVal = self.stripPrompt(data)
    if (retVal != None):
      retVal = retVal.strip('\n')
    if (self.debug >= 3): print "remote hash returned: '" + retVal + "'"
    return retVal
    
  # return the md5 sum of a file on the host
  def getLocalHash(self, filename):
    file = open(filename, 'rb')
    if (file == None):
      return None

    try:
      mdsum = hashlib.md5()
    except:
      return None

    while 1:
      data = file.read(1024)
      if not data:
        break
      mdsum.update(data)

    file.close()
    hexval = mdsum.hexdigest()
    if (self.debug >= 3): print "local hash returned: '" + hexval + "'"
    return hexval

  # Gets the device root for the testing area on the device
  # For all devices we will use / type slashes and depend on the device-agent
  # to sort those out.  The agent will return us the device location where we
  # should store things, we will then create our /tests structure relative to
  # that returned path.
  # Structure on the device is as follows:
  # /tests
  #       /<fennec>|<firefox>  --> approot
  #       /profile
  #       /xpcshell
  #       /reftest
  #       /mochitest
  def getDeviceRoot(self):
    # This caching of deviceRoot is causing issues if things fail
    # if (not self.deviceRoot):
    data = self.sendCMD(['testroot'])
    if (data == None):
      return '/tests'
    self.deviceRoot = self.stripPrompt(data).strip('\n') + '/tests'

    if (not self.dirExists(self.deviceRoot)):
      self.mkDir(self.deviceRoot)

    return self.deviceRoot

  # Either we will have /tests/fennec or /tests/firefox but we will never have
  # both.  Return the one that exists
  def getAppRoot(self):
    if (self.dirExists(self.getDeviceRoot() + '/fennec')):
      return self.getDeviceRoot() + '/fennec'
    elif (self.dirExists(self.getDeviceRoot() + '/firefox')):
      return self.getDeviceRoot() + '/firefox'
    else:
      return 'org.mozilla.fennec'

  # Gets the directory location on the device for a specific test type
  # Type is one of: xpcshell|reftest|mochitest
  def getTestRoot(self, type):
    if (re.search('xpcshell', type, re.I)):
      self.testRoot = self.getDeviceRoot() + '/xpcshell'
    elif (re.search('?(i)reftest', type)):
      self.testRoot = self.getDeviceRoot() + '/reftest'
    elif (re.search('?(i)mochitest', type)):
      self.testRoot = self.getDeviceRoot() + '/mochitest'
    return self.testRoot

  # Sends a specific process ID a signal code and action.
  # For Example: SIGINT and SIGDFL to process x
  def signal(self, processID, signalType, signalAction):
    # currently not implemented in device agent - todo
    pass

  # Get a return code from process ending -- needs support on device-agent
  # this is a todo
  def getReturnCode(self, processID):
    # todo make this real
    return 0

  def unpackFile(self, filename):
    dir = ''
    parts = filename.split('/')
    if (len(parts) > 1):
      if self.fileExists(filename):
        dir = '/'.join(parts[:-1])
    elif self.fileExists('/' + filename):
      dir = '/' + filename
    elif self.fileExists(self.getDeviceRoot() + '/' + filename):
      dir = self.getDeviceRoot() + '/' + filename
    else:
      return None

    return self.sendCMD(['cd ' + dir, 'unzp ' + filename])

  def reboot(self, wait = False):
    self.sendCMD(['rebt'])

    if wait == True:
      time.sleep(30)
      timeout = 270
      done = False
      while (not done):
        if self.listFiles('/') != None:
          return ''
        print "sleeping another 10 seconds"
        time.sleep(10)
        timeout = timeout - 10
        if (timeout <= 0):
          return None
    return ''

  # validate localDir from host to remoteDir on the device
  def validateDir(self, localDir, remoteDir):
    if (self.debug >= 2): print "validating directory: " + localDir + " to " + remoteDir
    for root, dirs, files in os.walk(localDir):
      parts = root.split(localDir)
      for file in files:
        remoteRoot = remoteDir + '/' + parts[1]
        remoteRoot = remoteRoot.replace('/', '/')
        if (parts[1] == ""): remoteRoot = remoteDir
        remoteName = remoteRoot + '/' + file
        if (self.validateFile(remoteName, os.path.join(root, file)) <> True):
            return None
    return True

  # Returns information about the device:
  # Directive indicates the information you want to get, your choices are:
  # os - name of the os
  # id - unique id of the device
  # uptime - uptime of the device
  # systime - system time of the device
  # screen - screen resolution
  # memory - memory stats
  # process - list of running processes (same as ps)
  # disk - total, free, available bytes on disk
  # power - power status (charge, battery temp)
  # all - all of them - or call it with no parameters to get all the information
  def getInfo(self, directive=None):
    data = None
    result = {}
    collapseSpaces = re.compile('  +')

    directives = ['os', 'id','uptime','systime','screen','memory','process',
                  'disk','power']
    if (directive in directives):
      directives = [directive]

    for d in directives:
      data = self.sendCMD(['info ' + d])
      if (data is None):
        continue
      data = self.stripPrompt(data)
      data = collapseSpaces.sub(' ', data)
      result[d] = data.split('\n')

    # Get rid of any 0 length members of the arrays
    for v in result.itervalues():
      while '' in v:
        v.remove('')
    
    # Format the process output
    if 'process' in result:
      proclist = []
      for l in result['process']:
        if l:
          proclist.append(l.split('\t'))
      result['process'] = proclist

    print "results: " + str(result)
    return result

  """
  Installs the application onto the device
  Application bundle - path to the application bundle on the device
  Destination - destination directory of where application should be
                installed to (optional)
  Returns None for success, or output if known failure
  TODO: we need a better way to know if this works or not
  """
  def installApp(self, appBundlePath, destPath=None):
    cmd = 'inst ' + appBundlePath
    if destPath:
      cmd += ' ' + destPath
    data = self.sendCMD([cmd])
    if (data is None):
      return None
    
    f = re.compile('Failure')
    for line in data.split():
      if (f.match(line)):
        return data
    return None

  """
  Uninstalls the named application from device and causes a reboot.
  Takes an optional argument of installation path - the path to where the application
  was installed.
  Returns True, but it doesn't mean anything other than the command was sent,
  the reboot happens and we don't know if this succeeds or not.
  """
  def uninstallAppAndReboot(self, appName, installPath=None):
    cmd = 'uninst ' + appName
    if installPath:
      cmd += ' ' + installPath
    data = self.sendCMD([cmd])
    if (self.debug > 3): print "uninstallAppAndReboot: " + str(data)
    return True

  """
  Updates the application on the device.
  Application bundle - path to the application bundle on the device
  Process name of application - used to end the process if the applicaiton is
                                currently running
  Destination - Destination directory to where the application should be
                installed (optional)
  ipAddr - IP address to await a callback ping to let us know that the device has updated
           properly - defaults to current IP.
  port - port to await a callback ping to let us know that the device has updated properly
         defaults to 30000, and counts up from there if it finds a conflict
  Returns True if succeeds, False if not
  
  NOTE: We have no real way to know if the device gets updated or not due to the
        reboot that the udpate call forces on us.  We can't install our own heartbeat
        listener here because we run the risk of racing with other heartbeat listeners.
  """
  def updateApp(self, appBundlePath, processName=None, destPath=None, ipAddr=None, port=30000):
    status = None
    cmd = 'updt '
    if (processName == None):
      # Then we pass '' for processName
      cmd += "'' " + appBundlePath
    else:
      cmd += processName + ' ' + appBundlePath

    if (destPath):
      cmd += " " + destPath

    if (self.debug > 3): print "updateApp using command: " + str(cmd)

    if (ipAddr is not None):
      ip, port = self.getCallbackIpAndPort(ipAddr, port)

      cmd += " %s %s" % (ip, port)

      # Set up our callback server
      callbacksvr = callbackServer(ip, port, self.debug)
      data = self.sendCMD([cmd])
      status = callbacksvr.disconnect()
      if (self.debug > 3): print "got status back: " + str(status)
    else:
      status = self.sendCMD([cmd])

    return status

  """
    return the current time on the device
  """
  def getCurrentTime(self):
    data = self.sendCMD(['clok'])
    if (data == None):
      return None
    return self.stripPrompt(data).strip('\n')

  """
    Connect the ipaddress and port for a callback ping.  Defaults to current IP address
    And ports starting at 30000.
    NOTE: the detection for current IP address only works on Linux!
  """
  def getCallbackIpAndPort(self, aIp, aPort):
    ip = aIp
    nettools = NetworkTools()
    if (ip == None):
      ip = nettools.getLanIp()
    if (aPort != None):
      port = nettools.findOpenPort(ip, aPort)
    else:
      port = nettools.findOpenPort(ip, 30000)
    return ip, port

  """
    Returns a properly formatted env string for the agent.
    Input - env, which is either None, '', or a dict
    Output - a quoted string of the form: '"envvar1=val1,envvar2=val2..."'
    If env is None or '' return '""' (empty quoted string)
  """
  def formatEnvString(self, env):
    if (env == None or env == ''):
      return '""'

    envstr = '"'
    # TODO: I believe this is inefficient for large dicts
    for k, v in env.items():
      envstr += ('%s=%s,' % (k, v))
    
    # kill the trailing comma, add the last quote
    envstr = envstr.rstrip(',')
    envstr += '"'

    return envstr

gCallbackData = ''

class callbackServer():
  def __init__(self, ip, port, debuglevel):
    self.ip = ip
    self.port = port
    self.connected = False
    self.debug = debuglevel
    if (self.debug > 3) : print "Creating server with " + str(ip) + ":" + str(port)
    self.server = SocketServer.TCPServer((ip, port), self.myhandler)
    self.server_thread = Thread(target=self.server.serve_forever) 
    self.server_thread.setDaemon(True)
    self.server_thread.start()

  def disconnect(self, step = 60, timeout = 600):
    t = 0
    if (self.debug > 3): print "Calling disconnect on callback server"
    while t < timeout:
      if (gCallbackData):
        # Got the data back
        if (self.debug > 3): print "Got data back from agent: " + str(gCallbackData)
        break
      time.sleep(step)
      t += step

    try:
      if (self.debug > 3): print "Shutting down server now"
      self.server.shutdown()
    except:
      print "Unable to shutdown callback server - check for a connection on port: " + str(self.port)
    return gCallbackData

  class myhandler(SocketServer.BaseRequestHandler):
    def handle(self):
      global gCallbackData
      gCallbackData = self.request.recv(1024)
      #print "Callback Handler got data: " + str(gCallbackData)
      self.request.send("OK")
  
class NetworkTools:
  def __init__(self):
    pass

  # Utilities to get the local ip address
  def getInterfaceIp(self, ifname):
    if os.name != "nt":
      import fcntl
      import struct
      s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
      return socket.inet_ntoa(fcntl.ioctl(
                              s.fileno(),
                              0x8915,  # SIOCGIFADDR
                              struct.pack('256s', ifname[:15])
                              )[20:24])
    else:
      return None

  def getLanIp(self):
    ip = socket.gethostbyname(socket.gethostname())
    if ip.startswith("127.") and os.name != "nt":
      interfaces = ["eth0","eth1","eth2","wlan0","wlan1","wifi0","ath0","ath1","ppp0"]
      for ifname in interfaces:
        try:
          ip = self.getInterfaceIp(ifname)
          break;
        except IOError:
          pass
    return ip

  # Gets an open port starting with the seed by incrementing by 1 each time
  def findOpenPort(self, ip, seed):
    try:
      s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
      s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
      connected = False
      if isinstance(seed, basestring):
        seed = int(seed)
      maxportnum = seed + 5000 # We will try at most 5000 ports to find an open one
      while not connected:
        try:
          s.bind((ip, seed))
          connected = True
          s.close()
        except:          
          if seed > maxportnum:
            print "Could not find open port after checking 5000 ports"
          raise
        seed += 1
    except:
      print "Socket error trying to find open port"
        
    return seed