Bug 1091284 - Remove systemMemory, environment from automationutils. r=jgriffin

build/automationutils.py

@@ -18,8 +18,6 @@ import mozinfo
 __all__ = [
   "dumpLeakLog",
   "processLeakLog",
-  'systemMemory',
-  'environment',
   'dumpScreen',
   "setAutomationLog",
   ]
@@ -282,136 +280,6 @@ def processLeakLog(leakLogFile, options):
       processSingleLeakFile(thisFile, processType, leakThreshold,
                             processType in ignoreMissingLeaks)
 
-def systemMemory():
-  """
-  Returns total system memory in kilobytes.
-  Works only on unix-like platforms where `free` is in the path.
-  """
-  return int(os.popen("free").readlines()[1].split()[1])
-
-def environment(xrePath, env=None, crashreporter=True, debugger=False, dmdPath=None, lsanPath=None):
-  """populate OS environment variables for mochitest"""
-
-  env = os.environ.copy() if env is None else env
-
-  assert os.path.isabs(xrePath)
-
-  if mozinfo.isMac:
-    ldLibraryPath = os.path.join(os.path.dirname(xrePath), "MacOS")
-  else:
-    ldLibraryPath = xrePath
-
-  envVar = None
-  dmdLibrary = None
-  preloadEnvVar = None
-  if 'toolkit' in mozinfo.info and mozinfo.info['toolkit'] == "gonk":
-    # Skip all of this, it's only valid for the host.
-    pass
-  elif mozinfo.isUnix:
-    envVar = "LD_LIBRARY_PATH"
-    env['MOZILLA_FIVE_HOME'] = xrePath
-    dmdLibrary = "libdmd.so"
-    preloadEnvVar = "LD_PRELOAD"
-  elif mozinfo.isMac:
-    envVar = "DYLD_LIBRARY_PATH"
-    dmdLibrary = "libdmd.dylib"
-    preloadEnvVar = "DYLD_INSERT_LIBRARIES"
-  elif mozinfo.isWin:
-    envVar = "PATH"
-    dmdLibrary = "dmd.dll"
-    preloadEnvVar = "MOZ_REPLACE_MALLOC_LIB"
-  if envVar:
-    envValue = ((env.get(envVar), str(ldLibraryPath))
-                if mozinfo.isWin
-                else (ldLibraryPath, dmdPath, env.get(envVar)))
-    env[envVar] = os.path.pathsep.join([path for path in envValue if path])
-
-  if dmdPath and dmdLibrary and preloadEnvVar:
-    env[preloadEnvVar] = os.path.join(dmdPath, dmdLibrary)
-
-  # crashreporter
-  env['GNOME_DISABLE_CRASH_DIALOG'] = '1'
-  env['XRE_NO_WINDOWS_CRASH_DIALOG'] = '1'
-
-  if crashreporter and not debugger:
-    env['MOZ_CRASHREPORTER_NO_REPORT'] = '1'
-    env['MOZ_CRASHREPORTER'] = '1'
-  else:
-    env['MOZ_CRASHREPORTER_DISABLE'] = '1'
-
-  # Crash on non-local network connections by default.
-  # MOZ_DISABLE_NONLOCAL_CONNECTIONS can be set to "0" to temporarily
-  # enable non-local connections for the purposes of local testing.  Don't
-  # override the user's choice here.  See bug 1049688.
-  env.setdefault('MOZ_DISABLE_NONLOCAL_CONNECTIONS', '1')
-
-  # Set WebRTC logging in case it is not set yet
-  env.setdefault('NSPR_LOG_MODULES', 'signaling:3,mtransport:4,datachannel:4,jsep:4,MediaPipelineFactory:4')
-  env.setdefault('R_LOG_LEVEL', '6')
-  env.setdefault('R_LOG_DESTINATION', 'stderr')
-  env.setdefault('R_LOG_VERBOSE', '1')
-
-  # ASan specific environment stuff
-  asan = bool(mozinfo.info.get("asan"))
-  if asan and (mozinfo.isLinux or mozinfo.isMac):
-    try:
-      # Symbolizer support
-      llvmsym = os.path.join(xrePath, "llvm-symbolizer")
-      if os.path.isfile(llvmsym):
-        env["ASAN_SYMBOLIZER_PATH"] = llvmsym
-        log.info("INFO | runtests.py | ASan using symbolizer at %s" % llvmsym)
-      else:
-        log.info("TEST-UNEXPECTED-FAIL | runtests.py | Failed to find ASan symbolizer at %s" % llvmsym)
-
-      totalMemory = systemMemory()
-
-      # Only 4 GB RAM or less available? Use custom ASan options to reduce
-      # the amount of resources required to do the tests. Standard options
-      # will otherwise lead to OOM conditions on the current test slaves.
-      message = "INFO | runtests.py | ASan running in %s configuration"
-      asanOptions = []
-      if totalMemory <= 1024 * 1024 * 4:
-        message = message % 'low-memory'
-        asanOptions = ['quarantine_size=50331648', 'malloc_context_size=5']
-      else:
-        message = message % 'default memory'
-
-      if lsanPath:
-        log.info("LSan enabled.")
-        asanOptions.append('detect_leaks=1')
-        lsanOptions = ["exitcode=0"]
-        suppressionsFile = os.path.join(lsanPath, 'lsan_suppressions.txt')
-        if os.path.exists(suppressionsFile):
-          log.info("LSan using suppression file " + suppressionsFile)
-          lsanOptions.append("suppressions=" + suppressionsFile)
-        else:
-          log.info("WARNING | runtests.py | LSan suppressions file does not exist! " + suppressionsFile)
-        env["LSAN_OPTIONS"] = ':'.join(lsanOptions)
-        # Run shutdown GCs and CCs to avoid spurious leaks.
-        env['MOZ_CC_RUN_DURING_SHUTDOWN'] = '1'
-
-      if len(asanOptions):
-        env['ASAN_OPTIONS'] = ':'.join(asanOptions)
-
-    except OSError,err:
-      log.info("Failed determine available memory, disabling ASan low-memory configuration: %s" % err.strerror)
-    except:
-      log.info("Failed determine available memory, disabling ASan low-memory configuration")
-    else:
-      log.info(message)
-
-  tsan = bool(mozinfo.info.get("tsan"))
-  if tsan and mozinfo.isLinux:
-    # Symbolizer support.
-    llvmsym = os.path.join(xrePath, "llvm-symbolizer")
-    if os.path.isfile(llvmsym):
-      env["TSAN_OPTIONS"] = "external_symbolizer_path=%s" % llvmsym
-      log.info("INFO | runtests.py | TSan using symbolizer at %s" % llvmsym)
-    else:
-      log.info("TEST-UNEXPECTED-FAIL | runtests.py | Failed to find TSan symbolizer at %s" % llvmsym)
-
-  return env
-
 def dumpScreen(utilityPath):
   """dumps a screenshot of the entire screen to a directory specified by
   the MOZ_UPLOAD_DIR environment variable"""

layout/tools/reftest/runreftest.py

@@ -23,7 +23,6 @@ sys.path.insert(0, SCRIPT_DIRECTORY)
 
 from automationutils import (
     dumpScreen,
-    environment,
     printstatus,
     processLeakLog
 )
@@ -33,6 +32,7 @@ import mozinfo
 import mozprocess
 import mozprofile
 import mozrunner
+from mozrunner.utils import test_environment
 
 here = os.path.abspath(os.path.dirname(__file__))
 
@@ -273,7 +273,8 @@ class RefTest(object):
     return profile
 
   def environment(self, **kwargs):
-    return environment(**kwargs)
+    kwargs['log'] = log
+    return test_environment(**kwargs)
 
   def buildBrowserEnv(self, options, profileDir):
     browserEnv = self.environment(xrePath = options.xrePath, debugger=options.debugger)

testing/mochitest/runtests.py

@@ -35,7 +35,6 @@ import zipfile
 import bisection
 
 from automationutils import (
-    environment,
     processLeakLog,
     dumpScreen,
     printstatus,
@@ -59,6 +58,7 @@ from mozprofile.permissions import ServerLocations
 from urllib import quote_plus as encodeURIComponent
 from mozlog.structured.formatters import TbplFormatter
 from mozlog.structured import commandline
+from mozrunner.utils import test_environment
 
 here = os.path.abspath(os.path.dirname(__file__))
 
@@ -359,7 +359,7 @@ class MochitestServer(object):
         "Run the Mochitest server, returning the process ID of the server."
 
         # get testing environment
-        env = environment(xrePath=self._xrePath)
+        env = test_environment(xrePath=self._xrePath, log=self._log)
         env["XPCOM_DEBUG_BREAK"] = "warn"
         env["LD_LIBRARY_PATH"] = self._xrePath
 
@@ -1064,7 +1064,7 @@ class SSLTunnel:
                 ssltunnel)
             exit(1)
 
-        env = environment(xrePath=self.xrePath)
+        env = test_environment(xrePath=self.xrePath, log=self.log)
         env["LD_LIBRARY_PATH"] = self.xrePath
         self.process = mozprocess.ProcessHandler([ssltunnel, self.configFile],
                                                  env=env)
@@ -1234,9 +1234,6 @@ class Mochitest(MochitestUtilsMixin):
     def __init__(self, logger_options):
         super(Mochitest, self).__init__(logger_options)
 
-        # environment function for browserEnv
-        self.environment = environment
-
         # Max time in seconds to wait for server startup before tests will fail -- if
         # this seems big, it's mostly for debug machines where cold startup
         # (particularly after a build) takes forever.
@@ -1254,6 +1251,10 @@ class Mochitest(MochitestUtilsMixin):
         self.expectedError = {}
         self.result = {}
 
+    def environment(self, **kwargs):
+        kwargs['log'] = self.log
+        return test_environment(**kwargs)
+
     def extraPrefs(self, extraPrefs):
         """interpolate extra preferences from option strings"""
 

testing/mozbase/mozrunner/mozrunner/utils.py

@@ -94,3 +94,152 @@ def uses_marionette(func):
 
         return ret
     return _
+
+
+def _raw_log():
+    import logging
+    return logging.getLogger(__name__)
+
+
+def test_environment(xrePath, env=None, crashreporter=True, debugger=False,
+                     dmdPath=None, lsanPath=None, log=None):
+    """
+    populate OS environment variables for mochitest and reftests.
+
+    Originally comes from automationutils.py. Don't use that for new code.
+    """
+
+    env = os.environ.copy() if env is None else env
+    log = log or _raw_log()
+
+    assert os.path.isabs(xrePath)
+
+    if mozinfo.isMac:
+        ldLibraryPath = os.path.join(os.path.dirname(xrePath), "MacOS")
+    else:
+        ldLibraryPath = xrePath
+
+    envVar = None
+    dmdLibrary = None
+    preloadEnvVar = None
+    if 'toolkit' in mozinfo.info and mozinfo.info['toolkit'] == "gonk":
+        # Skip all of this, it's only valid for the host.
+        pass
+    elif mozinfo.isUnix:
+        envVar = "LD_LIBRARY_PATH"
+        env['MOZILLA_FIVE_HOME'] = xrePath
+        dmdLibrary = "libdmd.so"
+        preloadEnvVar = "LD_PRELOAD"
+    elif mozinfo.isMac:
+        envVar = "DYLD_LIBRARY_PATH"
+        dmdLibrary = "libdmd.dylib"
+        preloadEnvVar = "DYLD_INSERT_LIBRARIES"
+    elif mozinfo.isWin:
+        envVar = "PATH"
+        dmdLibrary = "dmd.dll"
+        preloadEnvVar = "MOZ_REPLACE_MALLOC_LIB"
+    if envVar:
+        envValue = ((env.get(envVar), str(ldLibraryPath))
+                    if mozinfo.isWin
+                    else (ldLibraryPath, dmdPath, env.get(envVar)))
+        env[envVar] = os.path.pathsep.join([path for path in envValue if path])
+
+    if dmdPath and dmdLibrary and preloadEnvVar:
+        env[preloadEnvVar] = os.path.join(dmdPath, dmdLibrary)
+
+    # crashreporter
+    env['GNOME_DISABLE_CRASH_DIALOG'] = '1'
+    env['XRE_NO_WINDOWS_CRASH_DIALOG'] = '1'
+
+    if crashreporter and not debugger:
+        env['MOZ_CRASHREPORTER_NO_REPORT'] = '1'
+        env['MOZ_CRASHREPORTER'] = '1'
+    else:
+        env['MOZ_CRASHREPORTER_DISABLE'] = '1'
+
+    # Crash on non-local network connections by default.
+    # MOZ_DISABLE_NONLOCAL_CONNECTIONS can be set to "0" to temporarily
+    # enable non-local connections for the purposes of local testing.  Don't
+    # override the user's choice here.  See bug 1049688.
+    env.setdefault('MOZ_DISABLE_NONLOCAL_CONNECTIONS', '1')
+
+    # Set WebRTC logging in case it is not set yet
+    env.setdefault(
+        'NSPR_LOG_MODULES',
+        'signaling:3,mtransport:4,datachannel:4,jsep:4,MediaPipelineFactory:4'
+    )
+    env.setdefault('R_LOG_LEVEL', '6')
+    env.setdefault('R_LOG_DESTINATION', 'stderr')
+    env.setdefault('R_LOG_VERBOSE', '1')
+
+    # ASan specific environment stuff
+    asan = bool(mozinfo.info.get("asan"))
+    if asan and (mozinfo.isLinux or mozinfo.isMac):
+        try:
+            # Symbolizer support
+            llvmsym = os.path.join(xrePath, "llvm-symbolizer")
+            if os.path.isfile(llvmsym):
+                env["ASAN_SYMBOLIZER_PATH"] = llvmsym
+                log.info("INFO | runtests.py | ASan using symbolizer at %s"
+                         % llvmsym)
+            else:
+                log.info("TEST-UNEXPECTED-FAIL | runtests.py | Failed to find"
+                         " ASan symbolizer at %s" % llvmsym)
+
+            # Returns total system memory in kilobytes.
+            # Works only on unix-like platforms where `free` is in the path.
+            totalMemory = int(os.popen("free").readlines()[1].split()[1])
+
+            # Only 4 GB RAM or less available? Use custom ASan options to reduce
+            # the amount of resources required to do the tests. Standard options
+            # will otherwise lead to OOM conditions on the current test slaves.
+            message = "INFO | runtests.py | ASan running in %s configuration"
+            asanOptions = []
+            if totalMemory <= 1024 * 1024 * 4:
+                message = message % 'low-memory'
+                asanOptions = [
+                    'quarantine_size=50331648', 'malloc_context_size=5']
+            else:
+                message = message % 'default memory'
+
+            if lsanPath:
+                log.info("LSan enabled.")
+                asanOptions.append('detect_leaks=1')
+                lsanOptions = ["exitcode=0"]
+                suppressionsFile = os.path.join(
+                    lsanPath, 'lsan_suppressions.txt')
+                if os.path.exists(suppressionsFile):
+                    log.info("LSan using suppression file " + suppressionsFile)
+                    lsanOptions.append("suppressions=" + suppressionsFile)
+                else:
+                    log.info("WARNING | runtests.py | LSan suppressions file"
+                             " does not exist! " + suppressionsFile)
+                env["LSAN_OPTIONS"] = ':'.join(lsanOptions)
+                # Run shutdown GCs and CCs to avoid spurious leaks.
+                env['MOZ_CC_RUN_DURING_SHUTDOWN'] = '1'
+
+            if len(asanOptions):
+                env['ASAN_OPTIONS'] = ':'.join(asanOptions)
+
+        except OSError, err:
+            log.info("Failed determine available memory, disabling ASan"
+                     " low-memory configuration: %s" % err.strerror)
+        except:
+            log.info("Failed determine available memory, disabling ASan"
+                     " low-memory configuration")
+        else:
+            log.info(message)
+
+    tsan = bool(mozinfo.info.get("tsan"))
+    if tsan and mozinfo.isLinux:
+        # Symbolizer support.
+        llvmsym = os.path.join(xrePath, "llvm-symbolizer")
+        if os.path.isfile(llvmsym):
+            env["TSAN_OPTIONS"] = "external_symbolizer_path=%s" % llvmsym
+            log.info("INFO | runtests.py | TSan using symbolizer at %s"
+                     % llvmsym)
+        else:
+            log.info("TEST-UNEXPECTED-FAIL | runtests.py | Failed to find TSan"
+                     " symbolizer at %s" % llvmsym)
+
+    return env