From d4f0266bdc2904e9df004945d149f63851d8e0a8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Julien=20Pag=C3=A8s?= Date: Wed, 1 Jul 2015 22:00:07 +0200 Subject: [PATCH] Bug 1091284 - Remove systemMemory, environment from automationutils. r=jgriffin --- build/automationutils.py | 132 ---------------- layout/tools/reftest/runreftest.py | 5 +- testing/mochitest/runtests.py | 13 +- testing/mozbase/mozrunner/mozrunner/utils.py | 149 +++++++++++++++++++ 4 files changed, 159 insertions(+), 140 deletions(-) diff --git a/build/automationutils.py b/build/automationutils.py index 9bf4ea05ac2..4afd7315dc5 100644 --- a/build/automationutils.py +++ b/build/automationutils.py @@ -18,8 +18,6 @@ import mozinfo __all__ = [ "dumpLeakLog", "processLeakLog", - 'systemMemory', - 'environment', 'dumpScreen', "setAutomationLog", ] @@ -282,136 +280,6 @@ def processLeakLog(leakLogFile, options): processSingleLeakFile(thisFile, processType, leakThreshold, processType in ignoreMissingLeaks) -def systemMemory(): - """ - Returns total system memory in kilobytes. - Works only on unix-like platforms where `free` is in the path. - """ - return int(os.popen("free").readlines()[1].split()[1]) - -def environment(xrePath, env=None, crashreporter=True, debugger=False, dmdPath=None, lsanPath=None): - """populate OS environment variables for mochitest""" - - env = os.environ.copy() if env is None else env - - assert os.path.isabs(xrePath) - - if mozinfo.isMac: - ldLibraryPath = os.path.join(os.path.dirname(xrePath), "MacOS") - else: - ldLibraryPath = xrePath - - envVar = None - dmdLibrary = None - preloadEnvVar = None - if 'toolkit' in mozinfo.info and mozinfo.info['toolkit'] == "gonk": - # Skip all of this, it's only valid for the host. - pass - elif mozinfo.isUnix: - envVar = "LD_LIBRARY_PATH" - env['MOZILLA_FIVE_HOME'] = xrePath - dmdLibrary = "libdmd.so" - preloadEnvVar = "LD_PRELOAD" - elif mozinfo.isMac: - envVar = "DYLD_LIBRARY_PATH" - dmdLibrary = "libdmd.dylib" - preloadEnvVar = "DYLD_INSERT_LIBRARIES" - elif mozinfo.isWin: - envVar = "PATH" - dmdLibrary = "dmd.dll" - preloadEnvVar = "MOZ_REPLACE_MALLOC_LIB" - if envVar: - envValue = ((env.get(envVar), str(ldLibraryPath)) - if mozinfo.isWin - else (ldLibraryPath, dmdPath, env.get(envVar))) - env[envVar] = os.path.pathsep.join([path for path in envValue if path]) - - if dmdPath and dmdLibrary and preloadEnvVar: - env[preloadEnvVar] = os.path.join(dmdPath, dmdLibrary) - - # crashreporter - env['GNOME_DISABLE_CRASH_DIALOG'] = '1' - env['XRE_NO_WINDOWS_CRASH_DIALOG'] = '1' - - if crashreporter and not debugger: - env['MOZ_CRASHREPORTER_NO_REPORT'] = '1' - env['MOZ_CRASHREPORTER'] = '1' - else: - env['MOZ_CRASHREPORTER_DISABLE'] = '1' - - # Crash on non-local network connections by default. - # MOZ_DISABLE_NONLOCAL_CONNECTIONS can be set to "0" to temporarily - # enable non-local connections for the purposes of local testing. Don't - # override the user's choice here. See bug 1049688. - env.setdefault('MOZ_DISABLE_NONLOCAL_CONNECTIONS', '1') - - # Set WebRTC logging in case it is not set yet - env.setdefault('NSPR_LOG_MODULES', 'signaling:3,mtransport:4,datachannel:4,jsep:4,MediaPipelineFactory:4') - env.setdefault('R_LOG_LEVEL', '6') - env.setdefault('R_LOG_DESTINATION', 'stderr') - env.setdefault('R_LOG_VERBOSE', '1') - - # ASan specific environment stuff - asan = bool(mozinfo.info.get("asan")) - if asan and (mozinfo.isLinux or mozinfo.isMac): - try: - # Symbolizer support - llvmsym = os.path.join(xrePath, "llvm-symbolizer") - if os.path.isfile(llvmsym): - env["ASAN_SYMBOLIZER_PATH"] = llvmsym - log.info("INFO | runtests.py | ASan using symbolizer at %s" % llvmsym) - else: - log.info("TEST-UNEXPECTED-FAIL | runtests.py | Failed to find ASan symbolizer at %s" % llvmsym) - - totalMemory = systemMemory() - - # Only 4 GB RAM or less available? Use custom ASan options to reduce - # the amount of resources required to do the tests. Standard options - # will otherwise lead to OOM conditions on the current test slaves. - message = "INFO | runtests.py | ASan running in %s configuration" - asanOptions = [] - if totalMemory <= 1024 * 1024 * 4: - message = message % 'low-memory' - asanOptions = ['quarantine_size=50331648', 'malloc_context_size=5'] - else: - message = message % 'default memory' - - if lsanPath: - log.info("LSan enabled.") - asanOptions.append('detect_leaks=1') - lsanOptions = ["exitcode=0"] - suppressionsFile = os.path.join(lsanPath, 'lsan_suppressions.txt') - if os.path.exists(suppressionsFile): - log.info("LSan using suppression file " + suppressionsFile) - lsanOptions.append("suppressions=" + suppressionsFile) - else: - log.info("WARNING | runtests.py | LSan suppressions file does not exist! " + suppressionsFile) - env["LSAN_OPTIONS"] = ':'.join(lsanOptions) - # Run shutdown GCs and CCs to avoid spurious leaks. - env['MOZ_CC_RUN_DURING_SHUTDOWN'] = '1' - - if len(asanOptions): - env['ASAN_OPTIONS'] = ':'.join(asanOptions) - - except OSError,err: - log.info("Failed determine available memory, disabling ASan low-memory configuration: %s" % err.strerror) - except: - log.info("Failed determine available memory, disabling ASan low-memory configuration") - else: - log.info(message) - - tsan = bool(mozinfo.info.get("tsan")) - if tsan and mozinfo.isLinux: - # Symbolizer support. - llvmsym = os.path.join(xrePath, "llvm-symbolizer") - if os.path.isfile(llvmsym): - env["TSAN_OPTIONS"] = "external_symbolizer_path=%s" % llvmsym - log.info("INFO | runtests.py | TSan using symbolizer at %s" % llvmsym) - else: - log.info("TEST-UNEXPECTED-FAIL | runtests.py | Failed to find TSan symbolizer at %s" % llvmsym) - - return env - def dumpScreen(utilityPath): """dumps a screenshot of the entire screen to a directory specified by the MOZ_UPLOAD_DIR environment variable""" diff --git a/layout/tools/reftest/runreftest.py b/layout/tools/reftest/runreftest.py index d5163b197cb..f39dac67f75 100644 --- a/layout/tools/reftest/runreftest.py +++ b/layout/tools/reftest/runreftest.py @@ -23,7 +23,6 @@ sys.path.insert(0, SCRIPT_DIRECTORY) from automationutils import ( dumpScreen, - environment, printstatus, processLeakLog ) @@ -33,6 +32,7 @@ import mozinfo import mozprocess import mozprofile import mozrunner +from mozrunner.utils import test_environment here = os.path.abspath(os.path.dirname(__file__)) @@ -273,7 +273,8 @@ class RefTest(object): return profile def environment(self, **kwargs): - return environment(**kwargs) + kwargs['log'] = log + return test_environment(**kwargs) def buildBrowserEnv(self, options, profileDir): browserEnv = self.environment(xrePath = options.xrePath, debugger=options.debugger) diff --git a/testing/mochitest/runtests.py b/testing/mochitest/runtests.py index b2b411f8302..63fceb0dc45 100644 --- a/testing/mochitest/runtests.py +++ b/testing/mochitest/runtests.py @@ -35,7 +35,6 @@ import zipfile import bisection from automationutils import ( - environment, processLeakLog, dumpScreen, printstatus, @@ -59,6 +58,7 @@ from mozprofile.permissions import ServerLocations from urllib import quote_plus as encodeURIComponent from mozlog.structured.formatters import TbplFormatter from mozlog.structured import commandline +from mozrunner.utils import test_environment here = os.path.abspath(os.path.dirname(__file__)) @@ -359,7 +359,7 @@ class MochitestServer(object): "Run the Mochitest server, returning the process ID of the server." # get testing environment - env = environment(xrePath=self._xrePath) + env = test_environment(xrePath=self._xrePath, log=self._log) env["XPCOM_DEBUG_BREAK"] = "warn" env["LD_LIBRARY_PATH"] = self._xrePath @@ -1064,7 +1064,7 @@ class SSLTunnel: ssltunnel) exit(1) - env = environment(xrePath=self.xrePath) + env = test_environment(xrePath=self.xrePath, log=self.log) env["LD_LIBRARY_PATH"] = self.xrePath self.process = mozprocess.ProcessHandler([ssltunnel, self.configFile], env=env) @@ -1234,9 +1234,6 @@ class Mochitest(MochitestUtilsMixin): def __init__(self, logger_options): super(Mochitest, self).__init__(logger_options) - # environment function for browserEnv - self.environment = environment - # Max time in seconds to wait for server startup before tests will fail -- if # this seems big, it's mostly for debug machines where cold startup # (particularly after a build) takes forever. @@ -1254,6 +1251,10 @@ class Mochitest(MochitestUtilsMixin): self.expectedError = {} self.result = {} + def environment(self, **kwargs): + kwargs['log'] = self.log + return test_environment(**kwargs) + def extraPrefs(self, extraPrefs): """interpolate extra preferences from option strings""" diff --git a/testing/mozbase/mozrunner/mozrunner/utils.py b/testing/mozbase/mozrunner/mozrunner/utils.py index 3848547f456..6b5582648e8 100644 --- a/testing/mozbase/mozrunner/mozrunner/utils.py +++ b/testing/mozbase/mozrunner/mozrunner/utils.py @@ -94,3 +94,152 @@ def uses_marionette(func): return ret return _ + + +def _raw_log(): + import logging + return logging.getLogger(__name__) + + +def test_environment(xrePath, env=None, crashreporter=True, debugger=False, + dmdPath=None, lsanPath=None, log=None): + """ + populate OS environment variables for mochitest and reftests. + + Originally comes from automationutils.py. Don't use that for new code. + """ + + env = os.environ.copy() if env is None else env + log = log or _raw_log() + + assert os.path.isabs(xrePath) + + if mozinfo.isMac: + ldLibraryPath = os.path.join(os.path.dirname(xrePath), "MacOS") + else: + ldLibraryPath = xrePath + + envVar = None + dmdLibrary = None + preloadEnvVar = None + if 'toolkit' in mozinfo.info and mozinfo.info['toolkit'] == "gonk": + # Skip all of this, it's only valid for the host. + pass + elif mozinfo.isUnix: + envVar = "LD_LIBRARY_PATH" + env['MOZILLA_FIVE_HOME'] = xrePath + dmdLibrary = "libdmd.so" + preloadEnvVar = "LD_PRELOAD" + elif mozinfo.isMac: + envVar = "DYLD_LIBRARY_PATH" + dmdLibrary = "libdmd.dylib" + preloadEnvVar = "DYLD_INSERT_LIBRARIES" + elif mozinfo.isWin: + envVar = "PATH" + dmdLibrary = "dmd.dll" + preloadEnvVar = "MOZ_REPLACE_MALLOC_LIB" + if envVar: + envValue = ((env.get(envVar), str(ldLibraryPath)) + if mozinfo.isWin + else (ldLibraryPath, dmdPath, env.get(envVar))) + env[envVar] = os.path.pathsep.join([path for path in envValue if path]) + + if dmdPath and dmdLibrary and preloadEnvVar: + env[preloadEnvVar] = os.path.join(dmdPath, dmdLibrary) + + # crashreporter + env['GNOME_DISABLE_CRASH_DIALOG'] = '1' + env['XRE_NO_WINDOWS_CRASH_DIALOG'] = '1' + + if crashreporter and not debugger: + env['MOZ_CRASHREPORTER_NO_REPORT'] = '1' + env['MOZ_CRASHREPORTER'] = '1' + else: + env['MOZ_CRASHREPORTER_DISABLE'] = '1' + + # Crash on non-local network connections by default. + # MOZ_DISABLE_NONLOCAL_CONNECTIONS can be set to "0" to temporarily + # enable non-local connections for the purposes of local testing. Don't + # override the user's choice here. See bug 1049688. + env.setdefault('MOZ_DISABLE_NONLOCAL_CONNECTIONS', '1') + + # Set WebRTC logging in case it is not set yet + env.setdefault( + 'NSPR_LOG_MODULES', + 'signaling:3,mtransport:4,datachannel:4,jsep:4,MediaPipelineFactory:4' + ) + env.setdefault('R_LOG_LEVEL', '6') + env.setdefault('R_LOG_DESTINATION', 'stderr') + env.setdefault('R_LOG_VERBOSE', '1') + + # ASan specific environment stuff + asan = bool(mozinfo.info.get("asan")) + if asan and (mozinfo.isLinux or mozinfo.isMac): + try: + # Symbolizer support + llvmsym = os.path.join(xrePath, "llvm-symbolizer") + if os.path.isfile(llvmsym): + env["ASAN_SYMBOLIZER_PATH"] = llvmsym + log.info("INFO | runtests.py | ASan using symbolizer at %s" + % llvmsym) + else: + log.info("TEST-UNEXPECTED-FAIL | runtests.py | Failed to find" + " ASan symbolizer at %s" % llvmsym) + + # Returns total system memory in kilobytes. + # Works only on unix-like platforms where `free` is in the path. + totalMemory = int(os.popen("free").readlines()[1].split()[1]) + + # Only 4 GB RAM or less available? Use custom ASan options to reduce + # the amount of resources required to do the tests. Standard options + # will otherwise lead to OOM conditions on the current test slaves. + message = "INFO | runtests.py | ASan running in %s configuration" + asanOptions = [] + if totalMemory <= 1024 * 1024 * 4: + message = message % 'low-memory' + asanOptions = [ + 'quarantine_size=50331648', 'malloc_context_size=5'] + else: + message = message % 'default memory' + + if lsanPath: + log.info("LSan enabled.") + asanOptions.append('detect_leaks=1') + lsanOptions = ["exitcode=0"] + suppressionsFile = os.path.join( + lsanPath, 'lsan_suppressions.txt') + if os.path.exists(suppressionsFile): + log.info("LSan using suppression file " + suppressionsFile) + lsanOptions.append("suppressions=" + suppressionsFile) + else: + log.info("WARNING | runtests.py | LSan suppressions file" + " does not exist! " + suppressionsFile) + env["LSAN_OPTIONS"] = ':'.join(lsanOptions) + # Run shutdown GCs and CCs to avoid spurious leaks. + env['MOZ_CC_RUN_DURING_SHUTDOWN'] = '1' + + if len(asanOptions): + env['ASAN_OPTIONS'] = ':'.join(asanOptions) + + except OSError, err: + log.info("Failed determine available memory, disabling ASan" + " low-memory configuration: %s" % err.strerror) + except: + log.info("Failed determine available memory, disabling ASan" + " low-memory configuration") + else: + log.info(message) + + tsan = bool(mozinfo.info.get("tsan")) + if tsan and mozinfo.isLinux: + # Symbolizer support. + llvmsym = os.path.join(xrePath, "llvm-symbolizer") + if os.path.isfile(llvmsym): + env["TSAN_OPTIONS"] = "external_symbolizer_path=%s" % llvmsym + log.info("INFO | runtests.py | TSan using symbolizer at %s" + % llvmsym) + else: + log.info("TEST-UNEXPECTED-FAIL | runtests.py | Failed to find TSan" + " symbolizer at %s" % llvmsym) + + return env