diff --git a/build/pgo/profileserver.py b/build/pgo/profileserver.py index 16cc3a721f6..05212ca75a2 100644 --- a/build/pgo/profileserver.py +++ b/build/pgo/profileserver.py @@ -20,7 +20,7 @@ from automationutils import getDebuggerInfo, addCommonOptions PORT = 8888 PROFILE_DIRECTORY = os.path.abspath(os.path.join(SCRIPT_DIR, "./pgoprofile")) -MOZ_JAR_LOG_DIR = os.path.abspath(os.getenv("JARLOG_DIR")) +MOZ_JAR_LOG_FILE = os.path.abspath(os.getenv("JARLOG_FILE")) os.chdir(SCRIPT_DIR) class EasyServer(SocketServer.TCPServer): @@ -47,7 +47,7 @@ if __name__ == '__main__': automation.initializeProfile(PROFILE_DIRECTORY) browserEnv = automation.environment() browserEnv["XPCOM_DEBUG_BREAK"] = "warn" - browserEnv["MOZ_JAR_LOG_DIR"] = MOZ_JAR_LOG_DIR + browserEnv["MOZ_JAR_LOG_FILE"] = MOZ_JAR_LOG_FILE url = "http://localhost:%d/index.html" % PORT appPath = os.path.join(SCRIPT_DIR, automation.DEFAULT_APP) diff --git a/client.mk b/client.mk index 0d8dfc2effe..8bca03b4970 100644 --- a/client.mk +++ b/client.mk @@ -203,7 +203,8 @@ endif profiledbuild:: $(MAKE) -f $(TOPSRCDIR)/client.mk realbuild MOZ_PROFILE_GENERATE=1 MOZ_PGO_INSTRUMENTED=1 $(MAKE) -C $(PGO_OBJDIR) package MOZ_PGO_INSTRUMENTED=1 MOZ_INTERNAL_SIGNING_FORMAT= MOZ_EXTERNAL_SIGNING_FORMAT= - MOZ_PGO_INSTRUMENTED=1 OBJDIR=${PGO_OBJDIR} JARLOG_DIR=${PGO_OBJDIR}/jarlog/en-US $(PROFILE_GEN_SCRIPT) + rm -f ${PGO_OBJDIR}/jarlog/en-US.log + MOZ_PGO_INSTRUMENTED=1 OBJDIR=${PGO_OBJDIR} JARLOG_FILE=${PGO_OBJDIR}/jarlog/en-US.log $(PROFILE_GEN_SCRIPT) $(MAKE) -f $(TOPSRCDIR)/client.mk maybe_clobber_profiledbuild $(MAKE) -f $(TOPSRCDIR)/client.mk realbuild MOZ_PROFILE_USE=1 diff --git a/modules/libjar/nsZipArchive.cpp b/modules/libjar/nsZipArchive.cpp index ba60414970f..6592a1a4005 100644 --- a/modules/libjar/nsZipArchive.cpp +++ b/modules/libjar/nsZipArchive.cpp @@ -55,6 +55,9 @@ # endif #endif /* XP_UNIX */ +#ifdef XP_WIN +#include "private/pprio.h" // To get PR_ImportFile +#endif using namespace mozilla; @@ -70,6 +73,71 @@ static uint32_t HashName(const char* aName, uint16_t nameLen); static nsresult ResolveSymlink(const char *path); #endif +class ZipArchiveLogger { +public: + void Write(const nsACString &zip, const char *entry) const { + if (!fd) { + char *env = PR_GetEnv("MOZ_JAR_LOG_FILE"); + if (!env) + return; + + nsCOMPtr logFile; + nsresult rv = NS_NewLocalFile(NS_ConvertUTF8toUTF16(env), false, getter_AddRefs(logFile)); + if (NS_FAILED(rv)) + return; + + // Create the log file and its parent directory (in case it doesn't exist) + logFile->Create(nsIFile::NORMAL_FILE_TYPE, 0644); + + PRFileDesc* file; +#ifdef XP_WIN + // PR_APPEND is racy on Windows, so open a handle ourselves with flags that + // will work, and use PR_ImportFile to make it a PRFileDesc. + // This can go away when bug 840435 is fixed. + nsAutoString path; + logFile->GetPath(path); + if (path.IsEmpty()) + return; + HANDLE handle = CreateFileW(path.get(), FILE_APPEND_DATA, FILE_SHARE_WRITE, + NULL, OPEN_ALWAYS, 0, NULL); + if (handle == INVALID_HANDLE_VALUE) + return; + file = PR_ImportFile((PROsfd)handle); + if (!file) + return; +#else + rv = logFile->OpenNSPRFileDesc(PR_WRONLY|PR_CREATE_FILE|PR_APPEND, 0644, &file); + if (NS_FAILED(rv)) + return; +#endif + fd = file; + } + nsCString buf(zip); + buf.Append(" "); + buf.Append(entry); + buf.Append('\n'); + PR_Write(fd, buf.get(), buf.Length()); + } + + void AddRef() { + MOZ_ASSERT(refCnt >= 0); + ++refCnt; + } + + void Release() { + MOZ_ASSERT(refCnt > 0); + if ((0 == --refCnt) && fd) { + PR_Close(fd); + fd = NULL; + } + } +private: + int refCnt; + mutable PRFileDesc *fd; +}; + +static ZipArchiveLogger zipLog; + //*********************************************************** // For every inflation the following allocations are done: // malloc(1 * 9520) @@ -189,27 +257,10 @@ nsresult nsZipArchive::OpenArchive(nsZipHandle *aZipHandle) //-- get table of contents for archive nsresult rv = BuildFileList(); - char *env = PR_GetEnv("MOZ_JAR_LOG_DIR"); - if (env && NS_SUCCEEDED(rv) && aZipHandle->mFile) { - nsCOMPtr logFile; - nsresult rv2 = NS_NewLocalFile(NS_ConvertUTF8toUTF16(env), false, getter_AddRefs(logFile)); - - if (!NS_SUCCEEDED(rv2)) - return rv; - - // Create a directory for the log (in case it doesn't exist) - logFile->Create(nsIFile::DIRECTORY_TYPE, 0700); - - nsAutoString name; - nsCOMPtr file = aZipHandle->mFile.GetBaseFile(); - file->GetLeafName(name); - name.Append(NS_LITERAL_STRING(".log")); - logFile->Append(name); - - PRFileDesc* fd; - rv2 = logFile->OpenNSPRFileDesc(PR_WRONLY|PR_CREATE_FILE|PR_APPEND, 0644, &fd); - if (NS_SUCCEEDED(rv2)) - mLog = fd; + if (NS_SUCCEEDED(rv)) { + zipLog.AddRef(); + if (aZipHandle->mFile) + aZipHandle->mFile.GetURIString(mURI); } return rv; } @@ -276,6 +327,7 @@ nsresult nsZipArchive::CloseArchive() // Let us also cleanup the mFiles table for re-use on the next 'open' call memset(mFiles, 0, sizeof(mFiles)); mBuiltSynthetics = false; + zipLog.Release(); return NS_OK; } @@ -300,13 +352,8 @@ MOZ_WIN_MEM_TRY_BEGIN if ((len == item->nameLength) && (!memcmp(aEntryName, item->Name(), len))) { - if (mLog) { - // Successful GetItem() is a good indicator that the file is about to be read - char *tmp = PL_strdup(aEntryName); - tmp[len]='\n'; - PR_Write(mLog, tmp, len+1); - PL_strfree(tmp); - } + // Successful GetItem() is a good indicator that the file is about to be read + zipLog.Write(mURI, aEntryName); return item; //-- found it } item = item->next; diff --git a/modules/libjar/nsZipArchive.h b/modules/libjar/nsZipArchive.h index 4b8229bbebe..895a19ebd1d 100644 --- a/modules/libjar/nsZipArchive.h +++ b/modules/libjar/nsZipArchive.h @@ -217,9 +217,8 @@ private: // file handle nsRefPtr mFd; - // logging handle - mozilla::AutoFDClose mLog; - + // file URI, for logging + nsCString mURI; private: //--- private methods --- diff --git a/python/mozbuild/mozpack/chrome/manifest.py b/python/mozbuild/mozpack/chrome/manifest.py index 57a19a67ba3..39ab0029b69 100644 --- a/python/mozbuild/mozpack/chrome/manifest.py +++ b/python/mozbuild/mozpack/chrome/manifest.py @@ -349,7 +349,7 @@ def parse_manifest(root, path, fileobj=None): if not fileobj: fileobj = open(path) linenum = 0 - for line in fileobj.readlines(): + for line in fileobj: linenum += 1 with errors.context(path, linenum): e = parse_manifest_line(base, line) diff --git a/python/mozbuild/mozpack/mozjar.py b/python/mozbuild/mozpack/mozjar.py index 9baf13e2cda..4228ee48f44 100644 --- a/python/mozbuild/mozpack/mozjar.py +++ b/python/mozbuild/mozpack/mozjar.py @@ -11,6 +11,8 @@ from zipfile import ( ZIP_DEFLATED, ) from collections import OrderedDict +from urlparse import urlparse, ParseResult +import mozpack.path JAR_STORED = ZIP_STORED JAR_DEFLATED = ZIP_DEFLATED @@ -279,6 +281,12 @@ class JarFileReader(object): ''' return self.read().splitlines(True) + def __iter__(self): + ''' + Iterator, to support the "for line in fileobj" constructs. + ''' + return iter(self.readlines()) + def seek(self, pos, whence=os.SEEK_SET): ''' Change the current position in the uncompressed data. Subsequent reads @@ -732,3 +740,60 @@ class Deflater(object): if self.compressed: return self._deflated.getvalue() return self._data.getvalue() + + +class JarLog(dict): + ''' + Helper to read the file Gecko generates when setting MOZ_JAR_LOG_FILE. + The jar log is then available as a dict with the jar path as key (see + canonicalize for more details on the key value), and the corresponding + access log as a list value. Only the first access to a given member of + a jar is stored. + ''' + def __init__(self, file=None, fileobj=None): + if not fileobj: + fileobj = open(file, 'r') + urlmap = {} + for line in fileobj: + url, path = line.strip().split(None, 1) + if not url or not path: + continue + if url not in urlmap: + urlmap[url] = JarLog.canonicalize(url) + jar = urlmap[url] + entry = self.setdefault(jar, []) + if path not in entry: + entry.append(path) + + @staticmethod + def canonicalize(url): + ''' + The jar path is stored in a MOZ_JAR_LOG_FILE log as a url. This method + returns a unique value corresponding to such urls. + - file:///{path} becomes {path} + - jar:file:///{path}!/{subpath} becomes ({path}, {subpath}) + - jar:jar:file:///{path}!/{subpath}!/{subpath2} becomes + ({path}, {subpath}, {subpath2}) + ''' + if not isinstance(url, ParseResult): + # Assume that if it doesn't start with jar: or file:, it's a path. + if not url.startswith(('jar:', 'file:')): + url = 'file:///' + os.path.abspath(url) + url = urlparse(url) + assert url.scheme + assert url.scheme in ('jar', 'file') + if url.scheme == 'jar': + path = JarLog.canonicalize(url.path) + if isinstance(path, tuple): + return path[:-1] + tuple(path[-1].split('!/', 1)) + return tuple(path.split('!/', 1)) + if url.scheme == 'file': + assert os.path.isabs(url.path) + path = url.path + # On Windows, url.path will be /drive:/path ; on Unix systems, + # /path. As we want drive:/path instead of /drive:/path on Windows, + # remove the leading /. + if os.path.isabs(path[1:]): + path = path[1:] + path = os.path.realpath(path) + return mozpack.path.normsep(os.path.normcase(path)) diff --git a/python/mozbuild/mozpack/test/test_mozjar.py b/python/mozbuild/mozpack/test/test_mozjar.py index c4c73d8e22a..8e155be13dc 100644 --- a/python/mozbuild/mozpack/test/test_mozjar.py +++ b/python/mozbuild/mozpack/test/test_mozjar.py @@ -9,11 +9,16 @@ from mozpack.mozjar import ( JarReader, JarWriter, Deflater, - OrderedDict, + JarLog, ) +from collections import OrderedDict from mozpack.test.test_files import MockDest import unittest import mozunit +from cStringIO import StringIO +from urllib import pathname2url +import mozpack.path +import os class TestJarStruct(unittest.TestCase): @@ -255,5 +260,56 @@ class TestPreload(unittest.TestCase): self.assertEqual(files[2].filename, 'foo') +class TestJarLog(unittest.TestCase): + def test_jarlog(self): + base = 'file:' + pathname2url(os.path.abspath(os.curdir)) + s = StringIO('\n'.join([ + base + '/bar/baz.jar first', + base + '/bar/baz.jar second', + base + '/bar/baz.jar third', + base + '/bar/baz.jar second', + base + '/bar/baz.jar second', + 'jar:' + base + '/qux.zip!/omni.ja stuff', + base + '/bar/baz.jar first', + 'jar:' + base + '/qux.zip!/omni.ja other/stuff', + 'jar:' + base + '/qux.zip!/omni.ja stuff', + base + '/bar/baz.jar third', + 'jar:jar:' + base + '/qux.zip!/baz/baz.jar!/omni.ja nested/stuff', + 'jar:jar:jar:' + base + '/qux.zip!/baz/baz.jar!/foo.zip!/omni.ja' + + ' deeply/nested/stuff', + ])) + log = JarLog(fileobj=s) + canonicalize = lambda p: \ + mozpack.path.normsep(os.path.normcase(os.path.realpath(p))) + baz_jar = canonicalize('bar/baz.jar') + qux_zip = canonicalize('qux.zip') + self.assertEqual(set(log.keys()), set([ + baz_jar, + (qux_zip, 'omni.ja'), + (qux_zip, 'baz/baz.jar', 'omni.ja'), + (qux_zip, 'baz/baz.jar', 'foo.zip', 'omni.ja'), + ])) + self.assertEqual(log[baz_jar], [ + 'first', + 'second', + 'third', + ]) + self.assertEqual(log[(qux_zip, 'omni.ja')], [ + 'stuff', + 'other/stuff', + ]) + self.assertEqual(log[(qux_zip, 'baz/baz.jar', 'omni.ja')], + ['nested/stuff']) + self.assertEqual(log[(qux_zip, 'baz/baz.jar', 'foo.zip', + 'omni.ja')], ['deeply/nested/stuff']) + + # The above tests also indirectly check the value returned by + # JarLog.canonicalize for various jar: and file: urls, but + # JarLog.canonicalize also supports plain paths. + self.assertEqual(JarLog.canonicalize(os.path.abspath('bar/baz.jar')), + baz_jar) + self.assertEqual(JarLog.canonicalize('bar/baz.jar'), baz_jar) + + if __name__ == '__main__': mozunit.main() diff --git a/toolkit/mozapps/installer/packager.mk b/toolkit/mozapps/installer/packager.mk index fcbd3fda770..7b9c3004676 100644 --- a/toolkit/mozapps/installer/packager.mk +++ b/toolkit/mozapps/installer/packager.mk @@ -98,7 +98,7 @@ endif # LIBXUL_SDK _ABS_DIST = $(call core_abspath,$(DIST)) JARLOG_DIR = $(call core_abspath,$(DEPTH)/jarlog/) -JARLOG_DIR_AB_CD = $(JARLOG_DIR)/$(AB_CD) +JARLOG_FILE_AB_CD = $(JARLOG_DIR)/$(AB_CD).log TAR_CREATE_FLAGS := --exclude=.mkdir.done $(TAR_CREATE_FLAGS) CREATE_FINAL_TAR = $(TAR) -c --owner=0 --group=0 --numeric-owner \ @@ -591,7 +591,7 @@ stage-package: $(MOZ_PKG_MANIFEST) $(addprefix --removals ,$(MOZ_PKG_REMOVALS)) \ $(if $(filter-out 0,$(MOZ_PKG_FATAL_WARNINGS)),,--ignore-errors) \ $(if $(MOZ_PACKAGER_MINIFY),--minify) \ - $(if $(JARLOG_DIR),--jarlogs $(JARLOG_DIR_AB_CD)) \ + $(if $(JARLOG_DIR),$(addprefix --jarlog ,$(wildcard $(JARLOG_FILE_AB_CD)))) \ $(if $(OPTIMIZEJARS),--optimizejars) \ $(addprefix --unify ,$(UNIFY_DIST)) \ $(MOZ_PKG_MANIFEST) $(DIST) $(DIST)/$(STAGEPATH)$(MOZ_PKG_DIR) \ diff --git a/toolkit/mozapps/installer/packager.py b/toolkit/mozapps/installer/packager.py index 7cfdad3596b..20e420495ac 100644 --- a/toolkit/mozapps/installer/packager.py +++ b/toolkit/mozapps/installer/packager.py @@ -230,8 +230,8 @@ def main(): help='Transform errors into warnings.') parser.add_argument('--minify', action='store_true', default=False, help='Make some files more compact while packaging') - parser.add_argument('--jarlogs', default='', help='Base directory where ' + - 'to find jar content access logs') + parser.add_argument('--jarlog', default='', help='File containing jar ' + + 'access logs') parser.add_argument('--optimizejars', action='store_true', default=False, help='Enable jar optimizations') parser.add_argument('--unify', default='', @@ -331,13 +331,15 @@ def main(): libname))) # Setup preloading - if args.jarlogs: - jarlogs = FileFinder(args.jarlogs) - for p, log in jarlogs: - if p.endswith('.log'): - p = p[:-4] - if copier.contains(p) and isinstance(copier[p], Jarrer): - copier[p].preload([l.strip() for l in log.open().readlines()]) + if args.jarlog and os.path.exists(args.jarlog): + from mozpack.mozjar import JarLog + log = JarLog(args.jarlog) + for p, f in copier: + if not isinstance(f, Jarrer): + continue + key = JarLog.canonicalize(os.path.join(args.destination, p)) + if key in log: + f.preload(log[key]) # Fill startup cache if isinstance(formatter, OmniJarFormatter) and launcher.can_launch():