gecko/toolkit/mozapps/installer/packager.py
2013-01-23 11:23:14 +01:00

363 lines
13 KiB
Python

# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
from mozpack.packager.formats import (
FlatFormatter,
JarFormatter,
OmniJarFormatter,
)
from mozpack.packager import (
preprocess_manifest,
preprocess,
SimpleManifestSink,
)
from mozpack.files import (
GeneratedFile,
FileFinder,
File,
)
from mozpack.copier import (
FileCopier,
Jarrer,
)
from mozpack.errors import errors
from mozpack.unify import UnifiedBuildFinder
import mozpack.path
import buildconfig
from argparse import ArgumentParser
from createprecomplete import generate_precomplete
import os
import re
import sys
from StringIO import StringIO
import subprocess
import platform
# List of libraries to shlibsign.
SIGN_LIBS = [
'softokn3',
'nssdbm3',
'freebl3',
'freebl_32fpu_3',
'freebl_32int_3',
'freebl_32int64_3',
'freebl_64fpu_3',
'freebl_64int_3',
]
class ToolLauncher(object):
'''
Helper to execute tools like xpcshell with the appropriate environment.
launcher = ToolLauncher()
launcher.tooldir = '/path/to/tools'
launcher.launch(['xpcshell', '-e', 'foo.js'])
'''
def __init__(self):
self.tooldir = None
def launch(self, cmd, extra_linker_path=None, extra_env={}):
'''
Launch the given command, passed as a list. The first item in the
command list is the program name, without a path and without a suffix.
These are determined from the tooldir member and the BIN_SUFFIX value.
An extra_linker_path may be passed to give an additional directory
to add to the search paths for the dynamic linker.
An extra_env dict may be passed to give additional environment
variables to export when running the command.
'''
assert self.tooldir
cmd[0] = os.path.join(self.tooldir, 'bin',
cmd[0] + buildconfig.substs['BIN_SUFFIX'])
if not extra_linker_path:
extra_linker_path = os.path.join(self.tooldir, 'bin')
env = dict(os.environ)
for p in ['LD_LIBRARY_PATH', 'DYLD_LIBRARY_PATH']:
if p in env:
env[p] = extra_linker_path + ':' + env[p]
else:
env[p] = extra_linker_path
for e in extra_env:
env[e] = extra_env[e]
print >>errors.out, 'Executing', ' '.join(cmd)
errors.out.flush()
return subprocess.call(cmd, env=env)
def can_launch(self):
return self.tooldir is not None
launcher = ToolLauncher()
class LibSignFile(File):
'''
File class for shlibsign signatures.
'''
def copy(self, dest):
assert isinstance(dest, basestring)
# os.path.getmtime returns a result in seconds with precision up to the
# microsecond. But microsecond is too precise because shutil.copystat
# only copies milliseconds, and seconds is not enough precision.
if os.path.exists(dest) and \
int(os.path.getmtime(self.path) * 1000) <= \
int(os.path.getmtime(dest) * 1000):
return False
if launcher.launch(['shlibsign', '-v', '-o', dest, '-i', self.path]):
errors.fatal('Error while signing %s' % self.path)
def precompile_cache(formatter, source_path, gre_path, app_path):
'''
Create startup cache for the given application directory, using the
given GRE path.
- formatter is a Formatter instance where to add the startup cache.
- source_path is the base path of the package.
- gre_path is the GRE path, relative to source_path.
- app_path is the application path, relative to source_path.
Startup cache for all resources under resource://app/ are generated,
except when gre_path == app_path, in which case it's under
resource://gre/.
'''
from tempfile import mkstemp
source_path = os.path.abspath(source_path)
if app_path != gre_path:
resource = 'app'
else:
resource = 'gre'
app_path = os.path.join(source_path, app_path)
gre_path = os.path.join(source_path, gre_path)
fd, cache = mkstemp('.zip')
os.close(fd)
os.remove(cache)
try:
if launcher.launch(['xpcshell', '-g', gre_path, '-a', app_path,
'-f', os.path.join(os.path.dirname(__file__),
'precompile_cache.js'),
'-e', 'precompile_startupcache("resource://%s/");'
% resource],
extra_linker_path=gre_path,
extra_env={'MOZ_STARTUP_CACHE': cache}):
errors.fatal('Error while running startup cache precompilation')
return
from mozpack.mozjar import JarReader
jar = JarReader(cache)
resource = '/resource/%s/' % resource
for f in jar:
if resource in f.filename:
path = f.filename[f.filename.index(resource) + len(resource):]
if formatter.contains(path):
formatter.add(f.filename, GeneratedFile(f.read()))
jar.close()
finally:
if os.path.exists(cache):
os.remove(cache)
class RemovedFiles(GeneratedFile):
'''
File class for removed-files. Is used as a preprocessor parser.
'''
def __init__(self, copier):
self.copier = copier
GeneratedFile.__init__(self, '')
def handle_line(self, str):
f = str.strip()
if self.copier.contains(f):
errors.error('Removal of packaged file(s): %s' % f)
self.content += f + '\n'
def split_define(define):
'''
Give a VAR[=VAL] string, returns a (VAR, VAL) tuple, where VAL defaults to
1. Numeric VALs are returned as ints.
'''
if '=' in define:
name, value = define.split('=', 1)
try:
value = int(value)
except ValueError:
pass
return (name, value)
return (define, 1)
class NoPkgFilesRemover(object):
'''
Formatter wrapper to handle NO_PKG_FILES.
'''
def __init__(self, formatter, has_manifest):
assert 'NO_PKG_FILES' in os.environ
self._formatter = formatter
self._files = os.environ['NO_PKG_FILES'].split()
if has_manifest:
self._error = errors.error
self._msg = 'NO_PKG_FILES contains file listed in manifest: %s'
else:
self._error = errors.warn
self._msg = 'Skipping %s'
def add_base(self, base):
self._formatter.add_base(base)
def add(self, path, content):
if not any(mozpack.path.match(path, spec) for spec in self._files):
self._formatter.add(path, content)
else:
self._error(self._msg % path)
def add_manifest(self, entry):
self._formatter.add_manifest(entry)
def add_interfaces(self, path, content):
self._formatter.add_interfaces(path, content)
def contains(self, path):
return self._formatter.contains(path)
def main():
parser = ArgumentParser()
parser.add_argument('-D', dest='defines', action='append',
metavar="VAR[=VAL]", help='Define a variable')
parser.add_argument('--format', default='omni',
help='Choose the chrome format for packaging ' +
'(omni, jar or flat ; default: %(default)s)')
parser.add_argument('--removals', default=None,
help='removed-files source file')
parser.add_argument('--ignore-errors', action='store_true', default=False,
help='Transform errors into warnings.')
parser.add_argument('--minify', action='store_true', default=False,
help='Make some files more compact while packaging')
parser.add_argument('--jarlogs', default='', help='Base directory where ' +
'to find jar content access logs')
parser.add_argument('--optimizejars', action='store_true', default=False,
help='Enable jar optimizations')
parser.add_argument('--unify', default='',
help='Base directory of another build to unify with')
parser.add_argument('manifest', default=None, nargs='?',
help='Manifest file name')
parser.add_argument('source', metavar="source",
help='Source directory')
parser.add_argument('destination',
help='Destination directory')
args = parser.parse_args()
defines = dict(buildconfig.defines)
if args.ignore_errors:
errors.ignore_errors()
if args.defines:
for name, value in [split_define(d) for d in args.defines]:
defines[name] = value
copier = FileCopier()
if args.format == 'flat':
formatter = FlatFormatter(copier)
elif args.format == 'jar':
formatter = JarFormatter(copier, optimize=args.optimizejars)
elif args.format == 'omni':
formatter = OmniJarFormatter(copier,
buildconfig.substs['OMNIJAR_NAME'],
optimize=args.optimizejars)
else:
errors.fatal('Unknown format: %s', format)
# Adjust defines according to the requested format.
if isinstance(formatter, OmniJarFormatter):
defines['MOZ_OMNIJAR'] = 1
elif 'MOZ_OMNIJAR' in defines:
del defines['MOZ_OMNIJAR']
binpath = ''
if 'BINPATH' in defines:
binpath = SimpleManifestSink.normalize_path(defines['BINPATH'])
while binpath.startswith('/'):
binpath = binpath[1:]
if args.unify:
def is_native(path):
path = os.path.abspath(path)
return platform.machine() in mozpack.path.split(path)
# Invert args.unify and args.source if args.unify points to the
# native architecture.
args.source, args.unify = sorted([args.source, args.unify],
key=is_native, reverse=True)
if is_native(args.source):
launcher.tooldir = args.source
elif not buildconfig.substs['CROSS_COMPILE']:
launcher.tooldir = buildconfig.substs['LIBXUL_DIST']
with errors.accumulate():
if args.unify:
finder = UnifiedBuildFinder(args.source, args.unify,
minify=args.minify)
else:
finder = FileFinder(args.source, minify=args.minify)
if 'NO_PKG_FILES' in os.environ:
sinkformatter = NoPkgFilesRemover(formatter,
args.manifest is not None)
else:
sinkformatter = formatter
sink = SimpleManifestSink(finder, sinkformatter)
if args.manifest:
preprocess_manifest(sink, args.manifest, defines)
else:
sink.add('', 'bin/*')
sink.close(args.manifest is not None)
if args.removals:
lines = [l.lstrip() for l in open(args.removals).readlines()]
removals_in = StringIO(''.join(lines))
removals_in.name = args.removals
removals = RemovedFiles(copier)
preprocess(removals_in, removals, defines)
copier.add(mozpack.path.join(binpath, 'removed-files'), removals)
# shlibsign libraries
if launcher.can_launch():
for lib in SIGN_LIBS:
libbase = mozpack.path.join(binpath, '%s%s') \
% (buildconfig.substs['DLL_PREFIX'], lib)
libname = '%s%s' % (libbase, buildconfig.substs['DLL_SUFFIX'])
if copier.contains(libname):
copier.add(libbase + '.chk',
LibSignFile(os.path.join(args.destination, libname)))
# Setup preloading
if args.jarlogs:
jarlogs = FileFinder(args.jarlogs)
for p, log in jarlogs:
if p.endswith('.log'):
p = p[:-4]
if copier.contains(p) and isinstance(copier[p], Jarrer):
copier[p].preload([l.strip() for l in log.open().readlines()])
# Fill startup cache
if isinstance(formatter, OmniJarFormatter) and launcher.can_launch():
if buildconfig.substs['LIBXUL_SDK']:
gre_path = buildconfig.substs['LIBXUL_DIST']
else:
gre_path = None
for base in sorted([[p for p in [mozpack.path.join('bin', b), b]
if os.path.exists(os.path.join(args.source, p))][0]
for b in sink.packager.get_bases()]):
if not gre_path:
gre_path = base
base_path = sink.normalize_path(base)
if base_path in formatter.omnijars:
precompile_cache(formatter.omnijars[base_path],
args.source, gre_path, base)
copier.copy(args.destination)
generate_precomplete(os.path.normpath(os.path.join(args.destination,
binpath)))
if __name__ == '__main__':
main()