Imported Upstream version 5.0.0.42

Former-commit-id: fd56571888259555122d8a0f58c68838229cea2b

external/bockbuild/bockbuild/darwinprofile.py

@@ -0,0 +1,327 @@
+import os
+import shutil
+from plistlib import Plist
+from util.util import *
+from unixprofile import UnixProfile
+from profile import Profile
+import stat
+
+# staging helper functions
+
+
+def match_stageable_text(path, filetype):
+    if os.path.islink(path) or os.path.isdir(path):
+        return False
+    return path.endswith('.pc') or 'libtool library file' in filetype or 'text executable' in filetype
+
+
+def match_text(path, filetype):
+    if os.path.islink(path) or os.path.isdir(path):
+        return False
+    return match_stageable_text(path, filetype) or 'text' in filetype
+
+
+def match_stageable_binary(path, filetype):
+    if os.path.islink(path) or os.path.isdir(path):
+        return False
+    return 'Mach-O' in filetype and not path.endswith('.a') and not 'dSYM' in path
+
+
+def match_symlinks(path, filetype):
+    return os.path.islink(path)
+
+
+def match_real_files(path, filetype):
+    return (not os.path.islink(path) and not os.path.isdir(path))
+
+
+class DarwinProfile (UnixProfile):
+
+    # package order is very important.
+    # autoconf and automake don't depend on CC
+    # ccache uses a different CC since it's not installed yet
+    # every thing after ccache needs a working ccache
+    default_toolchain = [
+        'autoconf',
+        'automake',
+        'ccache',
+        'libtool',
+        'xz',
+        'tar',
+        'gtk-osx-docbook',
+        'gtk-doc',
+
+        # needed to autogen gtk+
+        'gtk-osx-docbook',
+        'gtk-doc'
+    ]
+
+    def attach (self, bockbuild):
+        UnixProfile.attach (self, bockbuild)
+        bockbuild.toolchain = list (DarwinProfile.default_toolchain)
+        self.name = 'darwin'
+
+        xcode_version = backtick('xcodebuild -version')[0]
+        self.env.set('xcode_version', xcode_version)
+        osx_sdk = backtick('xcrun --show-sdk-path')[0]
+        self.env.set('osx_sdk', osx_sdk)
+
+        if not os.path.exists(osx_sdk):
+            error('Mac OS X SDK not found under %s' % osx_sdk)
+
+        info('%s, %s' % (xcode_version, os.path.basename(osx_sdk)))
+
+        self.gcc_flags.extend([
+            '-D_XOPEN_SOURCE',
+            '-isysroot %s' % osx_sdk,
+            # needed to ensure install_name_tool can succeed staging binaries
+            '-Wl,-headerpad_max_install_names'
+        ])
+
+        self.ld_flags.extend([
+            # needed to ensure install_name_tool can succeed staging binaries
+            '-headerpad_max_install_names'
+        ])
+
+    def setup (self):
+        if self.min_version:
+            self.target_osx = '10.%s' % self.min_version
+            self.gcc_flags.extend(
+                ['-mmacosx-version-min=%s' % self.target_osx])
+            self.env.set('MACOSX_DEPLOYMENT_TARGET', self.target_osx)
+
+        if Profile.bockbuild.cmd_options.debug is True:
+            self.gcc_flags.extend(['-O0', '-ggdb3'])
+
+        if os.getenv('BOCKBUILD_USE_CCACHE') is None:
+            self.env.set('CC',  'xcrun gcc')
+            self.env.set('CXX', 'xcrun g++')
+        else:
+            self.env.set('CC',  'ccache xcrun gcc')
+            self.env.set('CXX', 'ccache xcrun g++')
+
+        self.debug_info = []
+
+        if self.bockbuild.cmd_options.arch == 'default':
+            self.bockbuild.cmd_options.arch = 'darwin-32'
+
+    def arch_build(self, arch, package):
+        if arch == 'darwin-universal':
+            package.local_ld_flags = ['-arch i386', '-arch x86_64']
+            package.local_gcc_flags = ['-arch i386', '-arch x86_64']
+        elif arch == 'darwin-32':
+            package.local_ld_flags = ['-arch i386', '-m32']
+            package.local_gcc_flags = ['-arch i386', '-m32']
+            package.local_configure_flags = [
+                '--build=i386-apple-darwin11.2.0', '--disable-dependency-tracking']
+        elif arch == 'darwin-64':
+            package.local_ld_flags = ['-arch x86_64 -m64']
+            package.local_gcc_flags = ['-arch x86_64 -m64']
+            package.local_configure_flags = ['--disable-dependency-tracking']
+        else:
+            error('Unknown arch %s' % arch)
+
+        configure_cache =  '%s/%s-%s.cache' % (self.bockbuild.build_root, package.name, arch)
+        package.aux_files.append (configure_cache)
+
+        package.local_configure_flags.extend(
+            ['--cache-file=%s' % configure_cache])
+
+        if package.name in self.debug_info:
+            package.local_gcc_flags.extend(['-g'])
+
+    def process_package(self, package):
+        failure_count = 0
+
+        def staging_harness(path, func, failure_count=failure_count):
+            def relocate_to_profile(token):
+                if token.find(package.staged_prefix) == -1 and token.find(package.staged_profile) == -1:
+                    newtoken = token.replace(
+                        package.package_prefix, package.staged_profile)
+                else:
+                    newtoken = token.replace(
+                        package.staged_prefix, package.staged_profile)
+
+                if newtoken != token:
+                    package.trace('%s:\n\t%s\t->\t%s' %
+                                  (os.path.basename(path), token, newtoken))
+                return newtoken
+
+            if (path.endswith('.release')):
+                error('Staging backup exists in dir we''re trying to stage: %s' % path)
+
+            backup = path + '.release'
+            shutil.copy2(path, backup)
+            try:
+                trace('Staging %s' % path)
+                func(path, relocate_to_profile)
+                if os.path.exists(path + '.stage'):
+                    os.remove(path)
+                    shutil.move(path + '.stage', path)
+                    shutil.copystat(backup, path)
+            except CommandException as e:
+                package.rm_if_exists(path)
+                shutil.copy2(backup, path)
+                package.rm(backup)
+                warn('Staging failed for %s' % os.path.basename(path))
+                error(str(e))
+                failure_count = failure_count + 1
+                if failure_count > 10:
+                    error('Possible staging issue, >10 staging failures')
+
+        extra_files = [os.path.join(package.staged_prefix, expand_macros(file, package))
+                       for file in package.extra_stage_files]
+
+        procs = []
+        if package.name in self.debug_info:
+            procs.append(self.generate_dsyms())
+
+        procs.append(self.stage_textfiles(harness=staging_harness,
+                                          match=match_stageable_text, extra_files=extra_files))
+        procs.append(self.stage_binaries(
+            harness=staging_harness, match=match_stageable_binary))
+
+        Profile.postprocess(self, procs, package.staged_prefix)
+
+    def process_release(self, directory):
+        # validate staged install
+        # TODO: move to end of '--build' instead of at the beginning of '--package'
+        # unprotect_dir (self.staged_prefix, recursive = True)
+        # Profile.postprocess (self, [self.validate_rpaths (match = self.match_stageable_binary),
+        # self.validate_symlinks (match = self.match_symlinks)],
+        # self.staged_prefix)
+
+        unprotect_dir(directory, recursive=True)
+
+        def destaging_harness(backup, func):
+            path = backup[0:-len('.release')]
+            trace(path)
+
+            def relocate_for_release(token):
+                newtoken = token.replace(self.staged_prefix, self.prefix).replace(directory, self.prefix)
+
+                if newtoken != token:
+                    trace('%s:\n\t%s\t->\t%s' %
+                          (os.path.basename(path), token, newtoken))
+
+                return newtoken
+
+            try:
+                trace('Destaging %s' % path)
+                func(path, relocate_for_release)
+                if os.path.exists(path + '.stage'):
+                    os.remove(path)
+                    shutil.move(path + '.stage', path)
+                    shutil.copystat(backup, path)
+                os.remove(backup)
+
+            except Exception as e:
+                warn ('Critical: Destaging failed for ''%s''' % path)
+                raise
+
+        procs = [self.stage_textfiles(harness=destaging_harness, match=match_text),
+                 self.stage_binaries(harness=destaging_harness, match=match_stageable_binary)]
+
+        Profile.postprocess(self, procs, directory,
+                            lambda l: l.endswith('.release'))
+
+    class validate_text_staging (Profile.FileProcessor):
+        problem_files = []
+
+        def __init__(self, package):
+            self.package = package
+            Profile.FileProcessor.__init__(self)
+
+        def process(self, path):
+            with open(path) as text:
+                stage_name = os.path.basename(self.package.stage_root)
+                for line in text:
+                    if stage_name in line:
+                        warn('String ''%s'' was found in %s' %
+                             (stage_name, self.relpath(path)))
+                        self.problem_files.append(self.relpath(path))
+
+        def end(self):
+            if len(self.problem_files) > 0:
+                error('Problematic staging files:\n' +
+                      '\n'.join(self.problem_files))
+
+    class validate_symlinks (Profile.FileProcessor):
+        problem_links = []
+
+        def process(self, path):
+            if path.endswith('.release'):
+                # get rid of these symlinks
+                os.remove(path)
+                return
+
+            target = os.path.realpath(path)
+            if not os.path.exists(target) or not target.startswith(self.root):
+                self.problem_links.append(
+                    '%s -> %s' % (self.relpath(path), target))
+
+        def end(self):
+            if len(self.problem_links) > 0:
+                # TODO: Turn into error when we handle them
+                warn('Broken symlinks:\n' + '\n'.join(self.problem_links))
+
+    class generate_dsyms (Profile.FileProcessor):
+
+        def __init__(self):
+            Profile.FileProcessor.__init__(self, match=match_stageable_binary)
+
+        def process(self, path):
+            run_shell('dsymutil -t 2 "%s" >/dev/null' % path)
+            run_shell('strip -S "%s" > /dev/null' % path)
+
+    class validate_rpaths (Profile.FileProcessor):
+
+        def process(self, path):
+            if path.endswith('.release'):
+                return
+            libs = backtick('otool -L %s' % path)
+            for line in libs:
+                # parse 'otool -L'
+                if not line.startswith('\t'):
+                    continue
+                rpath = line.strip().split(' ')[0]
+
+                if not os.path.exists(rpath):
+                    error('%s contains reference to non-existing file %s' %
+                          (self.relpath(path), rpath))
+                # if rpath.startswith (self.package.profile.MONO_ROOT):
+                # 	error ('%s is linking to external distribution %s' % (path, rpath))
+
+    class stage_textfiles (Profile.FileProcessor):
+
+        def process(self, path, fixup_func):
+            with open(path) as text:
+                output = open(path + '.stage', 'w')
+                for line in text:
+                    tokens = line.split(" ")
+                    for idx, token in enumerate(tokens):
+                        remap = fixup_func(token)
+                        tokens[idx] = remap
+
+                    output.write(" ".join(tokens))
+                output.close ()
+
+    class stage_binaries (Profile.FileProcessor):
+
+        def process(self, path, fixup_func):
+            staged_path = fixup_func(path)
+
+            run_shell('install_name_tool -id %s %s' %
+                      (staged_path, path), False)
+
+            libs = backtick('otool -L %s' % path)
+            for line in libs:
+                # parse 'otool -L'
+                if not line.startswith('\t'):
+                    continue
+                rpath = line.strip().split(' ')[0]
+
+                remap = fixup_func(rpath)
+                if remap != rpath:
+                    run_shell('install_name_tool -change %s %s %s' %
+                              (rpath, remap, path), False)

external/bockbuild/bockbuild/environment.py

@@ -0,0 +1,83 @@
+import os
+from util.util import *
+from util.csproj import *
+from collections import deque
+
+
+class EnvironmentItem:
+
+    def __init__(self, name, joinchar, values):
+        self.name = name
+        self.joinchar = joinchar
+        self.values = values
+
+    def __str__(self):
+        return self.joinchar.join(self.values)
+
+
+class Environment:
+
+    def __init__(self, profile):
+        self._profile = profile
+
+    def set(self, *argv):
+        args = deque(argv)
+        name = args.popleft()
+        joinchar = args.popleft()
+        if len(args) == 0:
+            values = list(self.iter_flatten(joinchar))
+            joinchar = ''
+        else:
+            values = list(self.iter_flatten(list(args)))
+
+        self.__dict__[name] = EnvironmentItem(name, joinchar, values)
+        return self.__dict__[name]
+
+    def compile(self):
+        expand_macros(self, self._profile)
+
+    def write_source_script(self, filename):
+
+        envscript = '#!/bin/sh\n'
+
+        for k in self.get_names():
+            envscript = envscript + 'export %s="%s"\n' % (k, self.__dict__[k])
+
+        with open(filename, 'w') as f:
+            f.write(envscript)
+
+        os.chmod(filename, 0o755)
+
+    def serialize(self):
+        names = sorted(self.get_names())
+        for k in names:
+            yield '%s = "%s"' % (k, self.__dict__[k])
+
+    def dump_csproj(self):
+        for k in self.get_names():
+            print '<Variable name="%s" value="%s" />' % (k, self.__dict__[k])
+
+    def write_csproj(self, file):
+        writer = csproj_writer(file, self)
+        writer.write()
+
+    def export(self):
+        for k in self.get_names():
+            os.environ[k] = str(self.__dict__[k])
+
+    def get_names(self):
+        for k in self.__dict__.keys():
+            if not k.startswith('_'):
+                yield k
+
+    def iter_flatten(self, iterable):
+        if not isinstance(iterable, (list, tuple)):
+            yield iterable
+            return
+        it = iter(iterable)
+        for e in it:
+            if isinstance(e, (list, tuple)):
+                for f in self.iter_flatten(e):
+                    yield f
+            else:
+                yield e

external/bockbuild/bockbuild/profile.py

@@ -0,0 +1,76 @@
+import os
+from optparse import OptionParser
+from util.util import *
+from util.csproj import *
+from environment import Environment
+from bockbuild.package import *
+import collections
+import hashlib
+import itertools
+
+
+class Profile:
+    def __init__ (self):
+        Profile.loaded = self
+
+    def attach (self, bockbuild):
+        Profile.bockbuild = bockbuild
+
+    class FileProcessor (object):
+
+        def __init__(self, harness=None, match=None, process=None,  extra_files=None):
+            self.harness = harness
+            self.match = match
+            self.files = list(extra_files) if extra_files else list()
+            self.root = None
+
+        def relpath(self, path):
+            return os.path.relpath(path, self.root)
+
+        def run(self):
+            for path in self.files:
+                self.harness(path, self.process)
+
+        def end(self):
+            return
+
+    def postprocess(self, processors, directory, filefilter=None):
+        def simple_harness(path, func):
+            if not os.path.lexists(path):
+                return  # file removed by previous processor function
+            # TODO: Fix so that it works on symlinks
+            # hash = hashlib.sha1(open(path).read()).hexdigest()
+            func(path)
+            if not os.path.lexists(path):
+                trace('Removed file: %s' % path)
+            # if hash != hashlib.sha1(open(path).read()).hexdigest():
+            #	warn ('Changed file: %s' % path)
+
+        for proc in processors:
+            proc.root = directory
+            if proc.harness is None:
+                proc.harness = simple_harness
+            if proc.match is None:
+                error('proc %s has no match function' %
+                      proc.__class__.__name__)
+
+        for path in filter(filefilter, iterate_dir(directory, with_dirs=True, with_links=True)):
+            filetype = get_filetype(path)
+            for proc in processors:
+                if proc.match(path, filetype) == True:
+                    trace('%s  matched %s / %s' % (proc.__class__.__name__,
+                                                   os.path.basename(path), filetype))
+                    proc.files.append(path)
+
+        for proc in processors:
+            verbose('%s: %s items' %
+                    (proc.__class__.__name__, len(proc.files)))
+            proc.run()
+
+        for proc in processors:
+            proc.end()
+            proc.harness = None
+            proc.files = []
+
+Profile.loaded = None
+Profile.bockbuild = None

external/bockbuild/bockbuild/unixprofile.py

@@ -0,0 +1,39 @@
+from profile import Profile
+from bockbuild.environment import Environment
+
+class UnixProfile (Profile):
+
+    def attach (self, bockbuild):
+        Profile.attach (self, bockbuild)
+
+        self.name = 'unix'
+        self.env = Environment(self)
+        self.staged_prefix = bockbuild.staged_prefix 
+        self.toolchain_root = bockbuild.toolchain_root
+
+        self.gcc_flags = ['-I%s/include' % self.staged_prefix]
+        self.ld_flags = ['-L%s/lib' % self.staged_prefix]
+
+        self.env.set('BUILD_PREFIX', '%{prefix}')
+
+        self.env.set('PATH', ':',
+                     '%{toolchain_root}/bin',
+                     '%{staged_prefix}/bin',
+                     '/usr/bin',
+                     '/bin',
+                     '/usr/local/git/bin')
+
+        self.env.set('C_INCLUDE_PATH',  '%{staged_prefix}/include')
+
+        #self.env.set ('LD_LIBRARY_PATH', '%{staged_prefix}/lib')
+
+        self.env.set('ACLOCAL_FLAGS',   '-I%{staged_prefix}/share/aclocal')
+        self.env.set('PKG_CONFIG_PATH', ':',
+                     '%{staged_prefix}/lib/pkgconfig',
+                     '%{staged_prefix}/share/pkgconfig',
+                     '%{toolchain_root}/lib/pkgconfig',
+                     '%{toolchain_root}/share/pkgconfig')
+
+        self.env.set('XDG_CONFIG_DIRS', '%{staged_prefix}/etc/xdg')
+        self.env.set('XDG_DATA_DIRS',   '%{staged_prefix}/share')
+        self.env.set('XDG_CONFIG_HOME', '$HOME/.config')

external/bockbuild/bockbuild/util/csproj.py

@@ -0,0 +1,93 @@
+#!/usr/bin/env python
+# coding: utf8
+
+import shutil
+import io
+
+from xml.etree import ElementTree
+from xml.etree.ElementTree import Element
+from xml.etree.ElementTree import Comment
+
+
+class csproj_writer:
+
+    def __init__(self, file, vars, condition="Debug|AnyCPU"):
+
+        self.vars = vars
+        self.file = file
+
+        self.tree = ElementTree.parse(file)
+
+        # get the namespace from the root Project element
+        self.ns = self.tree.getroot().tag[1:].split("}")[0]
+
+        # the python elementtree always puts a prefix,
+        # hardcode it so we can substitute it later
+        ElementTree.register_namespace("csproj0", self.ns)
+
+        # find PropertyGroup
+        group = self.tree.findall(".//{%s}PropertyGroup[@Condition]" % self.ns)
+
+        for node in group:
+            # only insert into Debug|AnyCPU
+            if condition in (node.get("Condition")):
+                self.insert_env_if_missing(node)
+                self.substitute_env_var(node)
+
+    def substitute_env_var(self, propertygroup):
+        node = propertygroup.find(
+            "./{%s}EnvironmentVariables/{%s}EnvironmentVariables" % (self.ns, self.ns))
+
+        for name in self.vars.get_names():
+
+            value = self.vars.__dict__[name]
+
+            # check if variable is already set
+            var = node.find("{%s}Variable[@name='%s']" % (self.ns, name))
+            if var is not None:
+                # update its value
+                var.set('value', "%s" % value)
+            else:
+                # insert new node
+                el = Element("{%s}Variable" % self.ns)
+                el.set('name', name)
+                el.set('value', "%s" % value)
+                self.insert(node, 0, el)
+
+    def insert_env_if_missing(self, node):
+        # test if environment variable is present - it is usually double
+        # wrapped
+        outer = node.find("{%s}EnvironmentVariables" % self.ns)
+        if outer is not None:
+            # see if the inner node is present, too
+            inner = node.find("{%s}EnvironmentVariables" % self.ns)
+            if inner is None:
+                inner = Element("{%s}EnvironmentVariables" % self.ns)
+                self.insert(outer, 0, inner)
+
+        else:
+            outer = Element("{%s}EnvironmentVariables" % self.ns)
+            inner = Element("{%s}EnvironmentVariables" % self.ns)
+            #self.insert (outer, 0, Comment ("AUTO GENERATED VARIABLES - DO NOT INCLUDE IN ANY GIT COMMITS!"))
+            self.insert(node, 1, outer)
+            self.insert(outer, 0, inner)
+
+    # wrapper arround Element.insert that appends a linebreak comment
+    # HACK etree xml library can not pretty print
+    def insert(self, node, pos, element):
+        brcomment = Comment("REPLACE_WITH_LINEBREAK")
+        node.insert(pos, brcomment)
+        node.insert(pos, element)
+
+    def write(self):
+
+        xml = ElementTree.tostring(self.tree.getroot(), encoding="utf-8")
+
+        # HACK the python xml library is nuts - manually remove the forced namespace
+        # prefix and re-establish a minimal form of pretty printing
+        xml = xml.replace("csproj0:", "")
+        xml = xml.replace("xmlns:csproj0", "xmlns")
+        xml = xml.replace("<!--REPLACE_WITH_LINEBREAK-->", "\n")
+
+        f = open(self.file, 'w')
+        f.write(xml)