Imported Upstream version 5.2.0.175

Former-commit-id: bb0468d0f257ff100aa895eb5fe583fb5dfbf900

external/bockbuild/bockbuild/darwinprofile.py

@@ -5,6 +5,7 @@ from util.util import *
 from unixprofile import UnixProfile
 from profile import Profile
 import stat
+from distutils.version import LooseVersion, StrictVersion
 
 # staging helper functions
 
@@ -56,20 +57,39 @@ class DarwinProfile (UnixProfile):
         'gtk-doc'
     ]
 
+    def use_Xcode(self, min_version='5.1.1', xcodebuild_version_prefix='Xcode '):
+        xcrun_cc_str = backtick('xcrun cc --version')[0]
+        cc_str = backtick('cc --version')[0]
+        if xcrun_cc_str != cc_str:
+            error('Multiple "cc" compiler versions found. (XCode-selected: "%s";"cc" at $PATH: "%s"' % (
+            xcrun_cc_str, cc_str))
+        xcodebuild_str = backtick('xcodebuild -version')[0]  # output: "Xcode X.X.X"
+        if not xcodebuild_str.startswith(xcodebuild_version_prefix):
+            error('Unexpected output from "xcodebuild" (first line: "%s"' % (xcodebuild_str))
+        xcode_version = StrictVersion(xcodebuild_str[len(xcodebuild_version_prefix):])
+        if xcode_version < StrictVersion(min_version):
+            error('Xcode version required %s, installed %s' % (min_version, xcode_version))
+
+        self.env.set('xcode_version', str(xcode_version))
+        return xcode_version
+
     def attach (self, bockbuild):
         UnixProfile.attach (self, bockbuild)
         bockbuild.toolchain = list (DarwinProfile.default_toolchain)
         self.name = 'darwin'
 
-        xcode_version = backtick('xcodebuild -version')[0]
-        self.env.set('xcode_version', xcode_version)
-        osx_sdk = backtick('xcrun --show-sdk-path')[0]
-        self.env.set('osx_sdk', osx_sdk)
+        xcode_version = self.use_Xcode ()
 
+        osx_sdk = backtick('xcrun --show-sdk-path')[0]
         if not os.path.exists(osx_sdk):
             error('Mac OS X SDK not found under %s' % osx_sdk)
 
-        info('%s, %s' % (xcode_version, os.path.basename(osx_sdk)))
+        info('Using Xcode %s, SDK %s' % (xcode_version, os.path.basename(osx_sdk)))
+
+        if xcode_version >= '8.0':
+            # based on https://github.com/Homebrew/brew/pull/970. This applies to XCode 8, OS X 10.11 and the 10.12 SDK. The following symbols will be unresolved
+            # when running binaries on a system of lower version than 10.12.
+            map(lambda t : self.configure_flags.append ('ac_cv_func_%s=no' % t), 'basename_r clock_getres clock_gettime clock_settime dirname_r getentropy mkostemp mkostemps'.split(' '))
 
         self.gcc_flags.extend([
             '-D_XOPEN_SOURCE',
@@ -127,6 +147,8 @@ class DarwinProfile (UnixProfile):
         package.local_configure_flags.extend(
             ['--cache-file=%s' % configure_cache])
 
+        package.local_configure_flags.extend(self.configure_flags)
+
         if package.name in self.debug_info:
             package.local_gcc_flags.extend(['-g'])
 

external/bockbuild/bockbuild/environment.py

@@ -37,7 +37,7 @@ class Environment:
         expand_macros(self, self._profile)
 
     def write_source_script(self, filename):
-
+        trace (filename)
         envscript = '#!/bin/sh\n'
 
         for k in self.get_names():
@@ -45,6 +45,7 @@ class Environment:
 
         with open(filename, 'w') as f:
             f.write(envscript)
+            trace(envscript)
 
         os.chmod(filename, 0o755)
 

external/bockbuild/bockbuild/package.py

@@ -95,6 +95,7 @@ class Package:
                 self.__dict__[k] = v
 
         self.makeinstall = self.makeinstall or 'make install DESTDIR=%{stage_root}'
+        self.fetched = False
 
     def extract_organization(self, source):
         if (not "git" in source) or ("http" in source):
@@ -140,6 +141,9 @@ class Package:
 
     @retry
     def fetch(self, dest):
+        if self.fetched and os.path.lexists(dest):
+            return
+
         scratch = self.profile.bockbuild.scratch
         resources = self.profile.bockbuild.resources
         source_cache_dir = self.profile.bockbuild.source_cache
@@ -148,11 +152,16 @@ class Package:
         scratch_workspace = os.path.join(scratch, '%s.workspace' % self.name)
 
         self.rm_if_exists(scratch_workspace)
-        if os.path.exists(dest):
-            shutil.move(dest, scratch_workspace)
+        if os.path.lexists(dest):
+            if os.path.islink(dest):
+                delete(dest)
+            elif os.path.isdir(dest):
+                shutil.move(dest, scratch_workspace)
+            else:
+                error ('Unexpected workspace found at %s' % dest)
+
 
         def checkout(self, source_url, cache_dir, workspace_dir):
-            self.is_local = os.path.isdir (source_url)
             def clean_git_workspace(dir):
                 trace('Cleaning git workspace: ' + self.name)
                 self.git('reset --hard', dir, hazard = True)
@@ -171,7 +180,7 @@ class Package:
                     self.rm(workspace_dir)
                 progress('Cloning git repo: %s' % source_url)
                 self.git('clone --mirror %s %s' %
-                         (source_url, cache_dir), scratch)
+                         (source_url, cache_dir), self.profile.bockbuild.root)
 
             def update_cache():
                 trace('Updating cache: ' + cache_dir)
@@ -181,7 +190,7 @@ class Package:
                     self.git('fetch origin %s' % self.git_branch, cache_dir)
 
             def create_workspace():
-                self.git('clone --local --shared %s %s' %
+                self.git('clone --local --shared --recursive %s %s' %
                          (cache_dir, workspace_dir), cache_dir)
 
             def update_workspace():
@@ -221,6 +230,7 @@ class Package:
                 if target_revision and (current_revision != target_revision):
                     self.git('reset --hard %s' %
                              target_revision, workspace_dir, hazard = True)
+
                 self.git('submodule update --recursive', workspace_dir)
 
                 current_revision = git_get_revision(self, workspace_dir)
@@ -244,25 +254,28 @@ class Package:
                 self.buildstring = ['%s <%s>' % (str, source_url)]
 
             if self.is_local:
-                link_dir (workspace_dir, source_url)
-                if git_is_dirty (self, workspace_dir):
+                self.rm_if_exists(workspace_dir)
+                work_committed = False
+                if git_is_dirty (self, source_url):
                     if self.profile.bockbuild.cmd_options.release_build:
                         error ('Release builds cannot have uncommitted local changes!')
                     else:
                         info ('The repository is dirty, your changes will be committed.')
-                        bockbuild_commit_msg = 'Bockbuild'
-                        top_commit_msg = git_get_commit_msg (self, workspace_dir)
+                        bockbuild_commit_msg = '"WIP (auto-committed by bockbuild)"'
+                        top_commit_msg = git_get_commit_msg (self, source_url)
                         if top_commit_msg == bockbuild_commit_msg:
-                            self.git ('commit -a --allow-empty --amend -m %s' % bockbuild_commit_msg, workspace_dir)
+                            self.git ('commit -a --allow-empty --amend -m', source_url, options = [bockbuild_commit_msg])
                         else:
-                            self.git('commit -a --allow-empty -m %s' % bockbuild_commit_msg, workspace_dir)
-
+                            self.git('commit -a --allow-empty -m', source_url, options = [bockbuild_commit_msg])
+                        work_committed = True
+                self.shadow_copy (source_url, workspace_dir)
+                if work_committed:
+                    self.git ('reset HEAD~1', source_url)
             else:
                 if os.path.exists(cache_dir):
                     update_cache()
                 else:
                     create_cache()
-
                 if os.path.exists(workspace_dir):
                     if self.dont_clean == True:  # previous workspace was left dirty, delete
                         clean_git_workspace(workspace_dir)
@@ -289,13 +302,18 @@ class Package:
                 pass
 
             def create_workspace(dir):
-                self.extract_archive(cache_dest, scratch, validate_only=False)
-                expected_path = os.path.join(scratch, self.source_dir_name)
-                if not os.path.exists(expected_path):
-                    error('Archive %s was extracted but not found at workspace path %s' % (
-                        cache_dest, expected_path))
-                if expected_path != dir:
-                    shutil.move(expected_path, dir)
+                filetype = get_filetype(cache_dest).lower()
+                if filetype.startswith(('gzip', 'xz', 'zip', 'bzip2')):
+                    self.extract_archive(cache_dest, scratch, validate_only=False)
+                    expected_path = os.path.join(scratch, self.source_dir_name)
+                    if not os.path.exists(expected_path):
+                        error('Archive %s was extracted but not found at workspace path %s' % (
+                            cache_dest, expected_path))
+                    if expected_path != dir:
+                        shutil.move(expected_path, dir)
+                else: # create the directory and just place the downloaded file inside
+                    ensure_dir(scratch_workspace)
+                    shutil.copy(cache_dest, scratch_workspace)
 
             def update_workspace():
                 pass
@@ -375,7 +393,11 @@ class Package:
                     resolved_source = scratch_workspace
 
                 elif source.startswith(('git://', 'file://', 'ssh://')) or source.endswith('.git') or (os.path.isdir(source) and git_isrootdir (self, source)):
-                    cache = get_git_cache_path()
+                    if os.path.isdir(source):
+                        self.is_local = True
+                        cache = None
+                    else:
+                        cache = get_git_cache_path()
                     clean_func = checkout(
                         self, source, cache, scratch_workspace)
                     resolved_source = scratch_workspace
@@ -430,6 +452,10 @@ class Package:
         self.workspace = dest
         shutil.move(scratch_workspace, self.workspace)
 
+        if not os.path.exists(self.workspace):
+            error ('Workspace was not created')
+        self.fetched = True
+
     def request_build(self, reason):
         self.needs_build = reason
 
@@ -463,7 +489,7 @@ class Package:
                 self.rm_if_exists(workspace_x64)
 
                 shutil.move(workspace, workspace_x86)
-                shutil.copytree(workspace_x86, workspace_x64)
+                self.shadow_copy(workspace_x86, workspace_x64)
 
                 self.link(workspace_x86, workspace)
                 package_stage = self.do_build(
@@ -491,10 +517,6 @@ class Package:
             self.make_artifact(package_stage, build_artifact)
         for target in self.deploy_requests:
             self.deploy_package(build_artifact, target)
-        if self.is_local:
-            verbose ('Cleaning local repo')
-            self.git ('reset --hard', workspace)
-
 
     def deploy_package(self, artifact, dest):
         trace('Deploying (%s -> %s)' %
@@ -582,7 +604,7 @@ class Package:
         except (Exception, KeyboardInterrupt) as e:
             self.rm_if_exists(self.stage_root)
             if isinstance(e, CommandException):
-                if os.path.exists(self.workspace) and not self.is_local:
+                if os.path.exists(self.workspace):
                     for path in self.aux_files:
                         self.rm_if_exists(path)
                     problem_dir = os.path.join(
@@ -619,6 +641,9 @@ class Package:
         if not isinstance(command, str):
             error('command arg must be a string: %s' % repr(command))
 
+        if not os.path.isdir(cwd):
+            error('Directory does not exist: %s' % cwd)
+
         try:
             env_command = '%s %s' % (
                 self.build_env, expand_macros(command, self))
@@ -775,6 +800,38 @@ class Package:
                     else:
                         warn("lipo: 32-bit version of file %s not found" % file)
 
+    #creates a deep hardlink copy of a directory
+    def shadow_copy (self, source, dest, exclude_git = False):
+        trace ('shadow_copy %s %s' % (source , dest))
+        if os.path.exists(dest):
+            error ('Destination directory must not exist')
+
+        # Bockbuild state may be under the directory if we are copying a local workspace. Avoid recursive copying
+        stateroot_parent = os.path.dirname (config.state_root)
+        stateroot_name = os.path.basename (config.state_root)
+        stateroot_found = False
+
+        if not os.path.commonprefix  ([source, config.state_root]) == source:
+            stateroot_found = True
+
+        for root, subdirs, filelist in os.walk (source):
+            relpath = os.path.relpath(root, source) # e.g. 'lib/mystuff'
+            destpath = os.path.join(dest, relpath)
+            os.makedirs(destpath)
+            if exclude_git:
+                subdirs[:] = [dir for dir in subdirs if dir != '.git']
+            if not stateroot_found and root == stateroot_parent:
+                subdirs [:] = [dir for dir in subdirs if dir != stateroot_name]
+                stateroot_found = True
+            for file in filelist:
+                fullpath = os.path.join (root, file)
+                if os.path.islink(fullpath):
+                    target = os.path.join(os.path.dirname(fullpath), os.readlink(fullpath))
+                    if not os.path.exists(fullpath) or os.path.commonprefix  ([config.state_root, target]) == config.state_root:
+                        break
+                os.link (fullpath, os.path.join (destpath, file))
+        trace ('shadow_copy done')
+
     def copy_side_by_side(self, src_dir, dest_dir, bin_subdir, suffix, orig_suffix=None):
         def add_suffix(filename, sfx):
             fileparts = filename.split('.', 1)

external/bockbuild/bockbuild/unixprofile.py

@@ -1,5 +1,6 @@
 from profile import Profile
 from bockbuild.environment import Environment
+from bockbuild.util.util import *
 
 class UnixProfile (Profile):
 
@@ -13,7 +14,9 @@ class UnixProfile (Profile):
 
         self.gcc_flags = ['-I%s/include' % self.staged_prefix]
         self.ld_flags = ['-L%s/lib' % self.staged_prefix]
+        self.configure_flags = []
 
+        self.env.set('bockbuild version', git_shortid(bockbuild, bockbuild.root))
         self.env.set('BUILD_PREFIX', '%{prefix}')
 
         self.env.set('PATH', ':',

external/bockbuild/bockbuild/util/util.py

@@ -27,6 +27,10 @@ class bcolors:
     BOLD = '\033[1m'
     UNDERLINE = '\033[4m'
 
+class exit_codes:
+    NOTSET = -1
+    SUCCESS = 0
+    FAILURE = 1
 
 class config:
     trace = False
@@ -38,7 +42,8 @@ class config:
     verbose = False
     protected_git_repos = [] # we do not allow modifying behavior on our profile repo or bockbuild repo.
     absolute_root = None # there is no file resolution beneath this path. Displayed paths are shortened by omitting this segment.
-
+    state_root = None
+    exit_code = exit_codes.NOTSET
 
 class CommandException (Exception):  # shell command failure
 
@@ -116,8 +121,8 @@ def loginit(message):
         Logger.monkeywrench = True
     elif sys.stdout.isatty():
         Logger.print_color = True
-        logprint('** %s **' % message, bcolors.BOLD)
-        print
+    logprint('** %s **' % message, bcolors.BOLD)
+    print
 
 
 def colorprint(message, color):
@@ -183,6 +188,10 @@ def warn(message):
         message = '%s %s' % ('(bockbuild warning)', message)
     logprint(message, bcolors.FAIL, header=get_caller())
 
+def finish (exit_code):
+    if exit_code > config.exit_code:
+        config.exit_code = exit_code
+    sys.exit(config.exit_code)
 
 def error(message, more_output=False):
     config.trace = False
@@ -190,12 +199,7 @@ def error(message, more_output=False):
         message = '%s %s' % ('(bockbuild error)', message)
     logprint(message, bcolors.FAIL, header=get_caller(), summary=True)
     if not more_output:
-        sys.exit(255)
-
-def finish():
-    logprint('\n** %s **\n' % 'Goodbye!', bcolors.BOLD)
-    sys.exit(0)
-
+        finish(exit_codes.FAILURE)
 
 def trace(message, skip=0):
     if config.trace == False:
@@ -206,7 +210,7 @@ def trace(message, skip=0):
     if config.filter is not None and config.filter not in caller:
         return
 
-    logprint(message, bcolors.FAIL, summary=True, header=caller, trace=True)
+    logprint(message, bcolors.FAIL, summary=False, header=caller, trace=True)
 
 
 def test(func):
@@ -325,6 +329,15 @@ def which(program):
 
     return None
 
+def parse_rootdir(result, cwd):
+    # http://stackoverflow.com/a/18339166
+    if os.path.basename(result) == '.git': # normal repo
+        return os.path.dirname(result)
+    elif result == '.':
+        return cwd
+    else:
+        return result
+
 
 def find_git(self, echo=False):
     git_bin = which('git')
@@ -332,14 +345,45 @@ def find_git(self, echo=False):
         error('git not found in PATH')
 
     @retry
-    def git_func(self, args, cwd, hazard = False):
+    def git_operation(self, args, cwd, hazard = False, allow_fail = False, singleline_output = False, options = None, allow_nonrootdir = False):
+        try:
+            cwd = os.path.realpath(cwd)
+            (exit, out, err) = run(git_bin, ['rev-parse', '--show-toplevel'], cwd)
+            if len(out) > 0:
+                root = out
+            else:
+                (exit, out, err) = run(git_bin, ['rev-parse', '--git-dir'], cwd)
+                root = parse_rootdir(out, cwd)
+        except:
+            raise
+        if root != cwd and not allow_nonrootdir:
+            error ('Git operations allowed only on the root directory of the repo (root: %s cwd: %s)' % (root, cwd))
         if hazard:
             root = git_rootdir (self, cwd)
             assert_modifiable_repo (root)
-        (exit, out, err) = run(git_bin, args.split(' '), cwd)
-        return out.split('\n')
+        try:
+            fullargs = args.split(' ')
+            if options:
+                if not isinstance(options, list):
+                    error ('options argument must be a list')
+                fullargs = fullargs + options
+            (exit, out, err) = run(git_bin, fullargs, cwd)
+        except CommandException:
+            if allow_fail:
+                return None
+            else:
+                raise
 
-    self.git = git_func.__get__(self, self.__class__)
+        lines = out.split('\n')
+        if singleline_output:
+            if len(lines) > 1:
+                error ('Single line output expected from git. Received the following:\n%s' % out)
+            else:
+                return lines[0]
+
+        return lines
+
+    self.git = git_operation.__get__(self, self.__class__)
     self.git_bin = git_bin
 
 
@@ -360,14 +404,7 @@ def git_get_revision(self, cwd):
 
 
 def git_get_branch(self, cwd):
-    revision = git_get_revision(self, cwd)
-    try:
-        output = self.git('symbolic-ref -q --short HEAD', cwd)
-    except:
-        return None  # detached HEAD
-    else:
-        return output[0]
-
+    return self.git('symbolic-ref -q --short HEAD', cwd, allow_fail = True, singleline_output = True)
 
 def git_is_dirty(self, cwd):
     return 'dirty' in git_shortid (self, cwd)
@@ -383,17 +420,24 @@ def git_shortid(self, cwd):
     if branch is None:
         return short_rev
     else:
-        return '%s-%s' % (branch, short_rev)
+        return '%s@%s' % (branch, short_rev)
+
+def git_rootdir(self, cwd):
+    # http://stackoverflow.com/a/18339166
+    result = self.git('rev-parse --show-toplevel', cwd, allow_nonrootdir=True, singleline_output=True)
+    if len(result) > 0:
+        return result
+    else:
+        result = self.git('rev-parse --git-dir', cwd, allow_nonrootdir=True, singleline_output=True)
+        return parse_rootdir(result)
 
 def git_isrootdir(self, cwd):
     try:
-        root = self.git('rev-parse --show-toplevel', cwd)[0]
-        return root == cwd
+        return git_rootdir (self, cwd) == cwd
     except:
-        return False
+        error('git_isrootdir')
+
 
-def git_rootdir(self, cwd):
-    return self.git('rev-parse --show-toplevel', cwd)[0]
 
 def git_get_commit_msg(self, cwd):
     return self.git('show -s --format=%B HEAD', cwd)[0]
@@ -637,9 +681,9 @@ def run(cmd, args, cwd, env=None):
 
     if not exit_code == 0:
         raise CommandException('"%s" failed, error code %s\nstderr:\n%s' % (
-            cmd, exit_code, stderr), cwd=cwd)
+            cmd + str(args), exit_code, stderr), cwd=cwd)
 
-    return (exit_code, stdout, stderr)
+    return (exit_code, stdout[:-1], stderr)
 
 
 def run_shell(cmd, print_cmd=False, cwd=None):