Imported Upstream version 6.10.0.49

Former-commit-id: 1d6753294b2993e1fbf92de9366bb9544db4189b

external/llvm-project/llvm/utils/docker/scripts/build_install_llvm.sh

@@ -0,0 +1,255 @@
+#!/usr/bin/env bash
+#===- llvm/utils/docker/scripts/build_install_llvm.sh ---------------------===//
+#
+#                     The LLVM Compiler Infrastructure
+#
+# This file is distributed under the University of Illinois Open Source
+# License. See LICENSE.TXT for details.
+#
+#===-----------------------------------------------------------------------===//
+
+set -e
+
+function show_usage() {
+  cat << EOF
+Usage: build_install_llvm.sh [options] -- [cmake-args]
+
+Checkout svn sources and run cmake with the specified arguments. Used
+inside docker container.
+Passes additional -DCMAKE_INSTALL_PREFIX and archives the contents of
+the directory to /tmp/clang.tar.gz.
+
+Available options:
+  -h|--help           show this help message
+  -b|--branch         svn branch to checkout, i.e. 'trunk',
+                      'branches/release_40'
+                      (default: 'trunk')
+  -r|--revision       svn revision to checkout
+  -c|--cherrypick     revision to cherry-pick. Can be specified multiple times.
+                      Cherry-picks are performed in the sorted order using the
+                      following command:
+                      'svn patch <(svn diff -c \$rev)'.
+  -p|--llvm-project   name of an svn project to checkout. Will also add the
+                      project to a list LLVM_ENABLE_PROJECTS, passed to CMake.
+                      For clang, please use 'clang', not 'cfe'.
+                      Project 'llvm' is always included and ignored, if
+                      specified.
+                      Can be specified multiple times.
+  -i|--install-target name of a cmake install target to build and include in
+                      the resulting archive. Can be specified multiple times.
+Required options: At least one --install-target.
+
+All options after '--' are passed to CMake invocation.
+EOF
+}
+
+LLVM_SVN_REV=""
+CHERRYPICKS=""
+LLVM_BRANCH=""
+CMAKE_ARGS=""
+CMAKE_INSTALL_TARGETS=""
+# We always checkout llvm
+LLVM_PROJECTS="llvm"
+CMAKE_LLVM_ENABLE_PROJECTS=""
+CLANG_TOOLS_EXTRA_ENABLED=0
+
+function contains_project() {
+  local TARGET_PROJ="$1"
+  local PROJ
+  for PROJ in $LLVM_PROJECTS; do
+    if [ "$PROJ" == "$TARGET_PROJ" ]; then
+      return 0
+    fi
+  done
+  return 1
+}
+
+function append_project() {
+  local PROJ="$1"
+
+  LLVM_PROJECTS="$LLVM_PROJECTS $PROJ"
+  if [ "$CMAKE_LLVM_ENABLE_PROJECTS" != "" ]; then
+    CMAKE_LLVM_ENABLE_PROJECTS="$CMAKE_LLVM_ENABLE_PROJECTS;$PROJ"
+  else
+    CMAKE_LLVM_ENABLE_PROJECTS="$PROJ"
+  fi
+}
+
+while [[ $# -gt 0 ]]; do
+  case "$1" in
+    -r|--revision)
+      shift
+      LLVM_SVN_REV="$1"
+      shift
+      ;;
+    -c|--cherrypick)
+      shift
+      CHERRYPICKS="$CHERRYPICKS $1"
+      shift
+      ;;
+    -b|--branch)
+      shift
+      LLVM_BRANCH="$1"
+      shift
+      ;;
+    -p|--llvm-project)
+      shift
+      PROJ="$1"
+      shift
+
+      if [ "$PROJ" == "cfe" ]; then
+        PROJ="clang"
+      fi
+
+      if [ "$PROJ" == "clang-tools-extra" ]; then
+        if [ $CLANG_TOOLS_EXTRA_ENABLED -ne 0 ]; then
+          echo "Project 'clang-tools-extra' is already enabled, ignoring extra occurences."
+        else
+          CLANG_TOOLS_EXTRA_ENABLED=1
+        fi
+
+        continue
+      fi
+
+      if ! contains_project "$PROJ" ; then
+        append_project "$PROJ"
+      else
+        echo "Project '$PROJ' is already enabled, ignoring extra occurences."
+      fi
+      ;;
+    -i|--install-target)
+      shift
+      CMAKE_INSTALL_TARGETS="$CMAKE_INSTALL_TARGETS $1"
+      shift
+      ;;
+    --)
+      shift
+      CMAKE_ARGS="$*"
+      shift $#
+      ;;
+    -h|--help)
+      show_usage
+      exit 0
+      ;;
+    *)
+      echo "Unknown option: $1"
+      exit 1
+  esac
+done
+
+if [ "$CMAKE_INSTALL_TARGETS" == "" ]; then
+  echo "No install targets. Please pass one or more --install-target."
+  exit 1
+fi
+
+if [ $CLANG_TOOLS_EXTRA_ENABLED -ne 0 ]; then
+  if ! contains_project "clang"; then
+    echo "Project 'clang-tools-extra' was enabled without 'clang'."
+    echo "Adding 'clang' to a list of projects."
+
+    append_project "clang"
+  fi
+fi
+
+if [ "$LLVM_BRANCH" == "" ]; then
+  LLVM_BRANCH="trunk"
+fi
+
+if [ "$LLVM_SVN_REV" != "" ]; then
+  SVN_REV_ARG="-r$LLVM_SVN_REV"
+  echo "Checking out svn revision r$LLVM_SVN_REV."
+else
+  SVN_REV_ARG=""
+  echo "Checking out latest svn revision."
+fi
+
+# Sort cherrypicks and remove duplicates.
+CHERRYPICKS="$(echo "$CHERRYPICKS" | xargs -n1 | sort | uniq | xargs)"
+
+function apply_cherrypicks() {
+  local CHECKOUT_DIR="$1"
+
+  [ "$CHERRYPICKS" == "" ] || echo "Applying cherrypicks"
+  pushd "$CHECKOUT_DIR"
+
+  # This function is always called on a sorted list of cherrypicks.
+  for CHERRY_REV in $CHERRYPICKS; do
+    echo "Cherry-picking r$CHERRY_REV into $CHECKOUT_DIR"
+
+    local PATCH_FILE="$(mktemp)"
+    svn diff -c $CHERRY_REV > "$PATCH_FILE"
+    svn patch "$PATCH_FILE"
+    rm "$PATCH_FILE"
+  done
+
+  popd
+}
+
+CLANG_BUILD_DIR=/tmp/clang-build
+CLANG_INSTALL_DIR=/tmp/clang-install
+
+mkdir "$CLANG_BUILD_DIR"
+
+# Get the sources from svn.
+echo "Checking out sources from svn"
+mkdir "$CLANG_BUILD_DIR/src"
+for LLVM_PROJECT in $LLVM_PROJECTS; do
+  if [ "$LLVM_PROJECT" == "clang" ]; then
+    SVN_PROJECT="cfe"
+  else
+    SVN_PROJECT="$LLVM_PROJECT"
+  fi
+
+  echo "Checking out https://llvm.org/svn/llvm-project/$SVN_PROJECT to $CLANG_BUILD_DIR/src/$LLVM_PROJECT"
+  svn co -q $SVN_REV_ARG \
+    "https://llvm.org/svn/llvm-project/$SVN_PROJECT/$LLVM_BRANCH" \
+    "$CLANG_BUILD_DIR/src/$LLVM_PROJECT"
+
+  # We apply cherrypicks to all repositories regardless of whether the revision
+  # changes this repository or not. For repositories not affected by the
+  # cherrypick, applying the cherrypick is a no-op.
+  apply_cherrypicks "$CLANG_BUILD_DIR/src/$LLVM_PROJECT"
+done
+
+if [ $CLANG_TOOLS_EXTRA_ENABLED -ne 0 ]; then
+  echo "Checking out https://llvm.org/svn/llvm-project/clang-tools-extra to $CLANG_BUILD_DIR/src/clang/tools/extra"
+  svn co -q $SVN_REV_ARG \
+    "https://llvm.org/svn/llvm-project/clang-tools-extra/$LLVM_BRANCH" \
+    "$CLANG_BUILD_DIR/src/clang/tools/extra"
+
+  apply_cherrypicks "$CLANG_BUILD_DIR/src/clang/tools/extra"
+fi
+
+CHECKSUMS_FILE="/tmp/checksums/checksums.txt"
+
+if [ -f "$CHECKSUMS_FILE" ]; then
+  echo "Validating checksums for LLVM checkout..."
+  python "$(dirname $0)/llvm_checksum/llvm_checksum.py" -c "$CHECKSUMS_FILE" \
+    --partial --multi_dir "$CLANG_BUILD_DIR/src"
+else
+  echo "Skipping checksumming checks..."
+fi
+
+mkdir "$CLANG_BUILD_DIR/build"
+pushd "$CLANG_BUILD_DIR/build"
+
+# Run the build as specified in the build arguments.
+echo "Running build"
+cmake -GNinja \
+  -DCMAKE_INSTALL_PREFIX="$CLANG_INSTALL_DIR" \
+  -DLLVM_ENABLE_PROJECTS="$CMAKE_LLVM_ENABLE_PROJECTS" \
+  $CMAKE_ARGS \
+  "$CLANG_BUILD_DIR/src/llvm"
+ninja $CMAKE_INSTALL_TARGETS
+
+popd
+
+# Pack the installed clang into an archive.
+echo "Archiving clang installation to /tmp/clang.tar.gz"
+cd "$CLANG_INSTALL_DIR"
+tar -czf /tmp/clang.tar.gz *
+
+# Cleanup.
+rm -rf "$CLANG_BUILD_DIR" "$CLANG_INSTALL_DIR"
+
+echo "Done"

external/llvm-project/llvm/utils/docker/scripts/llvm_checksum/llvm_checksum.py

@@ -0,0 +1,198 @@
+#!/usr/bin/python
+""" A small program to compute checksums of LLVM checkout.
+"""
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import hashlib
+import logging
+import re
+import sys
+from argparse import ArgumentParser
+from project_tree import *
+
+SVN_DATES_REGEX = re.compile(r"\$(Date|LastChangedDate)[^\$]+\$")
+
+
+def main():
+  parser = ArgumentParser()
+  parser.add_argument(
+      "-v", "--verbose", action="store_true", help="enable debug logging")
+  parser.add_argument(
+      "-c",
+      "--check",
+      metavar="reference_file",
+      help="read checksums from reference_file and " +
+      "check they match checksums of llvm_path.")
+  parser.add_argument(
+      "--partial",
+      action="store_true",
+      help="ignore projects from reference_file " +
+      "that are not checked out in llvm_path.")
+  parser.add_argument(
+      "--multi_dir",
+      action="store_true",
+      help="indicates llvm_path contains llvm, checked out " +
+      "into multiple directories, as opposed to a " +
+      "typical single source tree checkout.")
+  parser.add_argument("llvm_path")
+
+  args = parser.parse_args()
+  if args.check is not None:
+    with open(args.check, "r") as f:
+      reference_checksums = ReadLLVMChecksums(f)
+  else:
+    reference_checksums = None
+
+  if args.verbose:
+    logging.basicConfig(level=logging.DEBUG)
+
+  llvm_projects = CreateLLVMProjects(not args.multi_dir)
+  checksums = ComputeLLVMChecksums(args.llvm_path, llvm_projects)
+
+  if reference_checksums is None:
+    WriteLLVMChecksums(checksums, sys.stdout)
+    sys.exit(0)
+
+  if not ValidateChecksums(reference_checksums, checksums, args.partial):
+    sys.stdout.write("Checksums differ.\nNew checksums:\n")
+    WriteLLVMChecksums(checksums, sys.stdout)
+    sys.stdout.write("Reference checksums:\n")
+    WriteLLVMChecksums(reference_checksums, sys.stdout)
+    sys.exit(1)
+  else:
+    sys.stdout.write("Checksums match.")
+
+
+def ComputeLLVMChecksums(root_path, projects):
+  """Compute checksums for LLVM sources checked out using svn.
+
+  Args:
+    root_path: a directory of llvm checkout.
+    projects: a list of LLVMProject instances, which describe checkout paths,
+      relative to root_path.
+
+  Returns:
+    A dict mapping from project name to project checksum.
+  """
+  hash_algo = hashlib.sha256
+
+  def collapse_svn_substitutions(contents):
+    # Replace svn substitutions for $Date$ and $LastChangedDate$.
+    # Unfortunately, these are locale-specific.
+    return SVN_DATES_REGEX.sub("$\1$", contents)
+
+  def read_and_collapse_svn_subsitutions(file_path):
+    with open(file_path, "rb") as f:
+      contents = f.read()
+      new_contents = collapse_svn_substitutions(contents)
+      if contents != new_contents:
+        logging.debug("Replaced svn keyword substitutions in %s", file_path)
+        logging.debug("\n\tBefore\n%s\n\tAfter\n%s", contents, new_contents)
+      return new_contents
+
+  project_checksums = dict()
+  # Hash each project.
+  for proj in projects:
+    project_root = os.path.join(root_path, proj.relpath)
+    if not os.path.exists(project_root):
+      logging.info("Folder %s doesn't exist, skipping project %s", proj.relpath,
+                   proj.name)
+      continue
+
+    files = list()
+
+    def add_file_hash(file_path):
+      if os.path.islink(file_path) and not os.path.exists(file_path):
+        content = os.readlink(file_path)
+      else:
+        content = read_and_collapse_svn_subsitutions(file_path)
+      hasher = hash_algo()
+      hasher.update(content)
+      file_digest = hasher.hexdigest()
+      logging.debug("Checksum %s for file %s", file_digest, file_path)
+      files.append((file_path, file_digest))
+
+    logging.info("Computing checksum for %s", proj.name)
+    WalkProjectFiles(root_path, projects, proj, add_file_hash)
+
+    # Compute final checksum.
+    files.sort(key=lambda x: x[0])
+    hasher = hash_algo()
+    for file_path, file_digest in files:
+      file_path = os.path.relpath(file_path, project_root)
+      hasher.update(file_path)
+      hasher.update(file_digest)
+    project_checksums[proj.name] = hasher.hexdigest()
+  return project_checksums
+
+
+def WriteLLVMChecksums(checksums, f):
+  """Writes checksums to a text file.
+
+  Args:
+    checksums: a dict mapping from project name to project checksum (result of
+      ComputeLLVMChecksums).
+    f: a file object to write into.
+  """
+
+  for proj in sorted(checksums.keys()):
+    f.write("{} {}\n".format(checksums[proj], proj))
+
+
+def ReadLLVMChecksums(f):
+  """Reads checksums from a text file, produced by WriteLLVMChecksums.
+
+  Returns:
+    A dict, mapping from project name to project checksum.
+  """
+  checksums = {}
+  while True:
+    line = f.readline()
+    if line == "":
+      break
+    checksum, proj = line.split()
+    checksums[proj] = checksum
+  return checksums
+
+
+def ValidateChecksums(reference_checksums,
+                      new_checksums,
+                      allow_missing_projects=False):
+  """Validates that reference_checksums and new_checksums match.
+
+  Args:
+    reference_checksums: a dict of reference checksums, mapping from a project
+      name to a project checksum.
+    new_checksums: a dict of checksums to be checked, mapping from a project
+      name to a project checksum.
+    allow_missing_projects:
+      When True, reference_checksums may contain more projects than
+        new_checksums. Projects missing from new_checksums are ignored.
+      When False, new_checksums and reference_checksums must contain checksums
+        for the same set of projects. If there is a project in
+        reference_checksums, missing from new_checksums, ValidateChecksums
+        will return False.
+
+  Returns:
+    True, if checksums match with regards to allow_missing_projects flag value.
+    False, otherwise.
+  """
+  if not allow_missing_projects:
+    if len(new_checksums) != len(reference_checksums):
+      return False
+
+  for proj, checksum in new_checksums.iteritems():
+    # We never computed a checksum for this project.
+    if proj not in reference_checksums:
+      return False
+    # Checksum did not match.
+    if reference_checksums[proj] != checksum:
+      return False
+
+  return True
+
+
+if __name__ == "__main__":
+  main()

external/llvm-project/llvm/utils/docker/scripts/llvm_checksum/project_tree.py

@@ -0,0 +1,95 @@
+"""Contains helper functions to compute checksums for LLVM checkouts.
+"""
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import logging
+import os
+import os.path
+import sys
+
+
+class LLVMProject(object):
+  """An LLVM project with a descriptive name and a relative checkout path.
+  """
+
+  def __init__(self, name, relpath):
+    self.name = name
+    self.relpath = relpath
+
+  def is_subproject(self, other_project):
+    """ Check if self is checked out as a subdirectory of other_project.
+    """
+    return self.relpath.startswith(other_project.relpath)
+
+
+def WalkProjectFiles(checkout_root, all_projects, project, visitor):
+  """ Walk over all files inside a project without recursing into subprojects, '.git' and '.svn' subfolders.
+
+    checkout_root: root of the LLVM checkout.
+    all_projects: projects in the LLVM checkout.
+    project: a project to walk the files of. Must be inside all_projects.
+    visitor: a function called on each visited file.
+  """
+  assert project in all_projects
+
+  ignored_paths = set()
+  for other_project in all_projects:
+    if other_project != project and other_project.is_subproject(project):
+      ignored_paths.add(os.path.join(checkout_root, other_project.relpath))
+
+  def raise_error(err):
+    raise err
+
+  project_root = os.path.join(checkout_root, project.relpath)
+  for root, dirs, files in os.walk(project_root, onerror=raise_error):
+    dirs[:] = [
+        d for d in dirs
+        if d != ".svn" and d != ".git" and
+        os.path.join(root, d) not in ignored_paths
+    ]
+    for f in files:
+      visitor(os.path.join(root, f))
+
+
+def CreateLLVMProjects(single_tree_checkout):
+  """Returns a list of LLVMProject instances, describing relative paths of a typical LLVM checkout.
+
+  Args:
+    single_tree_checkout:
+      When True, relative paths for each project points to a typical single
+        source tree checkout.
+      When False, relative paths for each projects points to a separate
+        directory. However, clang-tools-extra is an exception, its relative path
+        will always be 'clang/tools/extra'.
+  """
+  # FIXME: cover all of llvm projects.
+
+  # Projects that reside inside 'projects/' in a single source tree checkout.
+  ORDINARY_PROJECTS = [
+      "compiler-rt", "dragonegg", "libcxx", "libcxxabi", "libunwind",
+      "parallel-libs", "test-suite"
+  ]
+  # Projects that reside inside 'tools/' in a single source tree checkout.
+  TOOLS_PROJECTS = ["clang", "lld", "lldb", "llgo"]
+
+  if single_tree_checkout:
+    projects = [LLVMProject("llvm", "")]
+    projects += [
+        LLVMProject(p, os.path.join("projects", p)) for p in ORDINARY_PROJECTS
+    ]
+    projects += [
+        LLVMProject(p, os.path.join("tools", p)) for p in TOOLS_PROJECTS
+    ]
+    projects.append(
+        LLVMProject("clang-tools-extra",
+                    os.path.join("tools", "clang", "tools", "extra")))
+  else:
+    projects = [LLVMProject("llvm", "llvm")]
+    projects += [LLVMProject(p, p) for p in ORDINARY_PROJECTS]
+    projects += [LLVMProject(p, p) for p in TOOLS_PROJECTS]
+    projects.append(
+        LLVMProject("clang-tools-extra", os.path.join("clang", "tools",
+                                                      "extra")))
+  return projects