vendor/llvm/llvm-trunk-r321017

10 files changed, 418 insertions, 44 deletions

diff --git a/utils/docker/build_docker_image.sh b/utils/docker/build_docker_image.sh
index 33f690ad5c436..ad7831925f9a4 100755
--- a/utils/docker/build_docker_image.sh
+++ b/utils/docker/build_docker_image.sh
@@ -15,7 +15,7 @@ DOCKER_TAG=""
 BUILDSCRIPT_ARGS=""
 
 function show_usage() {
-  usage=$(cat << EOF
+  cat << EOF
 Usage: build_docker_image.sh [options] [-- [cmake_args]...]
 
 Available options:
@@ -38,6 +38,9 @@ Available options:
                         Can be specified multiple times.
     -i|--install-target name of a cmake install target to build and include in
                         the resulting archive. Can be specified multiple times.
+    -c|--checksums      name of a file, containing checksums of llvm checkout.
+                        Script will fail if checksums of the checkout do not
+                        match.
 
 Required options: --source and --docker-repository, at least one
   --install-target.
@@ -64,10 +67,9 @@ $ ./build_docker_image.sh -s debian8 -d mydocker/clang-debian8 -t "latest" \
     -DCLANG_ENABLE_BOOTSTRAP=ON \ 
     -DCLANG_BOOTSTRAP_TARGETS="install-clang;install-clang-headers"
 EOF
-)
-  echo "$usage"
 }
 
+CHECKSUMS_FILE=""
 SEEN_INSTALL_TARGET=0
 while [[ $# -gt 0 ]]; do
   case "$1" in
@@ -97,6 +99,11 @@ while [[ $# -gt 0 ]]; do
       BUILDSCRIPT_ARGS="$BUILDSCRIPT_ARGS $1 $2"
       shift 2
       ;;
+    -c|--checksums)
+      shift
+      CHECKSUMS_FILE="$1"
+      shift
+      ;;
     --)
       shift
       BUILDSCRIPT_ARGS="$BUILDSCRIPT_ARGS -- $*"
@@ -130,30 +137,41 @@ if [ $SEEN_INSTALL_TARGET -eq 0 ]; then
   exit 1
 fi
 
-cd $(dirname $0)
-if [ ! -d $IMAGE_SOURCE ]; then
-  echo "No sources for '$IMAGE_SOURCE' were found in $PWD"
+SOURCE_DIR=$(dirname $0)
+if [ ! -d "$SOURCE_DIR/$IMAGE_SOURCE" ]; then
+  echo "No sources for '$IMAGE_SOURCE' were found in $SOURCE_DIR"
   exit 1
 fi
 
-echo "Building from $IMAGE_SOURCE"
+BUILD_DIR=$(mktemp -d)
+trap "rm -rf $BUILD_DIR" EXIT
+echo "Using a temporary directory for the build: $BUILD_DIR"
+
+cp -r "$SOURCE_DIR/$IMAGE_SOURCE" "$BUILD_DIR/$IMAGE_SOURCE"
+cp -r "$SOURCE_DIR/scripts" "$BUILD_DIR/scripts"
+
+mkdir "$BUILD_DIR/checksums"
+if [ "$CHECKSUMS_FILE" != "" ]; then
+  cp "$CHECKSUMS_FILE" "$BUILD_DIR/checksums/checksums.txt"
+fi
 
 if [ "$DOCKER_TAG" != "" ]; then
   DOCKER_TAG=":$DOCKER_TAG"
 fi
 
+echo "Building from $IMAGE_SOURCE"
 echo "Building $DOCKER_REPOSITORY-build$DOCKER_TAG"
 docker build -t "$DOCKER_REPOSITORY-build$DOCKER_TAG" \
   --build-arg "buildscript_args=$BUILDSCRIPT_ARGS" \
-  -f "$IMAGE_SOURCE/build/Dockerfile" .
+  -f "$BUILD_DIR/$IMAGE_SOURCE/build/Dockerfile" \
+  "$BUILD_DIR"
 
 echo "Copying clang installation to release image sources"
-docker run -v "$PWD/$IMAGE_SOURCE:/workspace" "$DOCKER_REPOSITORY-build$DOCKER_TAG" \
+docker run -v "$BUILD_DIR/$IMAGE_SOURCE:/workspace" "$DOCKER_REPOSITORY-build$DOCKER_TAG" \
   cp /tmp/clang.tar.gz /workspace/release
-trap "rm -f $PWD/$IMAGE_SOURCE/release/clang.tar.gz" EXIT
 
 echo "Building release image"
 docker build -t "${DOCKER_REPOSITORY}${DOCKER_TAG}" \
-  "$IMAGE_SOURCE/release"
+  "$BUILD_DIR/$IMAGE_SOURCE/release"
 
 echo "Done"
diff --git a/utils/docker/debian8/build/Dockerfile b/utils/docker/debian8/build/Dockerfile
index 13a11a73be6c8..5c5ed67449637 100644
--- a/utils/docker/debian8/build/Dockerfile
+++ b/utils/docker/debian8/build/Dockerfile
@@ -18,18 +18,30 @@ RUN grep deb /etc/apt/sources.list | \
 
 # Install compiler, python and subversion.
 RUN apt-get update && \
-    apt-get install -y --no-install-recommends build-essential python2.7 wget \
-            subversion ninja-build && \
+    apt-get install -y --no-install-recommends ca-certificates gnupg \
+           build-essential python wget subversion ninja-build && \
     rm -rf /var/lib/apt/lists/*
 
-# Install cmake version that can compile clang into /usr/local.
+# Import public key required for verifying signature of cmake download.
+RUN gpg --keyserver hkp://pgp.mit.edu --recv 0x2D2CEF1034921684
+
+# Download, verify and install cmake version that can compile clang into /usr/local.
 # (Version in debian8 repos is is too old)
-RUN wget -O - "https://cmake.org/files/v3.7/cmake-3.7.2-Linux-x86_64.tar.gz" | \
-    tar xzf - -C /usr/local --strip-components=1
+RUN mkdir /tmp/cmake-install && cd /tmp/cmake-install && \
+    wget "https://cmake.org/files/v3.7/cmake-3.7.2-SHA-256.txt.asc" && \
+    wget "https://cmake.org/files/v3.7/cmake-3.7.2-SHA-256.txt" && \
+    gpg --verify cmake-3.7.2-SHA-256.txt.asc cmake-3.7.2-SHA-256.txt && \
+    wget "https://cmake.org/files/v3.7/cmake-3.7.2-Linux-x86_64.tar.gz" && \
+    ( grep "cmake-3.7.2-Linux-x86_64.tar.gz" cmake-3.7.2-SHA-256.txt | \
+      sha256sum -c - ) && \
+    tar xzf cmake-3.7.2-Linux-x86_64.tar.gz -C /usr/local --strip-components=1 && \
+    cd / && rm -rf /tmp/cmake-install
+
+ADD checksums /tmp/checksums
+ADD scripts /tmp/scripts
 
 # Arguments passed to build_install_clang.sh.
 ARG buildscript_args
 
 # Run the build. Results of the build will be available as /tmp/clang.tar.gz.
-ADD scripts/build_install_llvm.sh /tmp
-RUN /tmp/build_install_llvm.sh ${buildscript_args}
+RUN /tmp/scripts/build_install_llvm.sh ${buildscript_args}
diff --git a/utils/docker/debian8/release/Dockerfile b/utils/docker/debian8/release/Dockerfile
index d0214b9c67af7..3a44a7d411667 100644
--- a/utils/docker/debian8/release/Dockerfile
+++ b/utils/docker/debian8/release/Dockerfile
@@ -12,10 +12,10 @@ FROM launcher.gcr.io/google/debian8:latest
 
 LABEL maintainer "LLVM Developers"
 
-# Install packages for minimal usefull image.
+# Install packages for minimal useful image.
 RUN apt-get update && \
     apt-get install -y --no-install-recommends libstdc++-4.9-dev binutils && \
     rm -rf /var/lib/apt/lists/*
 
 # Unpack clang installation into this image.
-ADD clang.tar.gz /
+ADD clang.tar.gz /usr/local/
diff --git a/utils/docker/example/build/Dockerfile b/utils/docker/example/build/Dockerfile
index 597ccfeb4f23d..be077f59f48a4 100644
--- a/utils/docker/example/build/Dockerfile
+++ b/utils/docker/example/build/Dockerfile
@@ -18,9 +18,11 @@ LABEL maintainer "Maintainer <maintainer@email>"
 # FIXME: Install llvm/clang build dependencies. Including compiler to
 # build stage1, cmake, subversion, ninja, etc.
 
-# Arguments to pass to build_install_clang.sh.
+ADD checksums /tmp/checksums
+ADD scripts /tmp/scripts
+
+# Arguments passed to build_install_clang.sh.
 ARG buildscript_args
 
 # Run the build. Results of the build will be available as /tmp/clang.tar.gz.
-ADD scripts/build_install_llvm.sh /tmp
-RUN /tmp/build_install_llvm.sh ${buildscript_args}
+RUN /tmp/scripts/build_install_llvm.sh ${buildscript_args}
diff --git a/utils/docker/example/release/Dockerfile b/utils/docker/example/release/Dockerfile
index 953d81fc99517..b088ad885ac5a 100644
--- a/utils/docker/example/release/Dockerfile
+++ b/utils/docker/example/release/Dockerfile
@@ -17,8 +17,8 @@ FROM ubuntu
 LABEL maintainer "Maintainer <maintainer@email>"
 
 # FIXME: Install all packages you want to have in your release container.
-# A minimal usefull installation must include libstdc++ and binutils.
+# A minimal useful installation must include libstdc++ and binutils.
 
 # Unpack clang installation into this container.
 # It is copied to this directory by build_docker_image.sh script.
-ADD clang.tar.gz /
+ADD clang.tar.gz /usr/local/
diff --git a/utils/docker/nvidia-cuda/build/Dockerfile b/utils/docker/nvidia-cuda/build/Dockerfile
index 619b80cbb61ae..cd353a2578bde 100644
--- a/utils/docker/nvidia-cuda/build/Dockerfile
+++ b/utils/docker/nvidia-cuda/build/Dockerfile
@@ -17,9 +17,15 @@ ARG buildscript_args
 
 # Install llvm build dependencies.
 RUN apt-get update && \
-    apt-get install -y --no-install-recommends cmake python2.7 subversion ninja-build && \
+    apt-get install -y --no-install-recommends ca-certificates cmake python \
+		    subversion ninja-build && \
     rm -rf /var/lib/apt/lists/*
 
+ADD checksums /tmp/checksums
+ADD scripts /tmp/scripts
+
+# Arguments passed to build_install_clang.sh.
+ARG buildscript_args
+
 # Run the build. Results of the build will be available as /tmp/clang.tar.gz.
-ADD scripts/build_install_llvm.sh /tmp
-RUN /tmp/build_install_llvm.sh ${buildscript_args}
+RUN /tmp/scripts/build_install_llvm.sh ${buildscript_args}
diff --git a/utils/docker/nvidia-cuda/release/Dockerfile b/utils/docker/nvidia-cuda/release/Dockerfile
index b9bcae1597805..a30d7d7e91ee1 100644
--- a/utils/docker/nvidia-cuda/release/Dockerfile
+++ b/utils/docker/nvidia-cuda/release/Dockerfile
@@ -18,6 +18,6 @@ FROM nvidia/cuda:8.0-devel
 LABEL maintainer "LLVM Developers"
 
 # Unpack clang installation into this container.
-ADD clang.tar.gz /
+ADD clang.tar.gz /usr/local/
 
 # C++ standard library and binutils are already included in the base package.
diff --git a/utils/docker/scripts/build_install_llvm.sh b/utils/docker/scripts/build_install_llvm.sh
index aef4e0cbca2cd..79ce7e50efdc2 100755
--- a/utils/docker/scripts/build_install_llvm.sh
+++ b/utils/docker/scripts/build_install_llvm.sh
@@ -11,7 +11,7 @@
 set -e
 
 function show_usage() {
-  usage=$(cat << EOF
+  cat << EOF
 Usage: build_install_llvm.sh [options] -- [cmake-args]
 
 Checkout svn sources and run cmake with the specified arguments. Used
@@ -37,8 +37,6 @@ Required options: At least one --install-target.
 
 All options after '--' are passed to CMake invocation.
 EOF
-)
-  echo "$usage"
 }
 
 LLVM_SVN_REV=""
@@ -48,6 +46,7 @@ CMAKE_INSTALL_TARGETS=""
 # We always checkout llvm
 LLVM_PROJECTS="llvm"
 CMAKE_LLVM_ENABLE_PROJECTS=""
+CLANG_TOOLS_EXTRA_ENABLED=0
 
 function contains_project() {
   local TARGET_PROJ="$1"
@@ -60,6 +59,17 @@ function contains_project() {
   return 1
 }
 
+function append_project() {
+  local PROJ="$1"
+
+  LLVM_PROJECTS="$LLVM_PROJECTS $PROJ"
+  if [ "$CMAKE_LLVM_ENABLE_PROJECTS" != "" ]; then
+    CMAKE_LLVM_ENABLE_PROJECTS="$CMAKE_LLVM_ENABLE_PROJECTS;$PROJ"
+  else
+    CMAKE_LLVM_ENABLE_PROJECTS="$PROJ"
+  fi
+}
+
 while [[ $# -gt 0 ]]; do
   case "$1" in
     -r|--revision)
@@ -75,19 +85,27 @@ while [[ $# -gt 0 ]]; do
     -p|--llvm-project)
       shift
       PROJ="$1"
+      shift
+
       if [ "$PROJ" == "cfe" ]; then
         PROJ="clang"
       fi
-      if ! contains_project "$PROJ" ; then
-        LLVM_PROJECTS="$LLVM_PROJECTS $PROJ"
-        if [ "$CMAKE_LLVM_ENABLE_PROJECTS" != "" ]; then
-          CMAKE_LLVM_ENABLE_PROJECTS="$CMAKE_LLVM_ENABLE_PROJECTS;"
+
+      if [ "$PROJ" == "clang-tools-extra" ]; then
+        if [ $CLANG_TOOLS_EXTRA_ENABLED -ne 0 ]; then
+          echo "Project 'clang-tools-extra' is already enabled, ignoring extra occurences."
+        else
+          CLANG_TOOLS_EXTRA_ENABLED=1
         fi
-        CMAKE_LLVM_ENABLE_PROJECTS="$CMAKE_LLVM_ENABLED_PROJECTS$PROJ"
+
+        continue
+      fi
+
+      if ! contains_project "$PROJ" ; then
+        append_project "$PROJ"
       else
         echo "Project '$PROJ' is already enabled, ignoring extra occurences."
       fi
-      shift
       ;;
     -i|--install-target)
       shift
@@ -114,14 +132,25 @@ if [ "$CMAKE_INSTALL_TARGETS" == "" ]; then
   exit 1
 fi
 
+if [ $CLANG_TOOLS_EXTRA_ENABLED -ne 0 ]; then
+  if ! contains_project "clang"; then
+    echo "Project 'clang-tools-extra' was enabled without 'clang'."
+    echo "Adding 'clang' to a list of projects."
+
+    append_project "clang"
+  fi
+fi
+
 if [ "$LLVM_BRANCH" == "" ]; then
   LLVM_BRANCH="trunk"
 fi
 
-if [ "$LLVM_SVN_REVISION" != "" ]; then
-  SVN_REV_ARG="-r$LLVM_SVN_REVISION"
+if [ "$LLVM_SVN_REV" != "" ]; then
+  SVN_REV_ARG="-r$LLVM_SVN_REV"
+  echo "Checking out svn revision r$LLVM_SVN_REV."
 else
   SVN_REV_ARG=""
+  echo "Checking out latest svn revision."
 fi
 
 CLANG_BUILD_DIR=/tmp/clang-build
@@ -140,14 +169,28 @@ for LLVM_PROJECT in $LLVM_PROJECTS; do
   fi
 
   echo "Checking out https://llvm.org/svn/llvm-project/$SVN_PROJECT to $CLANG_BUILD_DIR/src/$LLVM_PROJECT"
-  # FIXME: --trust-server-cert is required to workaround 'SSL issuer is not
-  #        trusted' error. Using https seems preferable to http either way,
-  #        albeit this is not secure.
-  svn co -q $SVN_REV_ARG --trust-server-cert \
+  svn co -q $SVN_REV_ARG \
     "https://llvm.org/svn/llvm-project/$SVN_PROJECT/$LLVM_BRANCH" \
     "$CLANG_BUILD_DIR/src/$LLVM_PROJECT"
 done
 
+if [ $CLANG_TOOLS_EXTRA_ENABLED -ne 0 ]; then
+  echo "Checking out https://llvm.org/svn/llvm-project/clang-tools-extra to $CLANG_BUILD_DIR/src/clang/tools/extra"
+  svn co -q $SVN_REV_ARG \
+    "https://llvm.org/svn/llvm-project/clang-tools-extra/$LLVM_BRANCH" \
+    "$CLANG_BUILD_DIR/src/clang/tools/extra"
+fi
+
+CHECKSUMS_FILE="/tmp/checksums/checksums.txt"
+
+if [ -f "$CHECKSUMS_FILE" ]; then
+  echo "Validating checksums for LLVM checkout..."
+  python "$(dirname $0)/llvm_checksum/llvm_checksum.py" -c "$CHECKSUMS_FILE" \
+    --partial --multi_dir "$CLANG_BUILD_DIR/src"
+else
+  echo "Skipping checksumming checks..."
+fi
+
 mkdir "$CLANG_BUILD_DIR/build"
 pushd "$CLANG_BUILD_DIR/build"
 
diff --git a/utils/docker/scripts/llvm_checksum/llvm_checksum.py b/utils/docker/scripts/llvm_checksum/llvm_checksum.py
new file mode 100755
index 0000000000000..584efa2598bfd
--- /dev/null
+++ b/utils/docker/scripts/llvm_checksum/llvm_checksum.py
@@ -0,0 +1,198 @@
+#!/usr/bin/python
+""" A small program to compute checksums of LLVM checkout.
+"""
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import hashlib
+import logging
+import re
+import sys
+from argparse import ArgumentParser
+from project_tree import *
+
+SVN_DATES_REGEX = re.compile(r"\$(Date|LastChangedDate)[^\$]+\$")
+
+
+def main():
+  parser = ArgumentParser()
+  parser.add_argument(
+      "-v", "--verbose", action="store_true", help="enable debug logging")
+  parser.add_argument(
+      "-c",
+      "--check",
+      metavar="reference_file",
+      help="read checksums from reference_file and " +
+      "check they match checksums of llvm_path.")
+  parser.add_argument(
+      "--partial",
+      action="store_true",
+      help="ignore projects from reference_file " +
+      "that are not checked out in llvm_path.")
+  parser.add_argument(
+      "--multi_dir",
+      action="store_true",
+      help="indicates llvm_path contains llvm, checked out " +
+      "into multiple directories, as opposed to a " +
+      "typical single source tree checkout.")
+  parser.add_argument("llvm_path")
+
+  args = parser.parse_args()
+  if args.check is not None:
+    with open(args.check, "r") as f:
+      reference_checksums = ReadLLVMChecksums(f)
+  else:
+    reference_checksums = None
+
+  if args.verbose:
+    logging.basicConfig(level=logging.DEBUG)
+
+  llvm_projects = CreateLLVMProjects(not args.multi_dir)
+  checksums = ComputeLLVMChecksums(args.llvm_path, llvm_projects)
+
+  if reference_checksums is None:
+    WriteLLVMChecksums(checksums, sys.stdout)
+    sys.exit(0)
+
+  if not ValidateChecksums(reference_checksums, checksums, args.partial):
+    sys.stdout.write("Checksums differ.\nNew checksums:\n")
+    WriteLLVMChecksums(checksums, sys.stdout)
+    sys.stdout.write("Reference checksums:\n")
+    WriteLLVMChecksums(reference_checksums, sys.stdout)
+    sys.exit(1)
+  else:
+    sys.stdout.write("Checksums match.")
+
+
+def ComputeLLVMChecksums(root_path, projects):
+  """Compute checksums for LLVM sources checked out using svn.
+
+  Args:
+    root_path: a directory of llvm checkout.
+    projects: a list of LLVMProject instances, which describe checkout paths,
+      relative to root_path.
+
+  Returns:
+    A dict mapping from project name to project checksum.
+  """
+  hash_algo = hashlib.sha256
+
+  def collapse_svn_substitutions(contents):
+    # Replace svn substitutions for $Date$ and $LastChangedDate$.
+    # Unfortunately, these are locale-specific.
+    return SVN_DATES_REGEX.sub("$\1$", contents)
+
+  def read_and_collapse_svn_subsitutions(file_path):
+    with open(file_path, "rb") as f:
+      contents = f.read()
+      new_contents = collapse_svn_substitutions(contents)
+      if contents != new_contents:
+        logging.debug("Replaced svn keyword substitutions in %s", file_path)
+        logging.debug("\n\tBefore\n%s\n\tAfter\n%s", contents, new_contents)
+      return new_contents
+
+  project_checksums = dict()
+  # Hash each project.
+  for proj in projects:
+    project_root = os.path.join(root_path, proj.relpath)
+    if not os.path.exists(project_root):
+      logging.info("Folder %s doesn't exist, skipping project %s", proj.relpath,
+                   proj.name)
+      continue
+
+    files = list()
+
+    def add_file_hash(file_path):
+      if os.path.islink(file_path) and not os.path.exists(file_path):
+        content = os.readlink(file_path)
+      else:
+        content = read_and_collapse_svn_subsitutions(file_path)
+      hasher = hash_algo()
+      hasher.update(content)
+      file_digest = hasher.hexdigest()
+      logging.debug("Checksum %s for file %s", file_digest, file_path)
+      files.append((file_path, file_digest))
+
+    logging.info("Computing checksum for %s", proj.name)
+    WalkProjectFiles(root_path, projects, proj, add_file_hash)
+
+    # Compute final checksum.
+    files.sort(key=lambda x: x[0])
+    hasher = hash_algo()
+    for file_path, file_digest in files:
+      file_path = os.path.relpath(file_path, project_root)
+      hasher.update(file_path)
+      hasher.update(file_digest)
+    project_checksums[proj.name] = hasher.hexdigest()
+  return project_checksums
+
+
+def WriteLLVMChecksums(checksums, f):
+  """Writes checksums to a text file.
+
+  Args:
+    checksums: a dict mapping from project name to project checksum (result of
+      ComputeLLVMChecksums).
+    f: a file object to write into.
+  """
+
+  for proj in sorted(checksums.keys()):
+    f.write("{} {}\n".format(checksums[proj], proj))
+
+
+def ReadLLVMChecksums(f):
+  """Reads checksums from a text file, produced by WriteLLVMChecksums.
+
+  Returns:
+    A dict, mapping from project name to project checksum.
+  """
+  checksums = {}
+  while True:
+    line = f.readline()
+    if line == "":
+      break
+    checksum, proj = line.split()
+    checksums[proj] = checksum
+  return checksums
+
+
+def ValidateChecksums(reference_checksums,
+                      new_checksums,
+                      allow_missing_projects=False):
+  """Validates that reference_checksums and new_checksums match.
+
+  Args:
+    reference_checksums: a dict of reference checksums, mapping from a project
+      name to a project checksum.
+    new_checksums: a dict of checksums to be checked, mapping from a project
+      name to a project checksum.
+    allow_missing_projects:
+      When True, reference_checksums may contain more projects than
+        new_checksums. Projects missing from new_checksums are ignored.
+      When False, new_checksums and reference_checksums must contain checksums
+        for the same set of projects. If there is a project in
+        reference_checksums, missing from new_checksums, ValidateChecksums
+        will return False.
+
+  Returns:
+    True, if checksums match with regards to allow_missing_projects flag value.
+    False, otherwise.
+  """
+  if not allow_missing_projects:
+    if len(new_checksums) != len(reference_checksums):
+      return False
+
+  for proj, checksum in new_checksums.iteritems():
+    # We never computed a checksum for this project.
+    if proj not in reference_checksums:
+      return False
+    # Checksum did not match.
+    if reference_checksums[proj] != checksum:
+      return False
+
+  return True
+
+
+if __name__ == "__main__":
+  main()
diff --git a/utils/docker/scripts/llvm_checksum/project_tree.py b/utils/docker/scripts/llvm_checksum/project_tree.py
new file mode 100644
index 0000000000000..31d8703ba234b
--- /dev/null
+++ b/utils/docker/scripts/llvm_checksum/project_tree.py
@@ -0,0 +1,95 @@
+"""Contains helper functions to compute checksums for LLVM checkouts.
+"""
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import logging
+import os
+import os.path
+import sys
+
+
+class LLVMProject(object):
+  """An LLVM project with a descriptive name and a relative checkout path.
+  """
+
+  def __init__(self, name, relpath):
+    self.name = name
+    self.relpath = relpath
+
+  def is_subproject(self, other_project):
+    """ Check if self is checked out as a subdirectory of other_project.
+    """
+    return self.relpath.startswith(other_project.relpath)
+
+
+def WalkProjectFiles(checkout_root, all_projects, project, visitor):
+  """ Walk over all files inside a project without recursing into subprojects, '.git' and '.svn' subfolders.
+
+    checkout_root: root of the LLVM checkout.
+    all_projects: projects in the LLVM checkout.
+    project: a project to walk the files of. Must be inside all_projects.
+    visitor: a function called on each visited file.
+  """
+  assert project in all_projects
+
+  ignored_paths = set()
+  for other_project in all_projects:
+    if other_project != project and other_project.is_subproject(project):
+      ignored_paths.add(os.path.join(checkout_root, other_project.relpath))
+
+  def raise_error(err):
+    raise err
+
+  project_root = os.path.join(checkout_root, project.relpath)
+  for root, dirs, files in os.walk(project_root, onerror=raise_error):
+    dirs[:] = [
+        d for d in dirs
+        if d != ".svn" and d != ".git" and
+        os.path.join(root, d) not in ignored_paths
+    ]
+    for f in files:
+      visitor(os.path.join(root, f))
+
+
+def CreateLLVMProjects(single_tree_checkout):
+  """Returns a list of LLVMProject instances, describing relative paths of a typical LLVM checkout.
+
+  Args:
+    single_tree_checkout:
+      When True, relative paths for each project points to a typical single
+        source tree checkout.
+      When False, relative paths for each projects points to a separate
+        directory. However, clang-tools-extra is an exception, its relative path
+        will always be 'clang/tools/extra'.
+  """
+  # FIXME: cover all of llvm projects.
+
+  # Projects that reside inside 'projects/' in a single source tree checkout.
+  ORDINARY_PROJECTS = [
+      "compiler-rt", "dragonegg", "libcxx", "libcxxabi", "libunwind",
+      "parallel-libs", "test-suite"
+  ]
+  # Projects that reside inside 'tools/' in a single source tree checkout.
+  TOOLS_PROJECTS = ["clang", "lld", "lldb", "llgo"]
+
+  if single_tree_checkout:
+    projects = [LLVMProject("llvm", "")]
+    projects += [
+        LLVMProject(p, os.path.join("projects", p)) for p in ORDINARY_PROJECTS
+    ]
+    projects += [
+        LLVMProject(p, os.path.join("tools", p)) for p in TOOLS_PROJECTS
+    ]
+    projects.append(
+        LLVMProject("clang-tools-extra",
+                    os.path.join("tools", "clang", "tools", "extra")))
+  else:
+    projects = [LLVMProject("llvm", "llvm")]
+    projects += [LLVMProject(p, p) for p in ORDINARY_PROJECTS]
+    projects += [LLVMProject(p, p) for p in TOOLS_PROJECTS]
+    projects.append(
+        LLVMProject("clang-tools-extra", os.path.join("clang", "tools",
+                                                      "extra")))
+  return projects