diff options
Diffstat (limited to 'utils/llvmbuild')
| -rwxr-xr-x | utils/llvmbuild | 778 |
1 files changed, 0 insertions, 778 deletions
diff --git a/utils/llvmbuild b/utils/llvmbuild deleted file mode 100755 index b623d32..0000000 --- a/utils/llvmbuild +++ /dev/null @@ -1,778 +0,0 @@ -#!/usr/bin/python3 -##===- utils/llvmbuild - Build the LLVM project ----------------*-python-*-===## -# -# The LLVM Compiler Infrastructure -# -# This file is distributed under the University of Illinois Open Source -# License. See LICENSE.TXT for details. -# -##===----------------------------------------------------------------------===## -# -# This script builds many different flavors of the LLVM ecosystem. It -# will build LLVM, Clang, llvm-gcc, and dragonegg as well as run tests -# on them. This script is convenient to use to check builds and tests -# before committing changes to the upstream repository -# -# A typical source setup uses three trees and looks like this: -# -# official -# dragonegg -# trunk -# gcc -# trunk -# llvm -# trunk -# tools -# clang -# tags -# RELEASE_28 -# tools -# clang -# llvm-gcc -# trunk -# tags -# RELEASE_28 -# staging -# dragonegg -# trunk -# gcc -# trunk -# llvm -# trunk -# tools -# clang -# tags -# RELEASE_28 -# tools -# clang -# llvm-gcc -# trunk -# tags -# RELEASE_28 -# commit -# dragonegg -# trunk -# gcc -# trunk -# llvm -# trunk -# tools -# clang -# tags -# RELEASE_28 -# tools -# clang -# llvm-gcc -# trunk -# tags -# RELEASE_28 -# -# "gcc" above is the upstream FSF gcc and "gcc/trunk" refers to the -# 4.5 branch as discussed in the dragonegg build guide. -# -# In a typical workflow, the "official" tree always contains unchanged -# sources from the main LLVM project repositories. The "staging" tree -# is where local work is done. A set of changes resides there waiting -# to be moved upstream. The "commit" tree is where changes from -# "staging" make their way upstream. Individual incremental changes -# from "staging" are applied to "commit" and committed upstream after -# a successful build and test run. A successful build is one in which -# testing results in no more failures than seen in the testing of the -# "official" tree. -# -# A build may be invoked as such: -# -# llvmbuild --src=~/llvm/commit --src=~/llvm/staging -# --src=~/llvm/official --branch=trunk --branch=tags/RELEASE_28 -# --build=debug --build=release --build=paranoid -# --prefix=/home/greened/install --builddir=/home/greened/build -# -# This will build the LLVM ecosystem, including LLVM, Clang, llvm-gcc, -# gcc 4.5 and dragonegg, putting build results in ~/build and -# installing tools in ~/install. llvmbuild creates separate build and -# install directories for each source/branch/build flavor. In the -# above example, llvmbuild will build debug, release and paranoid -# (debug+checks) flavors of the trunk and RELEASE_28 branches from -# each source tree (official, staging and commit) for a total of -# eighteen builds. All builds will be run in parallel. -# -# The user may control parallelism via the --jobs and --threads -# switches. --jobs tells llvmbuild the maximum total number of builds -# to activate in parallel. The user may think of it as equivalent to -# the GNU make -j switch. --threads tells llvmbuild how many worker -# threads to use to accomplish those builds. If --threads is less -# than --jobs, --threads workers will be launched and each one will -# pick a source/branch/flavor combination to build. Then llvmbuild -# will invoke GNU make with -j (--jobs / --threads) to use up the -# remaining job capacity. Once a worker is finished with a build, it -# will pick another combination off the list and start building it. -# -##===----------------------------------------------------------------------===## - -import optparse -import os -import sys -import threading -import queue -import logging -import traceback -import subprocess -import re - -# TODO: Use shutil.which when it is available (3.2 or later) -def find_executable(executable, path=None): - """Try to find 'executable' in the directories listed in 'path' (a - string listing directories separated by 'os.pathsep'; defaults to - os.environ['PATH']). Returns the complete filename or None if not - found - """ - if path is None: - path = os.environ['PATH'] - paths = path.split(os.pathsep) - extlist = [''] - if os.name == 'os2': - (base, ext) = os.path.splitext(executable) - # executable files on OS/2 can have an arbitrary extension, but - # .exe is automatically appended if no dot is present in the name - if not ext: - executable = executable + ".exe" - elif sys.platform == 'win32': - pathext = os.environ['PATHEXT'].lower().split(os.pathsep) - (base, ext) = os.path.splitext(executable) - if ext.lower() not in pathext: - extlist = pathext - for ext in extlist: - execname = executable + ext - if os.path.isfile(execname): - return execname - else: - for p in paths: - f = os.path.join(p, execname) - if os.path.isfile(f): - return f - else: - return None - -def is_executable(fpath): - return os.path.exists(fpath) and os.access(fpath, os.X_OK) - -def add_options(parser): - parser.add_option("-v", "--verbose", action="store_true", - default=False, - help=("Output informational messages" - " [default: %default]")) - parser.add_option("--src", action="append", - help=("Top-level source directory [default: %default]")) - parser.add_option("--build", action="append", - help=("Build types to run [default: %default]")) - parser.add_option("--branch", action="append", - help=("Source branch to build [default: %default]")) - parser.add_option("--cc", default=find_executable("cc"), - help=("The C compiler to use [default: %default]")) - parser.add_option("--cxx", default=find_executable("c++"), - help=("The C++ compiler to use [default: %default]")) - parser.add_option("--threads", default=4, type="int", - help=("The number of worker threads to use " - "[default: %default]")) - parser.add_option("--jobs", "-j", default=8, type="int", - help=("The number of simultaneous build jobs " - "[default: %default]")) - parser.add_option("--prefix", - help=("Root install directory [default: %default]")) - parser.add_option("--builddir", - help=("Root build directory [default: %default]")) - parser.add_option("--extra-llvm-config-flags", default="", - help=("Extra flags to pass to llvm configure [default: %default]")) - parser.add_option("--extra-llvm-gcc-config-flags", default="", - help=("Extra flags to pass to llvm-gcc configure [default: %default]")) - parser.add_option("--extra-gcc-config-flags", default="", - help=("Extra flags to pass to gcc configure [default: %default]")) - parser.add_option("--force-configure", default=False, action="store_true", - help=("Force reconfigure of all components")) - parser.add_option("--no-gcc", default=False, action="store_true", - help=("Do not build dragonegg and gcc")) - parser.add_option("--no-install", default=False, action="store_true", - help=("Do not do installs")) - return - -def check_options(parser, options, valid_builds): - # See if we're building valid flavors. - for build in options.build: - if (build not in valid_builds): - parser.error("'" + build + "' is not a valid build flavor " - + str(valid_builds)) - - # See if we can find source directories. - for src in options.src: - for component in components: - component = component.rstrip("2") - compsrc = src + "/" + component - if (not os.path.isdir(compsrc)): - parser.error("'" + compsrc + "' does not exist") - if (options.branch is not None): - for branch in options.branch: - if (not os.path.isdir(os.path.join(compsrc, branch))): - parser.error("'" + os.path.join(compsrc, branch) - + "' does not exist") - - # See if we can find the compilers - options.cc = find_executable(options.cc) - options.cxx = find_executable(options.cxx) - - return - -# Find a unique short name for the given set of paths. This searches -# back through path components until it finds unique component names -# among all given paths. -def get_path_abbrevs(paths): - # Find the number of common starting characters in the last component - # of the paths. - unique_paths = list(paths) - - class NotFoundException(Exception): pass - - # Find a unique component of each path. - unique_bases = unique_paths[:] - found = 0 - while len(unique_paths) > 0: - bases = [os.path.basename(src) for src in unique_paths] - components = { c for c in bases } - # Account for single entry in paths. - if len(components) > 1 or len(components) == len(bases): - # We found something unique. - for c in components: - if bases.count(c) == 1: - index = bases.index(c) - unique_bases[index] = c - # Remove the corresponding path from the set under - # consideration. - unique_paths[index] = None - unique_paths = [ p for p in unique_paths if p is not None ] - unique_paths = [os.path.dirname(src) for src in unique_paths] - - if len(unique_paths) > 0: - raise NotFoundException() - - abbrevs = dict(zip(paths, [base for base in unique_bases])) - - return abbrevs - -# Given a set of unique names, find a short character sequence that -# uniquely identifies them. -def get_short_abbrevs(unique_bases): - # Find a unique start character for each path base. - my_unique_bases = unique_bases[:] - unique_char_starts = unique_bases[:] - while len(my_unique_bases) > 0: - for start, char_tuple in enumerate(zip(*[base - for base in my_unique_bases])): - chars = { c for c in char_tuple } - # Account for single path. - if len(chars) > 1 or len(chars) == len(char_tuple): - # We found something unique. - for c in chars: - if char_tuple.count(c) == 1: - index = char_tuple.index(c) - unique_char_starts[index] = start - # Remove the corresponding path from the set under - # consideration. - my_unique_bases[index] = None - my_unique_bases = [ b for b in my_unique_bases - if b is not None ] - break - - if len(my_unique_bases) > 0: - raise NotFoundException() - - abbrevs = [abbrev[start_index:start_index+3] - for abbrev, start_index - in zip([base for base in unique_bases], - [index for index in unique_char_starts])] - - abbrevs = dict(zip(unique_bases, abbrevs)) - - return abbrevs - -class Builder(threading.Thread): - class ExecutableNotFound(Exception): pass - class FileNotExecutable(Exception): pass - - def __init__(self, work_queue, jobs, - build_abbrev, source_abbrev, branch_abbrev, - options): - super().__init__() - self.work_queue = work_queue - self.jobs = jobs - self.cc = options.cc - self.cxx = options.cxx - self.build_abbrev = build_abbrev - self.source_abbrev = source_abbrev - self.branch_abbrev = branch_abbrev - self.build_prefix = options.builddir - self.install_prefix = options.prefix - self.options = options - self.component_abbrev = dict( - llvm="llvm", - llvm_gcc="lgcc", - llvm2="llv2", - gcc="ugcc", - dagonegg="degg") - def run(self): - while True: - try: - source, branch, build = self.work_queue.get() - self.dobuild(source, branch, build) - except: - traceback.print_exc() - finally: - self.work_queue.task_done() - - def execute(self, command, execdir, env, component): - prefix = self.component_abbrev[component.replace("-", "_")] - pwd = os.getcwd() - if not os.path.exists(execdir): - os.makedirs(execdir) - - execenv = os.environ.copy() - - for key, value in env.items(): - execenv[key] = value - - self.logger.debug("[" + prefix + "] " + "env " + str(env) + " " - + " ".join(command)); - - try: - proc = subprocess.Popen(command, - cwd=execdir, - env=execenv, - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT) - - line = proc.stdout.readline() - while line: - self.logger.info("[" + prefix + "] " - + str(line, "utf-8").rstrip()) - line = proc.stdout.readline() - - except: - traceback.print_exc() - - # Get a list of C++ include directories to pass to clang. - def get_includes(self): - # Assume we're building with g++ for now. - command = [self.cxx] - command += ["-v", "-x", "c++", "/dev/null", "-fsyntax-only"] - includes = [] - self.logger.debug(command) - try: - proc = subprocess.Popen(command, - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT) - - gather = False - line = proc.stdout.readline() - while line: - self.logger.debug(line) - if re.search("End of search list", str(line)) is not None: - self.logger.debug("Stop Gather") - gather = False - if gather: - includes.append(str(line, "utf-8").strip()) - if re.search("#include <...> search starts", str(line)) is not None: - self.logger.debug("Start Gather") - gather = True - line = proc.stdout.readline() - except: - traceback.print_exc() - self.logger.debug(includes) - return includes - - def dobuild(self, source, branch, build): - build_suffix = "" - - ssabbrev = get_short_abbrevs([ab for ab in self.source_abbrev.values()]) - - if branch is not None: - sbabbrev = get_short_abbrevs([ab for ab in self.branch_abbrev.values()]) - - prefix = "[" + ssabbrev[self.source_abbrev[source]] + "-" + sbabbrev[self.branch_abbrev[branch]] + "-" + self.build_abbrev[build] + "]" - self.install_prefix += "/" + self.source_abbrev[source] + "/" + branch + "/" + build - build_suffix += self.source_abbrev[source] + "/" + branch + "/" + build - else: - prefix = "[" + ssabbrev[self.source_abbrev[source]] + "-" + self.build_abbrev[build] + "]" - self.install_prefix += "/" + self.source_abbrev[source] + "/" + build - build_suffix += "/" + self.source_abbrev[source] + "/" + build - - self.logger = logging.getLogger(prefix) - - self.logger.debug(self.install_prefix) - - # Assume we're building with gcc for now. - cxxincludes = self.get_includes() - cxxroot = cxxincludes[0] - cxxarch = os.path.basename(cxxincludes[1]) - - configure_flags = dict( - llvm=dict(debug=["--prefix=" + self.install_prefix, - "--with-extra-options=-Werror", - "--enable-assertions", - "--disable-optimized", - "--with-cxx-include-root=" + cxxroot, - "--with-cxx-include-arch=" + cxxarch], - release=["--prefix=" + self.install_prefix, - "--with-extra-options=-Werror", - "--enable-optimized", - "--with-cxx-include-root=" + cxxroot, - "--with-cxx-include-arch=" + cxxarch], - paranoid=["--prefix=" + self.install_prefix, - "--with-extra-options=-Werror", - "--enable-assertions", - "--enable-expensive-checks", - "--disable-optimized", - "--with-cxx-include-root=" + cxxroot, - "--with-cxx-include-arch=" + cxxarch]), - llvm_gcc=dict(debug=["--prefix=" + self.install_prefix, - "--enable-checking", - "--program-prefix=llvm-", - "--enable-llvm=" + self.build_prefix + "/llvm/" + build_suffix, -# Fortran install seems to be broken. -# "--enable-languages=c,c++,fortran"], - "--enable-languages=c,c++"], - release=["--prefix=" + self.install_prefix, - "--program-prefix=llvm-", - "--enable-llvm=" + self.build_prefix + "/llvm/" + build_suffix, -# Fortran install seems to be broken. -# "--enable-languages=c,c++,fortran"], - "--enable-languages=c,c++"], - paranoid=["--prefix=" + self.install_prefix, - "--enable-checking", - "--program-prefix=llvm-", - "--enable-llvm=" + self.build_prefix + "/llvm/" + build_suffix, -# Fortran install seems to be broken. -# "--enable-languages=c,c++,fortran"]), - "--enable-languages=c,c++"]), - llvm2=dict(debug=["--prefix=" + self.install_prefix, - "--with-extra-options=-Werror", - "--enable-assertions", - "--disable-optimized", - "--with-llvmgccdir=" + self.install_prefix + "/bin", - "--with-cxx-include-root=" + cxxroot, - "--with-cxx-include-arch=" + cxxarch], - release=["--prefix=" + self.install_prefix, - "--with-extra-options=-Werror", - "--enable-optimized", - "--with-llvmgccdir=" + self.install_prefix + "/bin", - "--with-cxx-include-root=" + cxxroot, - "--with-cxx-include-arch=" + cxxarch], - paranoid=["--prefix=" + self.install_prefix, - "--with-extra-options=-Werror", - "--enable-assertions", - "--enable-expensive-checks", - "--disable-optimized", - "--with-llvmgccdir=" + self.install_prefix + "/bin", - "--with-cxx-include-root=" + cxxroot, - "--with-cxx-include-arch=" + cxxarch]), - gcc=dict(debug=["--prefix=" + self.install_prefix, - "--enable-checking"], - release=["--prefix=" + self.install_prefix], - paranoid=["--prefix=" + self.install_prefix, - "--enable-checking"]), - dragonegg=dict(debug=[], - release=[], - paranoid=[])) - - configure_env = dict( - llvm=dict(debug=dict(CC=self.cc, - CXX=self.cxx), - release=dict(CC=self.cc, - CXX=self.cxx), - paranoid=dict(CC=self.cc, - CXX=self.cxx)), - llvm_gcc=dict(debug=dict(CC=self.cc, - CXX=self.cxx), - release=dict(CC=self.cc, - CXX=self.cxx), - paranoid=dict(CC=self.cc, - CXX=self.cxx)), - llvm2=dict(debug=dict(CC=self.cc, - CXX=self.cxx), - release=dict(CC=self.cc, - CXX=self.cxx), - paranoid=dict(CC=self.cc, - CXX=self.cxx)), - gcc=dict(debug=dict(CC=self.cc, - CXX=self.cxx), - release=dict(CC=self.cc, - CXX=self.cxx), - paranoid=dict(CC=self.cc, - CXX=self.cxx)), - dragonegg=dict(debug=dict(CC=self.cc, - CXX=self.cxx), - release=dict(CC=self.cc, - CXX=self.cxx), - paranoid=dict(CC=self.cc, - CXX=self.cxx))) - - make_flags = dict( - llvm=dict(debug=["-j" + str(self.jobs)], - release=["-j" + str(self.jobs)], - paranoid=["-j" + str(self.jobs)]), - llvm_gcc=dict(debug=["-j" + str(self.jobs), - "bootstrap"], - release=["-j" + str(self.jobs), - "bootstrap"], - paranoid=["-j" + str(self.jobs), - "bootstrap"]), - llvm2=dict(debug=["-j" + str(self.jobs)], - release=["-j" + str(self.jobs)], - paranoid=["-j" + str(self.jobs)]), - gcc=dict(debug=["-j" + str(self.jobs), - "bootstrap"], - release=["-j" + str(self.jobs), - "bootstrap"], - paranoid=["-j" + str(self.jobs), - "bootstrap"]), - dragonegg=dict(debug=["-j" + str(self.jobs)], - release=["-j" + str(self.jobs)], - paranoid=["-j" + str(self.jobs)])) - - make_env = dict( - llvm=dict(debug=dict(), - release=dict(), - paranoid=dict()), - llvm_gcc=dict(debug=dict(), - release=dict(), - paranoid=dict()), - llvm2=dict(debug=dict(), - release=dict(), - paranoid=dict()), - gcc=dict(debug=dict(), - release=dict(), - paranoid=dict()), - dragonegg=dict(debug=dict(GCC=self.install_prefix + "/bin/gcc", - LLVM_CONFIG=self.install_prefix + "/bin/llvm-config"), - release=dict(GCC=self.install_prefix + "/bin/gcc", - LLVM_CONFIG=self.install_prefix + "/bin/llvm-config"), - paranoid=dict(GCC=self.install_prefix + "/bin/gcc", - LLVM_CONFIG=self.install_prefix + "/bin/llvm-config"))) - - make_install_flags = dict( - llvm=dict(debug=["install"], - release=["install"], - paranoid=["install"]), - llvm_gcc=dict(debug=["install"], - release=["install"], - paranoid=["install"]), - llvm2=dict(debug=["install"], - release=["install"], - paranoid=["install"]), - gcc=dict(debug=["install"], - release=["install"], - paranoid=["install"]), - dragonegg=dict(debug=["install"], - release=["install"], - paranoid=["install"])) - - make_install_env = dict( - llvm=dict(debug=dict(), - release=dict(), - paranoid=dict()), - llvm_gcc=dict(debug=dict(), - release=dict(), - paranoid=dict()), - llvm2=dict(debug=dict(), - release=dict(), - paranoid=dict()), - gcc=dict(debug=dict(), - release=dict(), - paranoid=dict()), - dragonegg=dict(debug=dict(), - release=dict(), - paranoid=dict())) - - make_check_flags = dict( - llvm=dict(debug=["check"], - release=["check"], - paranoid=["check"]), - llvm_gcc=dict(debug=["check"], - release=["check"], - paranoid=["check"]), - llvm2=dict(debug=["check"], - release=["check"], - paranoid=["check"]), - gcc=dict(debug=["check"], - release=["check"], - paranoid=["check"]), - dragonegg=dict(debug=["check"], - release=["check"], - paranoid=["check"])) - - make_check_env = dict( - llvm=dict(debug=dict(), - release=dict(), - paranoid=dict()), - llvm_gcc=dict(debug=dict(), - release=dict(), - paranoid=dict()), - llvm2=dict(debug=dict(), - release=dict(), - paranoid=dict()), - gcc=dict(debug=dict(), - release=dict(), - paranoid=dict()), - dragonegg=dict(debug=dict(), - release=dict(), - paranoid=dict())) - - for component in components: - comp = component[:] - - if (self.options.no_gcc): - if (comp == 'gcc' or comp == 'dragonegg' or comp == 'llvm2'): - self.logger.info("Skipping " + component + " in " - + builddir) - continue - - srcdir = source + "/" + comp.rstrip("2") - builddir = self.build_prefix + "/" + comp + "/" + build_suffix - installdir = self.install_prefix - - if (branch is not None): - srcdir += "/" + branch - - comp_key = comp.replace("-", "_") - - config_args = configure_flags[comp_key][build][:] - config_args.extend(getattr(self.options, - "extra_" + comp_key.rstrip("2") - + "_config_flags").split()) - - self.logger.info("Configuring " + component + " in " + builddir) - self.configure(component, srcdir, builddir, - config_args, - configure_env[comp_key][build]) - - self.logger.info("Building " + component + " in " + builddir) - self.logger.info("Build: make " + str(make_flags[comp_key][build])) - self.make(component, srcdir, builddir, - make_flags[comp_key][build], - make_env[comp_key][build]) - - if (not self.options.no_install): - self.logger.info("Installing " + component + " in " + installdir) - self.make(component, srcdir, builddir, - make_install_flags[comp_key][build], - make_install_env[comp_key][build]) - - self.logger.info("Testing " + component + " in " + builddir) - self.logger.info("Test: make " - + str(make_check_flags[comp_key][build])) - self.make(component, srcdir, builddir, - make_check_flags[comp_key][build], - make_check_env[comp_key][build]) - - - def configure(self, component, srcdir, builddir, flags, env): - self.logger.debug("Configure " + str(flags) + " " + str(srcdir) + " -> " - + str(builddir)) - - configure_files = dict( - llvm=[(srcdir + "/configure", builddir + "/Makefile")], - llvm_gcc=[(srcdir + "/configure", builddir + "/Makefile"), - (srcdir + "/gcc/configure", builddir + "/gcc/Makefile")], - llvm2=[(srcdir + "/configure", builddir + "/Makefile")], - gcc=[(srcdir + "/configure", builddir + "/Makefile"), - (srcdir + "/gcc/configure", builddir + "/gcc/Makefile")], - dragonegg=[()]) - - - doconfig = False - for conf, mf in configure_files[component.replace("-", "_")]: - if not os.path.exists(conf): - return - if os.path.exists(conf) and os.path.exists(mf): - confstat = os.stat(conf) - makestat = os.stat(mf) - if confstat.st_mtime > makestat.st_mtime: - doconfig = True - break - else: - doconfig = True - break - - if not doconfig and not self.options.force_configure: - return - - program = srcdir + "/configure" - if not is_executable(program): - return - - args = [program] - args += ["--verbose"] - args += flags - self.execute(args, builddir, env, component) - - def make(self, component, srcdir, builddir, flags, env): - program = find_executable("make") - if program is None: - raise ExecutableNotFound - - if not is_executable(program): - raise FileNotExecutable - - args = [program] - args += flags - self.execute(args, builddir, env, component) - -# Global constants -build_abbrev = dict(debug="dbg", release="opt", paranoid="par") -#components = ["llvm", "llvm-gcc", "llvm2", "gcc", "dragonegg"] -components = ["llvm", "llvm2", "gcc", "dragonegg"] - -# Parse options -parser = optparse.OptionParser(version="%prog 1.0") -add_options(parser) -(options, args) = parser.parse_args() -check_options(parser, options, build_abbrev.keys()); - -if options.verbose: - logging.basicConfig(level=logging.DEBUG, - format='%(name)-13s: %(message)s') -else: - logging.basicConfig(level=logging.INFO, - format='%(name)-13s: %(message)s') - -source_abbrev = get_path_abbrevs(set(options.src)) - -branch_abbrev = None -if options.branch is not None: - branch_abbrev = get_path_abbrevs(set(options.branch)) - -work_queue = queue.Queue() - -jobs = options.jobs // options.threads -if jobs == 0: - jobs = 1 - -numthreads = options.threads - -logging.getLogger().info("Building with " + str(options.jobs) + " jobs and " - + str(numthreads) + " threads using " + str(jobs) - + " make jobs") - -for t in range(numthreads): - builder = Builder(work_queue, jobs, - build_abbrev, source_abbrev, branch_abbrev, - options) - builder.daemon = True - builder.start() - -for build in set(options.build): - for source in set(options.src): - if options.branch is not None: - for branch in set(options.branch): - work_queue.put((source, branch, build)) - else: - work_queue.put((source, None, build)) - -work_queue.join() |
