build-clang.py (33028B)
1 #!/usr/bin/python3 2 # This Source Code Form is subject to the terms of the Mozilla Public 3 # License, v. 2.0. If a copy of the MPL was not distributed with this 4 # file, You can obtain one at http://mozilla.org/MPL/2.0/. 5 6 # Only necessary for flake8 to be happy... 7 import argparse 8 import errno 9 import fnmatch 10 import glob 11 import json 12 import os 13 import os.path 14 import platform 15 import re 16 import shutil 17 import subprocess 18 import sys 19 import tarfile 20 from contextlib import contextmanager 21 from shutil import which 22 23 import zstandard 24 25 SUPPORTED_TARGETS = { 26 "x86_64-unknown-linux-gnu": ("Linux", "x86_64"), 27 "aarch64-unknown-linux-gnu": ("Linux", "aarch64"), 28 "x86_64-pc-windows-msvc": ("Windows", "AMD64"), 29 "aarch64-pc-windows-msvc": ("Windows", "ARM64"), 30 "x86_64-apple-darwin": ("Darwin", "x86_64"), 31 "aarch64-apple-darwin": ("Darwin", "arm64"), 32 } 33 34 35 def is_llvm_toolchain(cc, cxx): 36 return "clang" in cc and "clang" in cxx 37 38 39 def check_run(args): 40 print(" ".join(args), file=sys.stderr, flush=True) 41 if args[0] == "cmake": 42 # CMake `message(STATUS)` messages, as appearing in failed source code 43 # compiles, appear on stdout, so we only capture that. 44 p = subprocess.Popen(args, stdout=subprocess.PIPE) 45 lines = [] 46 for line in p.stdout: 47 lines.append(line) 48 sys.stdout.write(line.decode()) 49 sys.stdout.flush() 50 r = p.wait() 51 if r != 0 and os.environ.get("UPLOAD_DIR"): 52 cmake_output_re = re.compile(b'See also "(.*/CMakeOutput.log)"') 53 cmake_error_re = re.compile(b'See also "(.*/CMakeError.log)"') 54 55 def find_first_match(re): 56 for l in lines: 57 match = re.search(l) 58 if match: 59 return match 60 61 output_match = find_first_match(cmake_output_re) 62 error_match = find_first_match(cmake_error_re) 63 64 upload_dir = os.environ["UPLOAD_DIR"].encode("utf-8") 65 if output_match or error_match: 66 mkdir_p(upload_dir) 67 if output_match: 68 shutil.copy2(output_match.group(1), upload_dir) 69 if error_match: 70 shutil.copy2(error_match.group(1), upload_dir) 71 else: 72 r = subprocess.call(args) 73 assert r == 0 74 75 76 def run_in(path, args): 77 with chdir(path): 78 check_run(args) 79 80 81 @contextmanager 82 def chdir(path): 83 d = os.getcwd() 84 print('cd "%s"' % path, file=sys.stderr) 85 os.chdir(path) 86 try: 87 yield 88 finally: 89 print('cd "%s"' % d, file=sys.stderr) 90 os.chdir(d) 91 92 93 def patch(patch, srcdir): 94 patch = os.path.realpath(patch) 95 check_run(["patch", "-d", srcdir, "-p1", "-i", patch, "--fuzz=0", "-s"]) 96 97 98 def import_clang_tidy(source_dir, build_clang_tidy_alpha, build_clang_tidy_external): 99 clang_plugin_path = os.path.join(os.path.dirname(sys.argv[0]), "..", "clang-plugin") 100 clang_tidy_path = os.path.join(source_dir, "clang-tools-extra/clang-tidy") 101 sys.path.append(clang_plugin_path) 102 from import_mozilla_checks import do_import 103 104 import_options = { 105 "alpha": build_clang_tidy_alpha, 106 "external": build_clang_tidy_external, 107 } 108 do_import(clang_plugin_path, clang_tidy_path, import_options) 109 110 111 def build_package(package_build_dir, cmake_args): 112 if not os.path.exists(package_build_dir): 113 os.mkdir(package_build_dir) 114 # If CMake has already been run, it may have been run with different 115 # arguments, so we need to re-run it. Make sure the cached copy of the 116 # previous CMake run is cleared before running it again. 117 if os.path.exists(package_build_dir + "/CMakeCache.txt"): 118 os.remove(package_build_dir + "/CMakeCache.txt") 119 if os.path.exists(package_build_dir + "/CMakeFiles"): 120 shutil.rmtree(package_build_dir + "/CMakeFiles") 121 122 run_in(package_build_dir, ["cmake"] + cmake_args) 123 run_in(package_build_dir, ["ninja", "install", "-v"]) 124 125 126 @contextmanager 127 def updated_env(env): 128 old_env = os.environ.copy() 129 os.environ.update(env) 130 yield 131 os.environ.clear() 132 os.environ.update(old_env) 133 134 135 def build_tar_package(name, base, directory): 136 name = os.path.realpath(name) 137 print(f"tarring {name} from {base}/{directory}", file=sys.stderr) 138 assert name.endswith(".tar.zst") 139 140 cctx = zstandard.ZstdCompressor() 141 with open(name, "wb") as f, cctx.stream_writer(f) as z: 142 with tarfile.open(mode="w|", fileobj=z) as tf: 143 with chdir(base): 144 tf.add(directory) 145 146 147 def mkdir_p(path): 148 try: 149 os.makedirs(path) 150 except OSError as e: 151 if e.errno != errno.EEXIST or not os.path.isdir(path): 152 raise 153 154 155 def delete(path): 156 if os.path.isdir(path): 157 shutil.rmtree(path) 158 else: 159 try: 160 os.unlink(path) 161 except Exception: 162 pass 163 164 165 def install_import_library(build_dir, clang_dir): 166 shutil.copy2( 167 os.path.join(build_dir, "lib", "clang.lib"), os.path.join(clang_dir, "lib") 168 ) 169 170 171 def is_darwin(target): 172 return "-apple-darwin" in target 173 174 175 def is_linux(target): 176 return "-linux-gnu" in target 177 178 179 def is_windows(target): 180 return "-windows-msvc" in target 181 182 183 def is_cross_compile(target): 184 target_system, target_machine = SUPPORTED_TARGETS[target] 185 system, machine = (platform.system(), platform.machine()) 186 if system != target_system: 187 return True 188 # Don't consider x86 mac on arm64 mac a cross-compile so that we 189 # can build x86 mac clang on arm64 mac via Rosetta, as if they 190 # were building on x86. 191 if system == "Darwin" and machine == "arm64": 192 return False 193 return machine != target_machine 194 195 196 def build_one_stage( 197 cc, 198 cxx, 199 asm, 200 ar, 201 ranlib, 202 libtool, 203 ldflags, 204 src_dir, 205 stage_dir, 206 package_name, 207 build_type, 208 assertions, 209 target, 210 targets, 211 is_final_stage=False, 212 profile=None, 213 bolt=False, 214 ): 215 if not os.path.exists(stage_dir): 216 os.mkdir(stage_dir) 217 218 build_dir = stage_dir + "/build" 219 inst_dir = stage_dir + "/" + package_name 220 221 # cmake doesn't deal well with backslashes in paths. 222 def slashify_path(path): 223 return path.replace("\\", "/") 224 225 def cmake_base_args(cc, cxx, asm, ar, ranlib, libtool, ldflags, inst_dir): 226 if is_final_stage and targets: 227 machine_targets = targets 228 elif target.startswith("aarch64-"): 229 machine_targets = "AArch64" 230 else: 231 machine_targets = "X86" 232 233 # see llvm-project/clang/cmake/caches/BOLT.cmake 234 if bolt: 235 ldflags.append("-Wl,--emit-relocs,-znow") 236 237 # libxml2 2.13+ Windows builds introduced a hard dependency on bcrypt 238 # which must also be specified or else LibXml2 detection fails. 239 if is_windows(target) and is_final_stage: 240 ldflags.append("/DEFAULTLIB:bcrypt") 241 242 cmake_args = [ 243 "-GNinja", 244 "-DCMAKE_C_COMPILER=%s" % slashify_path(cc[0]), 245 "-DCMAKE_CXX_COMPILER=%s" % slashify_path(cxx[0]), 246 "-DCMAKE_ASM_COMPILER=%s" % slashify_path(asm[0]), 247 "-DCMAKE_AR=%s" % slashify_path(ar), 248 "-DCMAKE_C_FLAGS_INIT=%s" % " ".join(cc[1:]), 249 "-DCMAKE_CXX_FLAGS_INIT=%s" % " ".join(cxx[1:]), 250 "-DCMAKE_ASM_FLAGS_INIT=%s" % " ".join(asm[1:]), 251 "-DCMAKE_EXE_LINKER_FLAGS_INIT=%s" % " ".join(ldflags), 252 "-DCMAKE_SHARED_LINKER_FLAGS_INIT=%s" % " ".join(ldflags), 253 "-DCMAKE_BUILD_TYPE=%s" % build_type, 254 "-DCMAKE_INSTALL_PREFIX=%s" % inst_dir, 255 "-DLLVM_TARGETS_TO_BUILD=%s" % machine_targets, 256 "-DLLVM_ENABLE_PER_TARGET_RUNTIME_DIR=OFF", 257 "-DLLVM_ENABLE_ASSERTIONS=%s" % ("ON" if assertions else "OFF"), 258 "-DLLVM_ENABLE_BINDINGS=OFF", 259 "-DLLVM_ENABLE_CURL=OFF", 260 "-DLLVM_INCLUDE_TESTS=OFF", 261 "-DLLVM_HOST_TRIPLE=%s" % target, 262 "-DCMAKE_C_COMPILER_TARGET=%s" % target, 263 "-DCMAKE_CXX_COMPILER_TARGET=%s" % target, 264 "-DCMAKE_ASM_COMPILER_TARGET=%s" % target, 265 ] 266 if is_cross_compile(target): 267 cmake_args += [ 268 "-DCMAKE_SYSTEM_NAME=%s" % SUPPORTED_TARGETS[target][0], 269 ] 270 if is_llvm_toolchain(cc[0], cxx[0]): 271 cmake_args += ["-DLLVM_ENABLE_LLD=ON"] 272 elif is_windows(target) and is_cross_compile(target): 273 raise Exception( 274 "Cannot cross-compile for Windows with a compiler that is not clang" 275 ) 276 277 if "TASK_ID" in os.environ: 278 cmake_args += [ 279 "-DCLANG_REPOSITORY_STRING=taskcluster-%s" % os.environ["TASK_ID"], 280 ] 281 projects = ["clang", "lld"] 282 if is_final_stage: 283 projects.append("clang-tools-extra") 284 else: 285 cmake_args.append("-DLLVM_TOOL_LLI_BUILD=OFF") 286 287 if bolt: 288 projects.append("bolt") 289 cmake_args.append("-DCLANG_BOLT=INSTRUMENT") 290 cmake_args.append("-DCLANG_INCLUDE_TESTS=ON") 291 292 cmake_args.append("-DLLVM_ENABLE_PROJECTS=%s" % ";".join(projects)) 293 294 if is_final_stage: 295 cmake_args += ["-DLLVM_ENABLE_LIBXML2=FORCE_ON"] 296 if is_linux(target) and is_final_stage: 297 sysroot = os.path.join(os.environ.get("MOZ_FETCHES_DIR", ""), "sysroot") 298 if os.path.exists(sysroot): 299 cmake_args += ["-DLLVM_BINUTILS_INCDIR=/usr/include"] 300 cmake_args += ["-DCMAKE_SYSROOT=%s" % sysroot] 301 # Work around the LLVM build system not building the i386 compiler-rt 302 # because it doesn't allow to use a sysroot for that during the cmake 303 # checks. 304 cmake_args += ["-DCAN_TARGET_i386=1"] 305 cmake_args += ["-DLLVM_ENABLE_TERMINFO=OFF"] 306 libxml2 = os.path.join(os.environ.get("MOZ_FETCHES_DIR", ""), "libxml2") 307 if os.path.exists(libxml2): 308 cmake_args += [ 309 "-DLIBXML2_DEFINITIONS=-DLIBXML_STATIC", 310 f"-DLIBXML2_INCLUDE_DIR={libxml2}/include/libxml2", 311 f"-DLIBXML2_LIBRARIES={libxml2}/lib/libxml2.a", 312 ] 313 if is_windows(target): 314 cmake_args.insert(-1, "-DLLVM_EXPORT_SYMBOLS_FOR_PLUGINS=ON") 315 cmake_args.insert(-1, "-DCMAKE_MSVC_RUNTIME_LIBRARY=MultiThreaded") 316 if is_cross_compile(target): 317 cmake_args += [ 318 f"-DCMAKE_TOOLCHAIN_FILE={src_dir}/cmake/platforms/WinMsvc.cmake", 319 f"-DLLVM_NATIVE_TOOLCHAIN={os.path.dirname(os.path.dirname(cc[0]))}", 320 f"-DHOST_ARCH={target[: -len('-pc-windows-msvc')]}", 321 f"-DLLVM_WINSYSROOT={os.environ['VSINSTALLDIR']}", 322 "-DLLVM_DISABLE_ASSEMBLY_FILES=ON", 323 ] 324 if is_final_stage: 325 fetches = os.environ["MOZ_FETCHES_DIR"] 326 cmake_args += [ 327 "-DLIBXML2_DEFINITIONS=-DLIBXML_STATIC", 328 f"-DLIBXML2_INCLUDE_DIR={fetches}/libxml2/include/libxml2", 329 f"-DLIBXML2_LIBRARIES={fetches}/libxml2/lib/libxml2s.lib", 330 ] 331 else: 332 # libllvm as a shared library is not supported on Windows 333 cmake_args += ["-DLLVM_LINK_LLVM_DYLIB=ON"] 334 if ranlib is not None: 335 cmake_args += ["-DCMAKE_RANLIB=%s" % slashify_path(ranlib)] 336 if libtool is not None: 337 cmake_args += ["-DCMAKE_LIBTOOL=%s" % slashify_path(libtool)] 338 if is_darwin(target): 339 arch = "arm64" if target.startswith("aarch64") else "x86_64" 340 cmake_args += [ 341 "-DCMAKE_SYSTEM_VERSION=%s" % os.environ["MACOSX_DEPLOYMENT_TARGET"], 342 "-DCMAKE_OSX_SYSROOT=%s" % slashify_path(os.getenv("OSX_SYSROOT")), 343 "-DCMAKE_FIND_ROOT_PATH=%s" % slashify_path(os.getenv("OSX_SYSROOT")), 344 "-DCMAKE_FIND_ROOT_PATH_MODE_PROGRAM=NEVER", 345 "-DCMAKE_FIND_ROOT_PATH_MODE_LIBRARY=ONLY", 346 "-DCMAKE_FIND_ROOT_PATH_MODE_INCLUDE=ONLY", 347 "-DCMAKE_MACOSX_RPATH=ON", 348 "-DCMAKE_OSX_ARCHITECTURES=%s" % arch, 349 "-DDARWIN_osx_ARCHS=%s" % arch, 350 "-DDARWIN_osx_SYSROOT=%s" % slashify_path(os.getenv("OSX_SYSROOT")), 351 ] 352 if arch == "arm64": 353 cmake_args += [ 354 "-DDARWIN_osx_BUILTIN_ARCHS=arm64", 355 ] 356 # Starting in LLVM 11 (which requires SDK 10.12) the build tries to 357 # detect the SDK version by calling xcrun. Cross-compiles don't have 358 # an xcrun, so we have to set the version explicitly. 359 cmake_args += [ 360 "-DDARWIN_macosx_OVERRIDE_SDK_VERSION=%s" 361 % os.environ["MACOSX_DEPLOYMENT_TARGET"], 362 ] 363 364 if profile == "gen": 365 # Per https://releases.llvm.org/10.0.0/docs/HowToBuildWithPGO.html 366 cmake_args += [ 367 "-DLLVM_BUILD_INSTRUMENTED=IR", 368 "-DLLVM_BUILD_RUNTIME=No", 369 ] 370 elif profile: 371 cmake_args += [ 372 "-DLLVM_PROFDATA_FILE=%s" % profile, 373 ] 374 375 # Using LTO for both profile generation and usage to avoid most 376 # "function control flow change detected (hash mismatch)" error. 377 if profile and not is_windows(target): 378 cmake_args.append("-DLLVM_ENABLE_LTO=Thin") 379 return cmake_args 380 381 cmake_args = [] 382 cmake_args += cmake_base_args(cc, cxx, asm, ar, ranlib, libtool, ldflags, inst_dir) 383 cmake_args += [src_dir] 384 build_package(build_dir, cmake_args) 385 386 # For some reasons the import library clang.lib of clang.exe is not 387 # installed, so we copy it by ourselves. 388 if is_windows(target) and is_final_stage: 389 install_import_library(build_dir, inst_dir) 390 391 392 # Return the absolute path of a build tool. We first look to see if the 393 # variable is defined in the config file, and if so we make sure it's an 394 # absolute path to an existing tool, otherwise we look for a program in 395 # $PATH named "key". 396 # 397 # This expects the name of the key in the config file to match the name of 398 # the tool in the default toolchain on the system (for example, "ld" on Unix 399 # and "link" on Windows). 400 def get_tool(config, key): 401 f = None 402 if key in config: 403 f = config[key].format(**os.environ) 404 if os.path.isabs(f): 405 path, f = os.path.split(f) 406 # Searches for .exes on windows too, even if the extension is 407 # not given. which(absolute_path) doesn't do that until python 3.12. 408 f = which(f, path=path) 409 if not f: 410 raise ValueError("%s must point to an existing path" % key) 411 return f 412 413 # Assume that we have the name of some program that should be on PATH. 414 tool = which(f) if f else which(key) 415 if not tool: 416 raise ValueError("%s not found on PATH" % (f or key)) 417 return tool 418 419 420 # This function is intended to be called on the final build directory when 421 # building clang-tidy. Also clang-format binaries are included that can be used 422 # in conjunction with clang-tidy. 423 # As a separate binary we also ship clangd for the language server protocol that 424 # can be used as a plugin in `vscode`. 425 # Its job is to remove all of the files which won't be used for clang-tidy or 426 # clang-format to reduce the download size. Currently when this function 427 # finishes its job, it will leave final_dir with a layout like this: 428 # 429 # clang/ 430 # bin/ 431 # clang-apply-replacements 432 # clang-format 433 # clang-tidy 434 # clangd 435 # run-clang-tidy 436 # include/ 437 # * (nothing will be deleted here) 438 # lib/ 439 # clang/ 440 # 4.0.0/ 441 # include/ 442 # * (nothing will be deleted here) 443 # share/ 444 # clang/ 445 # clang-format-diff.py 446 # clang-tidy-diff.py 447 # run-clang-tidy.py 448 def prune_final_dir_for_clang_tidy(final_dir, target): 449 # Make sure we only have what we expect. 450 dirs = [ 451 "bin", 452 "include", 453 "lib", 454 "lib32", 455 "libexec", 456 "msbuild-bin", 457 "share", 458 "tools", 459 ] 460 if is_linux(target): 461 dirs.append("x86_64-unknown-linux-gnu") 462 for f in glob.glob("%s/*" % final_dir): 463 if os.path.basename(f) not in dirs: 464 raise Exception("Found unknown file %s in the final directory" % f) 465 if not os.path.isdir(f): 466 raise Exception("Expected %s to be a directory" % f) 467 468 kept_binaries = [ 469 "clang-apply-replacements", 470 "clang-format", 471 "clang-tidy", 472 "clangd", 473 "clang-query", 474 "run-clang-tidy", 475 ] 476 re_clang_tidy = re.compile(r"^(" + "|".join(kept_binaries) + r")(\.exe)?$", re.I) 477 for f in glob.glob("%s/bin/*" % final_dir): 478 if re_clang_tidy.search(os.path.basename(f)) is None: 479 delete(f) 480 481 # Keep include/ intact. 482 483 # Remove the target-specific files. 484 if is_linux(target): 485 if os.path.exists(os.path.join(final_dir, "x86_64-unknown-linux-gnu")): 486 shutil.rmtree(os.path.join(final_dir, "x86_64-unknown-linux-gnu")) 487 488 # In lib/, only keep lib/clang/N.M.O/include and the LLVM shared library. 489 re_ver_num = re.compile(r"^\d+(?:\.\d+\.\d+)?$", re.I) 490 for f in glob.glob("%s/lib/*" % final_dir): 491 name = os.path.basename(f) 492 if name == "clang": 493 continue 494 if is_darwin(target) and name in ["libLLVM.dylib", "libclang-cpp.dylib"]: 495 continue 496 if is_linux(target) and ( 497 fnmatch.fnmatch(name, "libLLVM*.so*") 498 or fnmatch.fnmatch(name, "libclang-cpp.so*") 499 ): 500 continue 501 delete(f) 502 for f in glob.glob("%s/lib/clang/*" % final_dir): 503 if re_ver_num.search(os.path.basename(f)) is None: 504 delete(f) 505 for f in glob.glob("%s/lib/clang/*/*" % final_dir): 506 if os.path.basename(f) != "include": 507 delete(f) 508 509 # Completely remove libexec/, msbuild-bin and tools, if it exists. 510 shutil.rmtree(os.path.join(final_dir, "libexec")) 511 for d in ("msbuild-bin", "tools"): 512 d = os.path.join(final_dir, d) 513 if os.path.exists(d): 514 shutil.rmtree(d) 515 516 # In share/, only keep share/clang/*tidy* 517 re_clang_tidy = re.compile(r"format|tidy", re.I) 518 for f in glob.glob("%s/share/*" % final_dir): 519 if os.path.basename(f) != "clang": 520 delete(f) 521 for f in glob.glob("%s/share/clang/*" % final_dir): 522 if re_clang_tidy.search(os.path.basename(f)) is None: 523 delete(f) 524 525 526 def main(): 527 parser = argparse.ArgumentParser() 528 parser.add_argument( 529 "-c", 530 "--config", 531 action="append", 532 required=True, 533 type=argparse.FileType("r"), 534 help="Clang configuration file", 535 ) 536 parser.add_argument( 537 "--clean", required=False, action="store_true", help="Clean the build directory" 538 ) 539 parser.add_argument( 540 "--skip-tar", 541 required=False, 542 action="store_true", 543 help="Skip tar packaging stage", 544 ) 545 parser.add_argument( 546 "--skip-patch", 547 required=False, 548 action="store_true", 549 help="Do not patch source", 550 ) 551 552 args = parser.parse_args() 553 554 if not os.path.exists("llvm/README.txt"): 555 raise Exception( 556 "The script must be run from the root directory of the llvm-project tree" 557 ) 558 source_dir = os.getcwd() 559 build_dir = source_dir + "/build" 560 561 if args.clean: 562 shutil.rmtree(build_dir) 563 os.sys.exit(0) 564 565 llvm_source_dir = source_dir + "/llvm" 566 567 config = {} 568 # Merge all the configs we got from the command line. 569 for c in args.config: 570 this_config_dir = os.path.dirname(c.name) 571 this_config = json.load(c) 572 patches = this_config.get("patches") 573 if patches: 574 this_config["patches"] = [os.path.join(this_config_dir, p) for p in patches] 575 for key, value in this_config.items(): 576 old_value = config.get(key) 577 if old_value is None: 578 config[key] = value 579 elif value is None: 580 if key in config: 581 del config[key] 582 elif type(old_value) is not type(value): 583 raise Exception( 584 f"{c.name} is overriding `{key}` with a value of the wrong type" 585 ) 586 elif isinstance(old_value, list): 587 for v in value: 588 if v not in old_value: 589 old_value.append(v) 590 elif isinstance(old_value, dict): 591 raise Exception(f"{c.name} is setting `{key}` to a dict?") 592 else: 593 config[key] = value 594 595 stages = 2 596 if "stages" in config: 597 stages = int(config["stages"]) 598 if stages not in (1, 2, 3, 4): 599 raise ValueError("We only know how to build 1, 2, 3, or 4 stages.") 600 skip_stages = 0 601 if "skip_stages" in config: 602 # The assumption here is that the compiler given in `cc` and other configs 603 # is the result of the last skip stage, built somewhere else. 604 skip_stages = int(config["skip_stages"]) 605 if skip_stages >= stages: 606 raise ValueError("Cannot skip more stages than are built.") 607 pgo = False 608 if "pgo" in config: 609 pgo = config["pgo"] 610 if pgo not in (True, False): 611 raise ValueError("Only boolean values are accepted for pgo.") 612 bolt = config.get("bolt", False) 613 if bolt not in (True, False): 614 raise ValueError("Only boolean values are accepted for bolt.") 615 build_type = "Release" 616 if "build_type" in config: 617 build_type = config["build_type"] 618 if build_type not in ("Release", "Debug", "RelWithDebInfo", "MinSizeRel"): 619 raise ValueError( 620 "We only know how to do Release, Debug, RelWithDebInfo or " 621 "MinSizeRel builds" 622 ) 623 targets = config.get("targets") 624 build_clang_tidy = False 625 if "build_clang_tidy" in config: 626 build_clang_tidy = config["build_clang_tidy"] 627 if build_clang_tidy not in (True, False): 628 raise ValueError("Only boolean values are accepted for build_clang_tidy.") 629 build_clang_tidy_alpha = False 630 # check for build_clang_tidy_alpha only if build_clang_tidy is true 631 if build_clang_tidy and "build_clang_tidy_alpha" in config: 632 build_clang_tidy_alpha = config["build_clang_tidy_alpha"] 633 if build_clang_tidy_alpha not in (True, False): 634 raise ValueError( 635 "Only boolean values are accepted for build_clang_tidy_alpha." 636 ) 637 build_clang_tidy_external = False 638 # check for build_clang_tidy_external only if build_clang_tidy is true 639 if build_clang_tidy and "build_clang_tidy_external" in config: 640 build_clang_tidy_external = config["build_clang_tidy_external"] 641 if build_clang_tidy_external not in (True, False): 642 raise ValueError( 643 "Only boolean values are accepted for build_clang_tidy_external." 644 ) 645 assertions = False 646 if "assertions" in config: 647 assertions = config["assertions"] 648 if assertions not in (True, False): 649 raise ValueError("Only boolean values are accepted for assertions.") 650 651 for t in SUPPORTED_TARGETS: 652 if not is_cross_compile(t): 653 host = t 654 break 655 else: 656 raise Exception( 657 f"Cannot use this script on {platform.system()} {platform.machine()}" 658 ) 659 660 target = config.get("target", host) 661 if target not in SUPPORTED_TARGETS: 662 raise ValueError(f"{target} is not a supported target.") 663 664 if is_cross_compile(target) and not is_linux(host): 665 raise Exception("Cross-compilation is only supported on Linux") 666 667 if is_darwin(target): 668 os.environ["MACOSX_DEPLOYMENT_TARGET"] = ( 669 "11.0" if target.startswith("aarch64") else "10.15" 670 ) 671 672 if is_windows(target): 673 exe_ext = ".exe" 674 cc_name = "clang-cl" 675 cxx_name = "clang-cl" 676 677 # Used by llvm/lib/DebugInfo/PDB 678 os.environ["VSCMD_ARG_TGT_ARCH"] = SUPPORTED_TARGETS[target][1].lower() 679 else: 680 exe_ext = "" 681 cc_name = "clang" 682 cxx_name = "clang++" 683 684 cc = get_tool(config, "cc") 685 cxx = get_tool(config, "cxx") 686 asm = get_tool(config, "ml" if is_windows(target) else "as") 687 # Not using lld here as default here because it's not in PATH. But clang 688 # knows how to find it when they are installed alongside each others. 689 ar = get_tool(config, "lib" if is_windows(target) else "ar") 690 ranlib = None if is_windows(target) else get_tool(config, "ranlib") 691 libtool = get_tool(config, "libtool") if is_darwin(target) else None 692 693 if not os.path.exists(source_dir): 694 os.makedirs(source_dir) 695 696 if not args.skip_patch: 697 for p in config.get("patches", []): 698 patch(p, source_dir) 699 700 package_name = "clang" 701 if build_clang_tidy: 702 package_name = "clang-tidy" 703 if not args.skip_patch: 704 import_clang_tidy( 705 source_dir, build_clang_tidy_alpha, build_clang_tidy_external 706 ) 707 708 if not os.path.exists(build_dir): 709 os.makedirs(build_dir) 710 711 stage1_dir = build_dir + "/stage1" 712 stage1_inst_dir = stage1_dir + "/" + package_name 713 714 final_stage_dir = stage1_dir 715 716 if is_darwin(target): 717 extra_cflags = [] 718 extra_cxxflags = [] 719 extra_cflags2 = [] 720 extra_cxxflags2 = [] 721 extra_asmflags = [] 722 # It's unfortunately required to specify the linker used here because 723 # the linker flags are used in LLVM's configure step before 724 # -DLLVM_ENABLE_LLD is actually processed. 725 extra_ldflags = [ 726 "-fuse-ld=lld", 727 "-Wl,-dead_strip", 728 ] 729 elif is_linux(target): 730 extra_cflags = [] 731 extra_cxxflags = [] 732 extra_cflags2 = ["-fPIC"] 733 # Silence clang's warnings about arguments not being used in compilation. 734 extra_cxxflags2 = [ 735 "-fPIC", 736 "-Qunused-arguments", 737 ] 738 extra_asmflags = [] 739 # Avoid libLLVM internal function calls going through the PLT. 740 extra_ldflags = ["-Wl,-Bsymbolic-functions"] 741 # For whatever reason, LLVM's build system will set things up to turn 742 # on -ffunction-sections and -fdata-sections, but won't turn on the 743 # corresponding option to strip unused sections. We do it explicitly 744 # here. LLVM's build system is also picky about turning on ICF, so 745 # we do that explicitly here, too. 746 747 # It's unfortunately required to specify the linker used here because 748 # the linker flags are used in LLVM's configure step before 749 # -DLLVM_ENABLE_LLD is actually processed. 750 if is_llvm_toolchain(cc, cxx): 751 extra_ldflags += ["-fuse-ld=lld", "-Wl,--icf=safe"] 752 extra_ldflags += ["-Wl,--gc-sections"] 753 elif is_windows(target): 754 extra_cflags = [] 755 extra_cxxflags = [] 756 # clang-cl would like to figure out what it's supposed to be emulating 757 # by looking at an MSVC install, but we don't really have that here. 758 # Force things on based on WinMsvc.cmake. 759 # Ideally, we'd just use WinMsvc.cmake as a toolchain file, but it only 760 # really works for cross-compiles, which this is not. 761 with open(os.path.join(llvm_source_dir, "cmake/platforms/WinMsvc.cmake")) as f: 762 compat = [ 763 item 764 for line in f 765 for item in line.split() 766 if "-fms-compatibility-version=" in item 767 ][0] 768 extra_cflags2 = [compat] 769 extra_cxxflags2 = [compat] 770 extra_asmflags = [] 771 extra_ldflags = [] 772 773 upload_dir = os.getenv("UPLOAD_DIR") 774 if assertions and upload_dir: 775 extra_cflags2 += ["-fcrash-diagnostics-dir=%s" % upload_dir] 776 extra_cxxflags2 += ["-fcrash-diagnostics-dir=%s" % upload_dir] 777 778 if skip_stages < 1: 779 build_one_stage( 780 [cc] + extra_cflags, 781 [cxx] + extra_cxxflags, 782 [asm] + extra_asmflags, 783 ar, 784 ranlib, 785 libtool, 786 extra_ldflags, 787 llvm_source_dir, 788 stage1_dir, 789 package_name, 790 build_type, 791 assertions, 792 target, 793 targets, 794 is_final_stage=(stages == 1), 795 ) 796 797 if stages >= 2 and skip_stages < 2: 798 stage2_dir = build_dir + "/stage2" 799 stage2_inst_dir = stage2_dir + "/" + package_name 800 final_stage_dir = stage2_dir 801 if skip_stages < 1: 802 cc = stage1_inst_dir + "/bin/%s%s" % (cc_name, exe_ext) 803 cxx = stage1_inst_dir + "/bin/%s%s" % (cxx_name, exe_ext) 804 asm = stage1_inst_dir + "/bin/%s%s" % (cc_name, exe_ext) 805 name_compression = [] 806 if is_windows(target) and is_cross_compile(target) and pgo: 807 # native llvm-profdata.exe on Windows can't read profile data 808 # if name compression is enabled (which cross-compiling enables 809 # by default) 810 name_compression = ["-mllvm", "--enable-name-compression=false"] 811 build_one_stage( 812 [cc] + extra_cflags2 + name_compression, 813 [cxx] + extra_cxxflags2 + name_compression, 814 [asm] + extra_asmflags, 815 ar, 816 ranlib, 817 libtool, 818 extra_ldflags, 819 llvm_source_dir, 820 stage2_dir, 821 package_name, 822 build_type, 823 assertions, 824 target, 825 targets, 826 is_final_stage=(stages == 2 and not pgo), 827 profile="gen" if pgo else None, 828 ) 829 830 if stages >= 3 and skip_stages < 3: 831 stage3_dir = build_dir + "/stage3" 832 if pgo: 833 profiles_dir = build_dir + "/profiles" 834 mkdir_p(profiles_dir) 835 os.environ["LLVM_PROFILE_FILE"] = profiles_dir + "/%m.profraw" 836 stage3_inst_dir = stage3_dir + "/" + package_name 837 final_stage_dir = stage3_dir 838 if skip_stages < 2: 839 cc = stage2_inst_dir + "/bin/%s%s" % (cc_name, exe_ext) 840 cxx = stage2_inst_dir + "/bin/%s%s" % (cxx_name, exe_ext) 841 asm = stage2_inst_dir + "/bin/%s%s" % (cc_name, exe_ext) 842 build_one_stage( 843 [cc] + extra_cflags2, 844 [cxx] + extra_cxxflags2, 845 [asm] + extra_asmflags, 846 ar, 847 ranlib, 848 libtool, 849 extra_ldflags, 850 llvm_source_dir, 851 stage3_dir, 852 package_name, 853 build_type, 854 assertions, 855 target, 856 targets, 857 is_final_stage=(stages == 3 and not pgo), 858 ) 859 if pgo: 860 del os.environ["LLVM_PROFILE_FILE"] 861 if skip_stages < 1: 862 llvm_profdata = stage1_inst_dir + "/bin/llvm-profdata%s" % exe_ext 863 else: 864 llvm_profdata = get_tool(config, "llvm-profdata") 865 merge_cmd = [llvm_profdata, "merge", "-o", "merged.profdata"] 866 profraw_files = glob.glob(os.path.join(profiles_dir, "*.profraw")) 867 run_in(stage3_dir, merge_cmd + profraw_files) 868 if stages == 3: 869 mkdir_p(upload_dir) 870 shutil.copy2(os.path.join(stage3_dir, "merged.profdata"), upload_dir) 871 return 872 873 if stages >= 4 and skip_stages < 4: 874 stage4_dir = build_dir + "/stage4" 875 final_stage_dir = stage4_dir 876 profile = None 877 if pgo: 878 if skip_stages == 3: 879 profile_dir = os.environ.get("MOZ_FETCHES_DIR", "") 880 else: 881 profile_dir = stage3_dir 882 profile = os.path.join(profile_dir, "merged.profdata") 883 if skip_stages < 3: 884 cc = stage3_inst_dir + "/bin/%s%s" % (cc_name, exe_ext) 885 cxx = stage3_inst_dir + "/bin/%s%s" % (cxx_name, exe_ext) 886 asm = stage3_inst_dir + "/bin/%s%s" % (cc_name, exe_ext) 887 build_one_stage( 888 [cc] + extra_cflags2, 889 [cxx] + extra_cxxflags2, 890 [asm] + extra_asmflags, 891 ar, 892 ranlib, 893 libtool, 894 extra_ldflags, 895 llvm_source_dir, 896 stage4_dir, 897 package_name, 898 build_type, 899 assertions, 900 target, 901 targets, 902 is_final_stage=(stages == 4), 903 profile=profile, 904 bolt=bolt, 905 ) 906 907 if build_clang_tidy: 908 prune_final_dir_for_clang_tidy( 909 os.path.join(final_stage_dir, package_name), target 910 ) 911 912 if not args.skip_tar: 913 build_tar_package("%s.tar.zst" % package_name, final_stage_dir, package_name) 914 915 916 if __name__ == "__main__": 917 main()