repack_rust.py (26251B)
1 #!/usr/bin/env python3 2 # This Source Code Form is subject to the terms of the Mozilla Public 3 # License, v. 2.0. If a copy of the MPL was not distributed with this 4 # file, You can obtain one at http://mozilla.org/MPL/2.0/. 5 6 """ 7 This script downloads and repacks official rust language builds 8 with the necessary tool and target support for the Firefox 9 build environment. 10 """ 11 12 import argparse 13 import errno 14 import hashlib 15 import os 16 import shutil 17 import subprocess 18 import tarfile 19 import tempfile 20 import textwrap 21 from contextlib import contextmanager 22 23 import zstandard 24 25 26 def log(msg): 27 print("repack: %s" % msg, flush=True) 28 29 30 def fetch_file(url): 31 """Download a file from the given url if it's not already present. 32 33 Returns the SHA-2 256-bit hash of the received file.""" 34 import requests 35 36 filename = os.path.basename(url) 37 sha = hashlib.sha256() 38 size = 4096 39 if os.path.exists(filename): 40 with open(filename, "rb") as fd: 41 while True: 42 block = fd.read(size) 43 if not block: 44 return sha.hexdigest() 45 sha.update(block) 46 log("Could not calculate checksum!") 47 return None 48 r = requests.get(url, stream=True) 49 r.raise_for_status() 50 with open(filename, "wb") as fd: 51 for chunk in r.iter_content(size): 52 fd.write(chunk) 53 sha.update(chunk) 54 return sha.hexdigest() 55 56 57 def check_call_with_input(cmd, input_data): 58 """Invoke a command, passing the input String over stdin. 59 60 This is like subprocess.check_call, but allows piping 61 input to interactive commands.""" 62 p = subprocess.Popen(cmd, stdin=subprocess.PIPE) 63 p.communicate(input_data) 64 if p.wait(): 65 raise subprocess.CalledProcessError(p.returncode, cmd) 66 67 68 def setup_gpg(): 69 """Add the signing key to the current gpg config. 70 71 Import a hard-coded copy of the release signing public key 72 and mark it trusted in the gpg database so subsequent 73 signature checks can succeed or fail cleanly.""" 74 keyid = "0x85AB96E6FA1BE5FE" 75 log("Importing signing key %s..." % keyid) 76 key = b""" 77 -----BEGIN PGP PUBLIC KEY BLOCK----- 78 79 mQINBFJEwMkBEADlPACa2K7reD4x5zd8afKx75QYKmxqZwywRbgeICeD4bKiQoJZ 80 dUjmn1LgrGaXuBMKXJQhyA34e/1YZel/8et+HPE5XpljBfNYXWbVocE1UMUTnFU9 81 CKXa4AhJ33f7we2/QmNRMUifw5adPwGMg4D8cDKXk02NdnqQlmFByv0vSaArR5kn 82 gZKnLY6o0zZ9Buyy761Im/ShXqv4ATUgYiFc48z33G4j+BDmn0ryGr1aFdP58tHp 83 gjWtLZs0iWeFNRDYDje6ODyu/MjOyuAWb2pYDH47Xu7XedMZzenH2TLM9yt/hyOV 84 xReDPhvoGkaO8xqHioJMoPQi1gBjuBeewmFyTSPS4deASukhCFOcTsw/enzJagiS 85 ZAq6Imehduke+peAL1z4PuRmzDPO2LPhVS7CDXtuKAYqUV2YakTq8MZUempVhw5n 86 LqVaJ5/XiyOcv405PnkT25eIVVVghxAgyz6bOU/UMjGQYlkUxI7YZ9tdreLlFyPR 87 OUL30E8q/aCd4PGJV24yJ1uit+yS8xjyUiMKm4J7oMP2XdBN98TUfLGw7SKeAxyU 88 92BHlxg7yyPfI4TglsCzoSgEIV6xoGOVRRCYlGzSjUfz0bCMCclhTQRBkegKcjB3 89 sMTyG3SPZbjTlCqrFHy13e6hGl37Nhs8/MvXUysq2cluEISn5bivTKEeeQARAQAB 90 tERSdXN0IExhbmd1YWdlIChUYWcgYW5kIFJlbGVhc2UgU2lnbmluZyBLZXkpIDxy 91 dXN0LWtleUBydXN0LWxhbmcub3JnPokCOAQTAQIAIgUCUkTAyQIbAwYLCQgHAwIG 92 FQgCCQoLBBYCAwECHgECF4AACgkQhauW5vob5f5fYQ//b1DWK1NSGx5nZ3zYZeHJ 93 9mwGCftIaA2IRghAGrNf4Y8DaPqR+w1OdIegWn8kCoGfPfGAVW5XXJg+Oxk6QIaD 94 2hJojBUrq1DALeCZVewzTVw6BN4DGuUexsc53a8DcY2Yk5WE3ll6UKq/YPiWiPNX 95 9r8FE2MJwMABB6mWZLqJeg4RCrriBiCG26NZxGE7RTtPHyppoVxWKAFDiWyNdJ+3 96 UnjldWrT9xFqjqfXWw9Bhz8/EoaGeSSbMIAQDkQQpp1SWpljpgqvctZlc5fHhsG6 97 lmzW5RM4NG8OKvq3UrBihvgzwrIfoEDKpXbk3DXqaSs1o81NH5ftVWWbJp/ywM9Q 98 uMC6n0YWiMZMQ1cFBy7tukpMkd+VPbPkiSwBhPkfZIzUAWd74nanN5SKBtcnymgJ 99 +OJcxfZLiUkXRj0aUT1GLA9/7wnikhJI+RvwRfHBgrssXBKNPOfXGWajtIAmZc2t 100 kR1E8zjBVLId7r5M8g52HKk+J+y5fVgJY91nxG0zf782JjtYuz9+knQd55JLFJCO 101 hhbv3uRvhvkqgauHagR5X9vCMtcvqDseK7LXrRaOdOUDrK/Zg/abi5d+NIyZfEt/ 102 ObFsv3idAIe/zpU6xa1nYNe3+Ixlb6mlZm3WCWGxWe+GvNW/kq36jZ/v/8pYMyVO 103 p/kJqnf9y4dbufuYBg+RLqC5Ag0EUkTAyQEQANxy2tTSeRspfrpBk9+ju+KZ3zc4 104 umaIsEa5DxJ2zIKHywVAR67Um0K1YRG07/F5+tD9TIRkdx2pcmpjmSQzqdk3zqa9 105 2Zzeijjz2RNyBY8qYmyE08IncjTsFFB8OnvdXcsAgjCFmI1BKnePxrABL/2k8X18 106 aysPb0beWqQVsi5FsSpAHu6k1kaLKc+130x6Hf/YJAjeo+S7HeU5NeOz3zD+h5bA 107 Q25qMiVHX3FwH7rFKZtFFog9Ogjzi0TkDKKxoeFKyADfIdteJWFjOlCI9KoIhfXq 108 Et9JMnxApGqsJElJtfQjIdhMN4Lnep2WkudHAfwJ/412fe7wiW0rcBMvr/BlBGRY 109 vM4sTgN058EwIuY9Qmc8RK4gbBf6GsfGNJjWozJ5XmXElmkQCAvbQFoAfi5TGfVb 110 77QQrhrQlSpfIYrvfpvjYoqj618SbU6uBhzh758gLllmMB8LOhxWtq9eyn1rMWyR 111 KL1fEkfvvMc78zP+Px6yDMa6UIez8jZXQ87Zou9EriLbzF4QfIYAqR9LUSMnLk6K 112 o61tSFmFEDobC3tc1jkSg4zZe/wxskn96KOlmnxgMGO0vJ7ASrynoxEnQE8k3WwA 113 +/YJDwboIR7zDwTy3Jw3mn1FgnH+c7Rb9h9geOzxKYINBFz5Hd0MKx7kZ1U6WobW 114 KiYYxcCmoEeguSPHABEBAAGJAh8EGAECAAkFAlJEwMkCGwwACgkQhauW5vob5f7f 115 FA//Ra+itJF4NsEyyhx4xYDOPq4uj0VWVjLdabDvFjQtbBLwIyh2bm8uO3AY4r/r 116 rM5WWQ8oIXQ2vvXpAQO9g8iNlFez6OLzbfdSG80AG74pQqVVVyCQxD7FanB/KGge 117 tAoOstFxaCAg4nxFlarMctFqOOXCFkylWl504JVIOvgbbbyj6I7qCUmbmqazBSMU 118 K8c/Nz+FNu2Uf/lYWOeGogRSBgS0CVBcbmPUpnDHLxZWNXDWQOCxbhA1Uf58hcyu 119 036kkiWHh2OGgJqlo2WIraPXx1cGw1Ey+U6exbtrZfE5kM9pZzRG7ZY83CXpYWMp 120 kyVXNWmf9JcIWWBrXvJmMi0FDvtgg3Pt1tnoxqdilk6yhieFc8LqBn6CZgFUBk0t 121 NSaWk3PsN0N6Ut8VXY6sai7MJ0Gih1gE1xadWj2zfZ9sLGyt2jZ6wK++U881YeXA 122 ryaGKJ8sIs182hwQb4qN7eiUHzLtIh8oVBHo8Q4BJSat88E5/gOD6IQIpxc42iRL 123 T+oNZw1hdwNyPOT1GMkkn86l3o7klwmQUWCPm6vl1aHp3omo+GHC63PpNFO5RncJ 124 Ilo3aBKKmoE5lDSMGE8KFso5awTo9z9QnVPkRsk6qeBYit9xE3x3S+iwjcSg0nie 125 aAkc0N00nc9V9jfPvt4z/5A5vjHh+NhFwH5h2vBJVPdsz6m5Ag0EVI9keAEQAL3R 126 oVsHncJTmjHfBOV4JJsvCum4DuJDZ/rDdxauGcjMUWZaG338ZehnDqG1Yn/ys7zE 127 aKYUmqyT+XP+M2IAQRTyxwlU1RsDlemQfWrESfZQCCmbnFScL0E7cBzy4xvtInQe 128 UaFgJZ1BmxbzQrx+eBBdOTDv7RLnNVygRmMzmkDhxO1IGEu1+3ETIg/DxFE7VQY0 129 It/Ywz+nHu1o4Hemc/GdKxu9hcYvcRVc/Xhueq/zcIM96l0m+CFbs0HMKCj8dgMe 130 Ng6pbbDjNM+cV+5BgpRdIpE2l9W7ImpbLihqcZt47J6oWt/RDRVoKOzRxjhULVyV 131 2VP9ESr48HnbvxcpvUAEDCQUhsGpur4EKHFJ9AmQ4zf91gWLrDc6QmlACn9o9ARU 132 fOV5aFsZI9ni1MJEInJTP37stz/uDECRie4LTL4O6P4Dkto8ROM2wzZq5CiRNfnT 133 PP7ARfxlCkpg+gpLYRlxGUvRn6EeYwDtiMQJUQPfpGHSvThUlgDEsDrpp4SQSmdA 134 CB+rvaRqCawWKoXs0In/9wylGorRUupeqGC0I0/rh+f5mayFvORzwy/4KK4QIEV9 135 aYTXTvSRl35MevfXU1Cumlaqle6SDkLr3ZnFQgJBqap0Y+Nmmz2HfO/pohsbtHPX 136 92SN3dKqaoSBvzNGY5WT3CsqxDtik37kR3f9/DHpABEBAAGJBD4EGAECAAkFAlSP 137 ZHgCGwICKQkQhauW5vob5f7BXSAEGQECAAYFAlSPZHgACgkQXLSpNHs7CdwemA/+ 138 KFoGuFqU0uKT9qblN4ugRyil5itmTRVffl4tm5OoWkW8uDnu7Ue3vzdzy+9NV8X2 139 wRG835qjXijWP++AGuxgW6LB9nV5OWiKMCHOWnUjJQ6pNQMAgSN69QzkFXVF/q5f 140 bkma9TgSbwjrVMyPzLSRwq7HsT3V02Qfr4cyq39QeILGy/NHW5z6LZnBy3BaVSd0 141 lGjCEc3yfH5OaB79na4W86WCV5n4IT7cojFM+LdL6P46RgmEtWSG3/CDjnJl6BLR 142 WqatRNBWLIMKMpn+YvOOL9TwuP1xbqWr1vZ66wksm53NIDcWhptpp0KEuzbU0/Dt 143 OltBhcX8tOmO36LrSadX9rwckSETCVYklmpAHNxPml011YNDThtBidvsicw1vZwR 144 HsXn+txlL6RAIRN+J/Rw3uOiJAqN9Qgedpx2q+E15t8MiTg/FXtB9SysnskFT/BH 145 z0USNKJUY0btZBw3eXWzUnZf59D8VW1M/9JwznCHAx0c9wy/gRDiwt9w4RoXryJD 146 VAwZg8rwByjldoiThUJhkCYvJ0R3xH3kPnPlGXDW49E9R8C2umRC3cYOL4U9dOQ1 147 5hSlYydF5urFGCLIvodtE9q80uhpyt8L/5jj9tbwZWv6JLnfBquZSnCGqFZRfXlb 148 Jphk9+CBQWwiZSRLZRzqQ4ffl4xyLuolx01PMaatkQbRaw/+JpgRNlurKQ0PsTrO 149 8tztO/tpBBj/huc2DGkSwEWvkfWElS5RLDKdoMVs/j5CLYUJzZVikUJRm7m7b+OA 150 P3W1nbDhuID+XV1CSBmGifQwpoPTys21stTIGLgznJrIfE5moFviOLqD/LrcYlsq 151 CQg0yleu7SjOs//8dM3mC2FyLaE/dCZ8l2DCLhHw0+ynyRAvSK6aGCmZz6jMjmYF 152 MXgiy7zESksMnVFMulIJJhR3eB0wx2GitibjY/ZhQ7tD3i0yy9ILR07dFz4pgkVM 153 afxpVR7fmrMZ0t+yENd+9qzyAZs0ksxORoc2ze90SCx2jwEX/3K+m4I0hP2H/w5W 154 gqdvuRLiqf+4BGW4zqWkLLlNIe/okt0r82SwHtDN0Ui1asmZTGj6sm8SXtwx+5cE 155 38MttWqjDiibQOSthRVcETByRYM8KcjYSUCi4PoBc3NpDONkFbZm6XofR/f5mTcl 156 2jDw6fIeVc4Hd1jBGajNzEqtneqqbdAkPQaLsuD2TMkQfTDJfE/IljwjrhDa9Mi+ 157 odtnMWq8vlwOZZ24/8/BNK5qXuCYL67O7AJB4ZQ6BT+g4z96iRLbupzu/XJyXkQF 158 rOY/Ghegvn7fDrnt2KC9MpgeFBXzUp+k5rzUdF8jbCx5apVjA1sWXB9Kh3L+DUwF 159 Mve696B5tlHyc1KxjHR6w9GRsh4= 160 =5FXw 161 -----END PGP PUBLIC KEY BLOCK----- 162 """ 163 check_call_with_input(["gpg", "--import"], key) 164 check_call_with_input( 165 ["gpg", "--command-fd", "0", "--edit-key", keyid], b"trust\n5\ny\n" 166 ) 167 168 169 def verify_sha(filename, sha): 170 """Verify that the checksum file matches the given sha digest.""" 171 sha_filename = filename + ".sha256" 172 with open(sha_filename) as f: 173 # Older sha256 files would contain `sha filename`, but more recent 174 # ones only contain `sha`. 175 checksum = f.readline().split()[0] 176 if checksum != sha: 177 raise ValueError("Checksum mismatch in %s" % filename) 178 return True 179 log("No checksum file for %s!" % filename) 180 return False 181 182 183 def fetch(url, validate=True): 184 """Download and verify a package url.""" 185 base = os.path.basename(url) 186 log("Fetching %s..." % base) 187 if validate: 188 fetch_file(url + ".asc") 189 fetch_file(url + ".sha256") 190 sha = fetch_file(url) 191 if validate: 192 log("Verifying %s..." % base) 193 verify_sha(base, sha) 194 subprocess.check_call([ 195 "gpg", 196 "--keyid-format", 197 "0xlong", 198 "--verify", 199 base + ".asc", 200 base, 201 ]) 202 return sha 203 204 205 def install(filename, target): 206 """Run a package's installer script against the given target directory.""" 207 log("Unpacking %s..." % filename) 208 subprocess.check_call(["tar", "xf", filename]) 209 basename = filename.split(".tar")[0] 210 log("Installing %s..." % basename) 211 install_cmd = [os.path.join(basename, "install.sh")] 212 install_cmd += ["--prefix=" + os.path.abspath(target)] 213 install_cmd += ["--disable-ldconfig"] 214 subprocess.check_call(install_cmd) 215 log("Cleaning %s..." % basename) 216 shutil.rmtree(basename) 217 218 219 def package(manifest, pkg, target): 220 """Pull out the package dict for a particular package and target 221 from the given manifest.""" 222 version = manifest["pkg"][pkg]["version"] 223 if target in manifest["pkg"][pkg]["target"]: 224 info = manifest["pkg"][pkg]["target"][target] 225 else: 226 # rust-src is the same for all targets, and has a literal '*' in the 227 # section key/name instead of a target 228 info = manifest["pkg"][pkg]["target"]["*"] 229 if "xz_url" in info: 230 info["url"] = info.pop("xz_url") 231 info["hash"] = info.pop("xz_hash") 232 return (version, info) 233 234 235 def fetch_package(manifest, pkg, host): 236 version, info = package(manifest, pkg, host) 237 if not info["available"]: 238 log("%s marked unavailable for %s" % (pkg, host)) 239 raise KeyError 240 241 log("%s %s\n %s\n %s" % (pkg, version, info["url"], info["hash"])) 242 sha = fetch(info["url"], info["hash"] is not None) 243 if info["hash"] and sha != info["hash"]: 244 log( 245 "Checksum mismatch: package resource is different from manifest\n %s" % sha 246 ) 247 raise AssertionError 248 return info 249 250 251 def fetch_std(manifest, targets): 252 stds = [] 253 for target in targets: 254 stds.append(fetch_package(manifest, "rust-std", target)) 255 analysis = fetch_optional(manifest, "rust-analysis", target) 256 if analysis: 257 stds.append(analysis) 258 else: 259 log(f"Missing rust-analysis for {target}") 260 # If it's missing for one of the searchfox targets, explicitly 261 # error out. 262 if target in ( 263 "x86_64-unknown-linux-gnu", 264 "x86_64-apple-darwin", 265 "x86_64-pc-windows-msvc", 266 "thumbv7neon-linux-androideabi", 267 ): 268 raise AssertionError 269 270 return stds 271 272 273 def fetch_optional(manifest, pkg, host): 274 try: 275 return fetch_package(manifest, pkg, host) 276 except KeyError: 277 # The package is not available, oh well! 278 return None 279 280 281 @contextmanager 282 def chdir(path): 283 d = os.getcwd() 284 log('cd "%s"' % path) 285 os.chdir(path) 286 try: 287 yield 288 finally: 289 log('cd "%s"' % d) 290 os.chdir(d) 291 292 293 def build_tar_package(name, base, directory): 294 name = os.path.realpath(name) 295 log(f"tarring {name} from {base}/{directory}") 296 assert name.endswith(".tar.zst") 297 298 cctx = zstandard.ZstdCompressor() 299 with open(name, "wb") as f, cctx.stream_writer(f) as z: 300 with tarfile.open(mode="w|", fileobj=z) as tf: 301 with chdir(base): 302 tf.add(directory) 303 304 305 def fetch_manifest(channel="stable", host=None, targets=()): 306 import requests 307 import toml 308 309 if channel.startswith("bors-"): 310 assert host 311 rev = channel[len("bors-") :] 312 base_url = "https://s3-us-west-1.amazonaws.com/rust-lang-ci2/rustc-builds" 313 manifest = { 314 "date": "some date", 315 "pkg": {}, 316 } 317 318 def target(url): 319 return { 320 "url": url, 321 "hash": None, 322 "available": requests.head(url).status_code == 200, 323 } 324 325 for pkg in ( 326 "cargo", 327 "rustc", 328 "rustfmt-preview", 329 "clippy-preview", 330 "rust-analyzer-preview", 331 ): 332 manifest["pkg"][pkg] = { 333 "version": "bors", 334 "target": { 335 host: target(f"{base_url}/{rev}/{pkg}-nightly-{host}.tar.xz"), 336 }, 337 } 338 manifest["pkg"]["rust-src"] = { 339 "version": "bors", 340 "target": { 341 "*": target(f"{base_url}/{rev}/rust-src-nightly.tar.xz"), 342 }, 343 } 344 for pkg in ("rust-std", "rust-analysis"): 345 manifest["pkg"][pkg] = { 346 "version": "bors", 347 "target": { 348 t: target(f"{base_url}/{rev}/{pkg}-nightly-{t}.tar.xz") 349 for t in sorted(set(targets) | set([host])) 350 }, 351 } 352 return manifest 353 if channel.startswith(("beta-", "nightly-")): 354 channel, date = channel.split("-", 1) 355 prefix = "/" + date 356 else: 357 prefix = "" 358 url = "https://static.rust-lang.org/dist%s/channel-rust-%s.toml" % (prefix, channel) 359 req = requests.get(url) 360 req.raise_for_status() 361 manifest = toml.loads(req.text) 362 if manifest["manifest-version"] != "2": 363 raise NotImplementedError( 364 "Unrecognized manifest version %s." % manifest["manifest-version"] 365 ) 366 return manifest 367 368 369 def patch_src(patch, module): 370 log(f"Patching Rust src... {module} with {patch}") 371 patch = os.path.realpath(patch) 372 subprocess.check_call(["patch", "-d", module, "-p1", "-i", patch, "--fuzz=0", "-s"]) 373 374 375 def build_src(install_dir, host, targets, patches): 376 install_dir = os.path.abspath(install_dir) 377 fetches = os.environ["MOZ_FETCHES_DIR"] 378 rust_dir = os.path.join(fetches, "rust") 379 patch_dir = os.path.join(os.environ["GECKO_PATH"], "build", "build-rust") 380 381 # Clear and remake any previous install directory. 382 try: 383 shutil.rmtree(install_dir) 384 except OSError as e: 385 if e.errno != errno.ENOENT: 386 raise 387 os.makedirs(install_dir) 388 389 # Patch the src (see the --patch flag's description for details) 390 for p in patches: 391 module, colon, file = p.partition(":") 392 if not colon: 393 module, file = "", p 394 patch_file = os.path.join(patch_dir, file) 395 patch_module = os.path.join(rust_dir, module) 396 patch_src(patch_file, patch_module) 397 398 log("Building Rust...") 399 400 example_config = "" 401 for example_toml in ("config.example.toml", "config.toml.example"): 402 path = os.path.join(rust_dir, example_toml) 403 if os.path.exists(path): 404 with open(path) as file: 405 example_config = file.read() 406 break 407 408 if "ignore-git" in example_config: 409 omit_git_hash = "ignore-git" 410 else: 411 omit_git_hash = "omit-git-hash" 412 413 # Rust builds are configured primarily through a config.toml file. 414 # 415 # `sysconfdir` is overloaded to be relative instead of absolute. 416 # This is the default of `install.sh`, but for whatever reason 417 # `x.py install` has its own default of `/etc` which we don't want. 418 base_config = textwrap.dedent( 419 f""" 420 [build] 421 docs = false 422 sanitizers = true 423 profiler = true 424 extended = true 425 tools = ["analysis", "cargo", "rustdoc", "rustfmt", "clippy", "src", "rust-analyzer"] 426 cargo-native-static = true 427 428 [rust] 429 {omit_git_hash} = false 430 use-lld = true 431 432 [install] 433 prefix = "/" 434 435 [llvm] 436 download-ci-llvm = false 437 """ 438 ) 439 if "msvc" in host: 440 assert all("msvc" in target for target in targets) 441 base_config += textwrap.dedent( 442 f""" 443 ldflags = "-winsysroot:{fetches}/vs" 444 445 [llvm.build-config] 446 CMAKE_MT = "llvm-mt.exe" 447 CMAKE_ASM_MASM_COMPILER = "ml64.exe" 448 """ 449 ) 450 target_config = textwrap.dedent( 451 """ 452 [target.{target}] 453 cc = "clang-cl.bat" 454 cxx = "clang-cl.bat" 455 linker = "lld-link.bat" 456 ar = "llvm-lib" 457 458 """ 459 ) 460 else: 461 assert all("msvc" not in target for target in targets) 462 # Rust requires these to be specified per-target 463 target_config = textwrap.dedent( 464 """ 465 [target.{target}] 466 cc = "clang" 467 cxx = "clang++" 468 linker = "clang" 469 470 """ 471 ) 472 473 final_config = base_config 474 for target in sorted(set(targets) | set([host])): 475 final_config = final_config + target_config.format(target=target) 476 477 with open(os.path.join(rust_dir, "config.toml"), "w") as file: 478 file.write(final_config) 479 480 # Setup the env so compilers and toolchains are visible 481 clang = os.path.join(fetches, "clang") 482 clang_bin = os.path.join(clang, "bin") 483 clang_lib = os.path.join(clang, "lib") 484 sysroot = os.path.join(fetches, "sysroot") 485 486 with tempfile.TemporaryDirectory() as tmpdir: 487 # The rust build doesn't offer much in terms of overriding compiler flags 488 # when it builds LLVM's compiler-rt, but we want to build with a sysroot. 489 # So, we create wrappers for clang and clang++ that add the sysroot to the 490 # command line. 491 for exe in ("clang", "clang++"): 492 tmp_exe = os.path.join(tmpdir, exe) 493 with open(tmp_exe, "w") as fh: 494 fh.write("#!/bin/sh\n") 495 fh.write(f'exec {clang_bin}/{exe} --sysroot={sysroot} "$@"\n') 496 os.chmod(tmp_exe, 0o755) 497 498 if "msvc" in host: 499 # Same thing for the linker on windows, where we want to add the MSVC 500 # sysroot 501 with open(os.path.join(tmpdir, "lld-link.bat"), "w") as fh: 502 fh.write("@echo off\n") 503 fh.write(f"{clang_bin}/lld-link.exe -winsysroot:{fetches}/vs %*") 504 # And clang-cl. 505 with open(os.path.join(tmpdir, "clang-cl.bat"), "w") as fh: 506 fh.write("@echo off\n") 507 fh.write( 508 f"{clang_bin}/clang-cl.exe -Xclang -ivfsoverlay -Xclang {fetches}/vs/overlay.yaml -winsysroot {fetches}/vs %*" 509 ) 510 # cc-rs expects to find ml64.exe to build asm files, without a way to override it. 511 shutil.copy(f"{clang_bin}/llvm-ml.exe", f"{tmpdir}/ml64.exe") 512 513 env = os.environ.copy() 514 env.update({ 515 "PATH": os.pathsep.join((tmpdir, clang_bin, os.environ["PATH"])), 516 "LD_LIBRARY_PATH": clang_lib, 517 "DESTDIR": install_dir, 518 }) 519 if "msvc" in host: 520 cmake_bin = os.path.join(fetches, "cmake", "bin") 521 ninja_bin = os.path.join(fetches, "ninja", "bin") 522 env.update({ 523 "PATH": os.pathsep.join((env["PATH"], cmake_bin, ninja_bin)), 524 "CC_x86_64_pc_windows_msvc": "clang-cl.bat", 525 "CXX_x86_64_pc_windows_msvc": "clang-cl.bat", 526 "AR_x86_64_pc_windows_msvc": "llvm-lib", 527 }) 528 529 # x.py install does everything we need for us. 530 # If you're running into issues, consider using `-vv` to debug it. 531 command = ["python3", "x.py", "install", "-v", "--host", host, "--build", host] 532 for target in targets: 533 command.extend(["--target", target]) 534 535 subprocess.check_call(command, stderr=subprocess.STDOUT, env=env, cwd=rust_dir) 536 537 538 def repack( 539 host, 540 targets, 541 channel="stable", 542 cargo_channel=None, 543 patches=[], 544 ): 545 install_dir = "rustc" 546 if channel == "dev": 547 build_src(install_dir, host, targets, patches) 548 else: 549 if patches: 550 raise ValueError( 551 'Patch specified, but channel "%s" is not "dev"!' 552 "\nPatches are only for building from source." % channel 553 ) 554 setup_gpg() 555 log("Repacking rust for %s supporting %s..." % (host, targets)) 556 manifest = fetch_manifest(channel, host, targets) 557 log("Using manifest for rust %s as of %s." % (channel, manifest["date"])) 558 if cargo_channel == channel: 559 cargo_manifest = manifest 560 else: 561 cargo_manifest = fetch_manifest(cargo_channel, host, targets) 562 log( 563 "Using manifest for cargo %s as of %s." 564 % (cargo_channel, cargo_manifest["date"]) 565 ) 566 567 log("Fetching packages...") 568 rustc = fetch_package(manifest, "rustc", host) 569 cargo = fetch_package(cargo_manifest, "cargo", host) 570 stds = fetch_std(manifest, targets) 571 rustsrc = fetch_package(manifest, "rust-src", host) 572 rustfmt = fetch_optional(manifest, "rustfmt-preview", host) 573 clippy = fetch_optional(manifest, "clippy-preview", host) 574 rust_analyzer = fetch_optional(manifest, "rust-analyzer-preview", host) 575 576 log("Installing packages...") 577 578 # Clear any previous install directory. 579 try: 580 shutil.rmtree(install_dir) 581 except OSError as e: 582 if e.errno != errno.ENOENT: 583 raise 584 install(os.path.basename(rustc["url"]), install_dir) 585 install(os.path.basename(cargo["url"]), install_dir) 586 install(os.path.basename(rustsrc["url"]), install_dir) 587 if rustfmt: 588 install(os.path.basename(rustfmt["url"]), install_dir) 589 if clippy: 590 install(os.path.basename(clippy["url"]), install_dir) 591 if rust_analyzer: 592 install(os.path.basename(rust_analyzer["url"]), install_dir) 593 for std in stds: 594 install(os.path.basename(std["url"]), install_dir) 595 pass 596 597 log("Creating archive...") 598 tar_file = install_dir + ".tar.zst" 599 build_tar_package(tar_file, ".", install_dir) 600 shutil.rmtree(install_dir) 601 log("%s is ready." % tar_file) 602 603 upload_dir = os.environ.get("UPLOAD_DIR") 604 if upload_dir: 605 # Create the upload directory if it doesn't exist. 606 try: 607 log("Creating upload directory in %s..." % os.path.abspath(upload_dir)) 608 os.makedirs(upload_dir) 609 except OSError as e: 610 if e.errno != errno.EEXIST: 611 raise 612 # Move the tarball to the output directory for upload. 613 log("Moving %s to the upload directory..." % tar_file) 614 shutil.move(tar_file, upload_dir) 615 616 617 def expand_platform(name): 618 """Expand a shortcut name to a full Rust platform string.""" 619 platforms = { 620 "android": "armv7-linux-androideabi", 621 "android_x86": "i686-linux-android", 622 "android_x86-64": "x86_64-linux-android", 623 "android_aarch64": "aarch64-linux-android", 624 "linux64": "x86_64-unknown-linux-gnu", 625 "mac": "x86_64-apple-darwin", 626 "macos": "x86_64-apple-darwin", 627 "mac64": "x86_64-apple-darwin", 628 "mac32": "i686-apple-darwin", 629 "win64": "x86_64-pc-windows-msvc", 630 "win32": "i686-pc-windows-msvc", 631 "mingw32": "i686-pc-windows-gnu", 632 } 633 return platforms.get(name, name) 634 635 636 def validate_channel(channel): 637 """Require a specific release version. 638 639 Packaging from meta-channels, like `stable`, `beta`, or `nightly` 640 doesn't give repeatable output. Reject such channels.""" 641 channel_prefixes = ("stable", "beta", "nightly") 642 if any([channel.startswith(c) for c in channel_prefixes]): 643 if "-" not in channel: 644 raise ValueError( 645 'Generic channel "%s" specified!' 646 "\nPlease give a specific release version" 647 ' like "1.24.0" or "beta-2018-02-20".' % channel 648 ) 649 650 651 def args(): 652 """Read command line arguments and return options.""" 653 parser = argparse.ArgumentParser() 654 parser.add_argument( 655 "--channel", 656 help="Release channel to use:" 657 " 1.xx.y, beta-yyyy-mm-dd," 658 " nightly-yyyy-mm-dd," 659 " bors-$rev (grab a build from rust's CI)," 660 " or dev (build from source).", 661 required=True, 662 ) 663 parser.add_argument( 664 "--allow-generic-channel", 665 action="store_true", 666 help='Allow to use e.g. "nightly" without a date as a channel.', 667 ) 668 parser.add_argument( 669 "--patch", 670 dest="patches", 671 action="append", 672 default=[], 673 help="apply the given patch file to a dev build." 674 " Patch files should be placed in /build/build-rust." 675 " Patches can be prefixed with `module-path:` to specify they" 676 " apply to that git submodule in the Rust source." 677 " e.g. `src/llvm-project:mypatch.diff` patches rust's llvm." 678 " Can be given more than once.", 679 ) 680 parser.add_argument( 681 "--cargo-channel", 682 help="Release channel version to use for cargo." 683 " Defaults to the same as --channel.", 684 ) 685 parser.add_argument( 686 "--host", 687 help="Host platform for the toolchain executable:" 688 " e.g. linux64 or aarch64-linux-android." 689 " Defaults to linux64.", 690 ) 691 parser.add_argument( 692 "--target", 693 dest="targets", 694 action="append", 695 default=[], 696 help="Additional target platform to support:" 697 " e.g. linux64 or i686-pc-windows-gnu." 698 " can be given more than once.", 699 ) 700 args = parser.parse_args() 701 if not args.cargo_channel: 702 args.cargo_channel = args.channel 703 if not args.allow_generic_channel: 704 validate_channel(args.channel) 705 validate_channel(args.cargo_channel) 706 if not args.host: 707 args.host = "linux64" 708 args.host = expand_platform(args.host) 709 args.targets = [expand_platform(t) for t in args.targets] 710 delattr(args, "allow_generic_channel") 711 712 return args 713 714 715 if __name__ == "__main__": 716 args = vars(args()) 717 repack(**args)