bootstrap.py (16009B)
1 #!/usr/bin/env python3 2 # This Source Code Form is subject to the terms of the Mozilla Public 3 # License, v. 2.0. If a copy of the MPL was not distributed with this file, 4 # You can obtain one at http://mozilla.org/MPL/2.0/. 5 6 # This script provides one-line bootstrap support to configure systems to build 7 # the tree. It does so by cloning the repo before calling directly into `mach 8 # bootstrap`. 9 10 # Note that this script can't assume anything in particular about the host 11 # Python environment (except that it's run with a sufficiently recent version of 12 # Python 3), so we are restricted to stdlib modules. 13 14 import sys 15 16 MINIMUM_MINOR_VERSION = 9 17 18 major, minor = sys.version_info[:2] 19 if (major < 3) or (major == 3 and minor < MINIMUM_MINOR_VERSION): 20 print( 21 f"Bootstrap currently only runs on Python 3.{MINIMUM_MINOR_VERSION}+." 22 f"Please try re-running with python3.{MINIMUM_MINOR_VERSION}+." 23 ) 24 sys.exit(1) 25 26 import ctypes 27 import os 28 import shutil 29 import subprocess 30 import tempfile 31 from optparse import OptionParser 32 from pathlib import Path 33 34 CLONE_MERCURIAL_PULL_FAIL = """ 35 Failed to pull from hg.mozilla.org. 36 37 This is most likely because of unstable network connection. 38 Try running `cd %s && hg pull https://hg.mozilla.org/mozilla-unified` manually, 39 or download a mercurial bundle and use it: 40 https://firefox-source-docs.mozilla.org/contributing/vcs/mercurial_bundles.html""" 41 42 WINDOWS = sys.platform.startswith("win32") or sys.platform.startswith("msys") 43 VCS_HUMAN_READABLE = { 44 "hg": "Mercurial", 45 "git": "Git", 46 "git-cinnabar": "Git", 47 } 48 GIT_REPO = "https://github.com/mozilla-firefox/firefox" 49 HG_REPO = "https://hg.mozilla.org/mozilla-unified" 50 51 52 def which(name): 53 """Python implementation of which. 54 55 It returns the path of an executable or None if it couldn't be found. 56 """ 57 search_dirs = os.environ["PATH"].split(os.pathsep) 58 potential_names = [name] 59 if WINDOWS: 60 potential_names.insert(0, name + ".exe") 61 62 for path in search_dirs: 63 for executable_name in potential_names: 64 test = Path(path) / executable_name 65 if test.is_file() and os.access(test, os.X_OK): 66 return test 67 68 return None 69 70 71 def validate_clone_dest(dest: Path): 72 dest = dest.resolve() 73 74 if not dest.exists(): 75 return dest 76 77 if not dest.is_dir(): 78 print(f"ERROR! Destination {dest} exists but is not a directory.") 79 return None 80 81 if not any(dest.iterdir()): 82 return dest 83 else: 84 print(f"ERROR! Destination directory {dest} exists but is nonempty.") 85 print( 86 f"To re-bootstrap the existing checkout, go into '{dest}' and run './mach bootstrap'." 87 ) 88 return None 89 90 91 def input_clone_dest(vcs, no_interactive): 92 repo = GIT_REPO if vcs == "git" else HG_REPO 93 repo_name = repo.rpartition("/")[2] 94 print(f"Cloning into {repo_name} using {VCS_HUMAN_READABLE[vcs]}...") 95 while True: 96 dest = None 97 if not no_interactive: 98 dest = input( 99 f"Destination directory for clone (leave empty to use " 100 f"default destination of {repo_name}): " 101 ).strip() 102 if not dest: 103 dest = repo_name 104 dest = validate_clone_dest(Path(dest).expanduser()) 105 if dest: 106 return dest 107 if no_interactive: 108 return None 109 110 111 def hg_clone_firefox(hg: Path, dest: Path, head_repo, head_rev): 112 # We create an empty repo then modify the config before adding data. 113 # This is necessary to ensure storage settings are optimally 114 # configured. 115 args = [ 116 str(hg), 117 # The unified repo is generaldelta, so ensure the client is as 118 # well. 119 "--config", 120 "format.generaldelta=true", 121 "init", 122 str(dest), 123 ] 124 res = subprocess.call(args) 125 if res: 126 print("unable to create destination repo; please try cloning manually") 127 return None 128 129 # Strictly speaking, this could overwrite a config based on a template 130 # the user has installed. Let's pretend this problem doesn't exist 131 # unless someone complains about it. 132 with open(dest / ".hg" / "hgrc", "a") as fh: 133 fh.write("[paths]\n") 134 fh.write(f"default = {HG_REPO}\n") 135 fh.write("\n") 136 137 # The server uses aggressivemergedeltas which can blow up delta chain 138 # length. This can cause performance to tank due to delta chains being 139 # too long. Limit the delta chain length to something reasonable 140 # to bound revlog read time. 141 fh.write("[format]\n") 142 fh.write("# This is necessary to keep performance in check\n") 143 fh.write("maxchainlen = 10000\n") 144 145 # Pulling a specific revision into an empty repository induces a lot of 146 # load on the Mercurial server, so we always pull from mozilla-unified (which, 147 # when done from an empty repository, is equivalent to a clone), and then pull 148 # the specific revision we want (if we want a specific one, otherwise we just 149 # use the "central" bookmark), at which point it will be an incremental pull, 150 # that the server can process more easily. 151 # This is the same thing that robustcheckout does on automation. 152 res = subprocess.call([str(hg), "pull", HG_REPO], cwd=str(dest)) 153 if not res and head_repo: 154 res = subprocess.call( 155 [str(hg), "pull", head_repo, "-r", head_rev], cwd=str(dest) 156 ) 157 print("") 158 if res: 159 print(CLONE_MERCURIAL_PULL_FAIL % dest) 160 return None 161 162 head_rev = head_rev or "central" 163 print(f'updating to "{head_rev}" - the development head of Gecko and Firefox') 164 res = subprocess.call([str(hg), "update", "-r", head_rev], cwd=str(dest)) 165 if res: 166 print( 167 f"error updating; you will need to `cd {dest} && hg update -r central` " 168 "manually" 169 ) 170 return dest 171 172 173 def git_clone_firefox(git: Path, dest: Path, head_repo, head_rev): 174 if head_repo and "hg.mozilla.org" in head_repo: 175 print("GECKO_HEAD_REPOSITORY cannot be a Mercurial repository when using Git") 176 return None 177 178 subprocess.check_call( 179 [ 180 str(git), 181 "clone", 182 "-n", 183 GIT_REPO, 184 str(dest), 185 ], 186 ) 187 subprocess.check_call([str(git), "config", "pull.ff", "only"], cwd=str(dest)) 188 189 if head_repo: 190 subprocess.check_call( 191 [str(git), "fetch", head_repo, head_rev], 192 cwd=str(dest), 193 ) 194 195 subprocess.check_call( 196 [ 197 str(git), 198 "checkout", 199 "FETCH_HEAD" if head_rev else "main", 200 "--", 201 ], 202 cwd=str(dest), 203 ) 204 205 return dest 206 207 208 def git_cinnabar_clone_firefox(git: Path, dest: Path, head_repo, head_rev): 209 tempdir = None 210 cinnabar = None 211 env = dict(os.environ) 212 try: 213 cinnabar = which("git-cinnabar") 214 if not cinnabar: 215 from urllib.request import urlopen 216 217 cinnabar_url = "https://github.com/glandium/git-cinnabar/" 218 # If git-cinnabar isn't installed already, that's fine; we can 219 # download a temporary copy. `mach bootstrap` will install a copy 220 # in the state dir; we don't want to copy all that logic to this 221 # tiny bootstrapping script. 222 tempdir = Path(tempfile.mkdtemp()) 223 with open(tempdir / "download.py", "wb") as fh: 224 shutil.copyfileobj( 225 urlopen(f"{cinnabar_url}/raw/master/download.py"), fh 226 ) 227 228 subprocess.check_call( 229 [sys.executable, str(tempdir / "download.py")], 230 cwd=str(tempdir), 231 ) 232 env["PATH"] = str(tempdir) + os.pathsep + env["PATH"] 233 print( 234 "WARNING! git-cinnabar is required for Firefox development " 235 "with git. After the clone is complete, the bootstrapper " 236 "will ask if you would like to configure git; answer yes, " 237 "and be sure to add git-cinnabar to your PATH according to " 238 "the bootstrapper output." 239 ) 240 241 # We're guaranteed to have `git-cinnabar` installed now. 242 # Configure git per the git-cinnabar requirements. 243 subprocess.check_call( 244 [ 245 str(git), 246 "-c", 247 "fetch.prune=true", 248 "-c", 249 f"cinnabar.graft={GIT_REPO}", 250 "-c", 251 "cinnabar.refs=bookmarks", 252 "-c", 253 "remote.origin.fetch=refs/heads/central:refs/remotes/origin/main", 254 "clone", 255 "--no-checkout", 256 f"hg::{HG_REPO}", 257 str(dest), 258 ], 259 env=env, 260 ) 261 subprocess.check_call( 262 [str(git), "config", "fetch.prune", "true"], cwd=str(dest), env=env 263 ) 264 subprocess.check_call( 265 [str(git), "config", "cinnabar.refs", "bookmarks"], cwd=str(dest), env=env 266 ) 267 subprocess.check_call( 268 [ 269 str(git), 270 "config", 271 "--add", 272 "remote.origin.fetch", 273 "refs/heads/central:refs/remotes/origin/main", 274 ], 275 cwd=str(dest), 276 env=env, 277 ) 278 subprocess.check_call( 279 [str(git), "config", "pull.ff", "only"], cwd=str(dest), env=env 280 ) 281 282 if head_repo: 283 subprocess.check_call( 284 [str(git), "cinnabar", "fetch", f"hg::{head_repo}", head_rev], 285 cwd=str(dest), 286 env=env, 287 ) 288 289 subprocess.check_call( 290 [ 291 str(git), 292 "checkout", 293 "FETCH_HEAD" if head_rev else "main", 294 "--", 295 ], 296 cwd=str(dest), 297 env=env, 298 ) 299 300 return dest 301 finally: 302 if tempdir: 303 shutil.rmtree(str(tempdir)) 304 305 306 def add_microsoft_defender_antivirus_exclusions(dest, no_system_changes): 307 if no_system_changes: 308 return 309 310 if not WINDOWS: 311 return 312 313 powershell_exe = which("powershell") 314 315 if not powershell_exe: 316 return 317 318 def print_attempt_exclusion(path): 319 print( 320 f"Attempting to add exclusion path to Microsoft Defender Antivirus for: {path}" 321 ) 322 323 powershell_exe = str(powershell_exe) 324 paths = [] 325 326 # mozilla-unified / clone dest 327 repo_dir = Path.cwd() / dest 328 paths.append(repo_dir) 329 print_attempt_exclusion(repo_dir) 330 331 # MOZILLABUILD 332 mozillabuild_dir = os.getenv("MOZILLABUILD") 333 if mozillabuild_dir: 334 paths.append(mozillabuild_dir) 335 print_attempt_exclusion(mozillabuild_dir) 336 337 # .mozbuild 338 mozbuild_dir = Path.home() / ".mozbuild" 339 paths.append(mozbuild_dir) 340 print_attempt_exclusion(mozbuild_dir) 341 342 args = ";".join(f"Add-MpPreference -ExclusionPath '{path}'" for path in paths) 343 command = f'-Command "{args}"' 344 345 # This will attempt to run as administrator by triggering a UAC prompt 346 # for admin credentials. If "No" is selected, no exclusions are added. 347 ctypes.windll.shell32.ShellExecuteW(None, "runas", powershell_exe, command, None, 0) 348 349 350 def clone(options): 351 vcs = options.vcs 352 no_interactive = options.no_interactive 353 no_system_changes = options.no_system_changes 354 355 if vcs == "hg": 356 hg = which("hg") 357 if not hg: 358 print("Mercurial is not installed. Mercurial is required to clone Firefox.") 359 try: 360 # We're going to recommend people install the Mercurial package with 361 # pip3. That will work if `pip3` installs binaries to a location 362 # that's in the PATH, but it might not be. To help out, if we CAN 363 # import "mercurial" (in which case it's already been installed), 364 # offer that as a solution. 365 import mercurial # noqa: F401 366 367 print( 368 "Hint: have you made sure that Mercurial is installed to a " 369 "location in your PATH?" 370 ) 371 except ImportError: 372 print("Try installing hg with `pip3 install Mercurial`.") 373 return None 374 binary = hg 375 else: 376 binary = which("git") 377 if not binary: 378 print("Git is not installed.") 379 print("Try installing git using your system package manager.") 380 return None 381 382 dest = input_clone_dest(vcs, no_interactive) 383 if not dest: 384 return None 385 386 add_microsoft_defender_antivirus_exclusions(dest, no_system_changes) 387 388 print(f"Cloning Firefox {VCS_HUMAN_READABLE[vcs]} repository to {dest}") 389 390 head_repo = os.environ.get("GECKO_HEAD_REPOSITORY") 391 head_rev = os.environ.get("GECKO_HEAD_REV") 392 393 if vcs == "hg": 394 return hg_clone_firefox(binary, dest, head_repo, head_rev) 395 elif vcs == "git-cinnabar": 396 return git_cinnabar_clone_firefox(binary, dest, head_repo, head_rev) 397 else: 398 return git_clone_firefox(binary, dest, head_repo, head_rev) 399 400 401 def bootstrap(srcdir: Path, application_choice, no_interactive, no_system_changes): 402 args = [sys.executable, "mach"] 403 404 if no_interactive: 405 # --no-interactive is a global argument, not a command argument, 406 # so it needs to be specified before "bootstrap" is appended. 407 args += ["--no-interactive"] 408 409 args += ["bootstrap"] 410 411 if application_choice: 412 args += ["--application-choice", application_choice] 413 if no_system_changes: 414 args += ["--no-system-changes"] 415 416 print("Running `%s`" % " ".join(args)) 417 return subprocess.call(args, cwd=str(srcdir)) 418 419 420 def main(args): 421 parser = OptionParser() 422 parser.add_option( 423 "--application-choice", 424 dest="application_choice", 425 help='Pass in an application choice (see "APPLICATIONS" in ' 426 "python/mozboot/mozboot/bootstrap.py) instead of using the " 427 "default interactive prompt.", 428 ) 429 parser.add_option( 430 "--vcs", 431 dest="vcs", 432 default="git", 433 choices=["git", "git-cinnabar", "hg"], 434 help="VCS (hg or git) to use for downloading the source code, " 435 "instead of using the default interactive prompt.", 436 ) 437 parser.add_option( 438 "--no-interactive", 439 dest="no_interactive", 440 action="store_true", 441 help="Answer yes to any (Y/n) interactive prompts.", 442 ) 443 parser.add_option( 444 "--no-system-changes", 445 dest="no_system_changes", 446 action="store_true", 447 help="Only executes actions that leave the system configuration alone.", 448 ) 449 450 options, leftover = parser.parse_args(args) 451 try: 452 srcdir = clone(options) 453 if not srcdir: 454 return 1 455 print("Clone complete.") 456 print( 457 "If you need to run the tooling bootstrapping again, " 458 "then consider running './mach bootstrap' instead." 459 ) 460 if not options.no_interactive: 461 remove_bootstrap_file = input( 462 "Unless you are going to have more local copies of Firefox source code, " 463 "this 'bootstrap.py' file is no longer needed and can be deleted. " 464 "Clean up the bootstrap.py file? (Y/n)" 465 ) 466 if not remove_bootstrap_file: 467 remove_bootstrap_file = "y" 468 if options.no_interactive or remove_bootstrap_file == "y": 469 try: 470 Path(sys.argv[0]).unlink() 471 except FileNotFoundError: 472 print("File could not be found !") 473 return bootstrap( 474 srcdir, 475 options.application_choice, 476 options.no_interactive, 477 options.no_system_changes, 478 ) 479 except Exception: 480 print("Could not bootstrap Firefox! Consider filing a bug.") 481 raise 482 483 484 if __name__ == "__main__": 485 sys.exit(main(sys.argv))