update-angle.py (15603B)
1 #! /usr/bin/env python3 2 # This Source Code Form is subject to the terms of the Mozilla Public 3 # License, v. 2.0. If a copy of the MPL was not distributed with this 4 # file, You can obtain one at http://mozilla.org/MPL/2.0/. 5 6 assert __name__ == "__main__" 7 8 r""" 9 To update ANGLE in Gecko, use Windows with git-bash, and setup depot_tools, python2, and 10 python3. Because depot_tools expects `python` to be `python2` (shame!), python2 must come 11 before python3 in your path. 12 13 Upstream: https://chromium.googlesource.com/angle/angle 14 15 Our repo: https://github.com/mozilla/angle 16 It has branches like 'firefox-60' which is the branch we use for pulling into 17 Gecko with this script. 18 19 This script leaves a record of the merge-base and cherry-picks that we pull into 20 Gecko. (gfx/angle/cherries.log) 21 22 ANGLE<->Chrome version mappings are here: https://omahaproxy.appspot.com/ 23 An easy choice is to grab Chrome's Beta's ANGLE branch. 24 25 ## Usage 26 27 Prepare your env: 28 29 * If in `cmd`: 30 `export PATH="$PATH:/path/to/depot_tools"` 31 * If in `powershell`: 32 `$env:Path += ";C:\path\to\depot_tools"` 33 34 If this is a new repo, don't forget: 35 36 ~~~ 37 # In the angle repo: 38 ./scripts/bootstrap.py 39 gclient sync 40 ~~~ 41 42 Update: (in the angle repo) 43 44 ~~~ 45 # In the angle repo: 46 /path/to/gecko/gfx/angle/update-angle.py origin 47 git push moz # Push the firefox-XX branch to github.com/mozilla/angle 48 ~~~~ 49 50 """ 51 52 import json 53 import os 54 import pathlib 55 import re 56 import shutil 57 import subprocess 58 import sys 59 from typing import * # mypy annotations 60 61 REPO_DIR = pathlib.Path.cwd() 62 GECKO_ANGLE_DIR = pathlib.Path(__file__).parent 63 64 OUT_DIR = pathlib.Path("out") 65 66 COMMON_HEADER = [ 67 "# Generated by update-angle.py", 68 "", 69 'include("../../moz.build.common")', 70 ] 71 72 ROOTS = ["//:translator", "//:libEGL", "//:libGLESv2"] 73 74 CHECK_ONLY = False 75 args = sys.argv[1:] 76 while True: 77 arg = args.pop(0) 78 if arg == "--check": 79 CHECK_ONLY = True 80 continue 81 args.insert(0, arg) 82 break 83 84 GN_ENV = dict(os.environ) 85 GN_ENV["DEPOT_TOOLS_WIN_TOOLCHAIN"] = "0" 86 87 (GIT_REMOTE,) = args # Not always 'origin'! 88 89 # ------------------------------------------------------------------------------ 90 91 92 def run_checked(*args, **kwargs): 93 print(" ", args) 94 sys.stdout.flush() 95 return subprocess.run(args, check=True, **kwargs) 96 97 98 def sorted_items(x): 99 for k in sorted(x.keys()): 100 yield (k, x[k]) 101 102 103 def collapse_dotdots(path): 104 split = path.split("/") 105 106 ret = [] 107 for x in split: 108 if x == ".." and ret: 109 ret.pop() 110 continue 111 ret.append(x) 112 continue 113 114 return "/".join(ret) 115 116 117 def dag_traverse(root_keys: Sequence[str], pre_recurse_func: Callable[[str], list]): 118 visited_keys: Set[str] = set() 119 120 def recurse(key): 121 if key in visited_keys: 122 return 123 visited_keys.add(key) 124 125 t = pre_recurse_func(key) 126 try: 127 (next_keys, post_recurse_func) = t 128 except ValueError: 129 (next_keys,) = t 130 post_recurse_func = None 131 132 for x in next_keys: 133 recurse(x) 134 135 if post_recurse_func: 136 post_recurse_func(key) 137 return 138 139 for x in root_keys: 140 recurse(x) 141 return 142 143 144 # ------------------------------------------------------------------------------ 145 146 print("Importing graph") 147 148 # shutil.rmtree(str(OUT_DIR), True) 149 OUT_DIR.mkdir(exist_ok=True) 150 151 GN_ARGS = b""" 152 # Build arguments go here. 153 # See "gn args <out_dir> --list" for available build arguments. 154 is_clang = true 155 is_debug = false 156 angle_build_all = false 157 angle_enable_abseil = false 158 angle_enable_apple_translator_workarounds = true 159 angle_enable_essl = true 160 angle_enable_gl = false 161 angle_enable_gl_desktop_frontend = false 162 angle_enable_glsl = true 163 angle_enable_null = false 164 angle_enable_share_context_lock = true 165 angle_enable_vulkan = false 166 angle_has_astc_encoder = false 167 use_custom_libcxx = false 168 """[ 169 1: 170 ] 171 args_gn_path = OUT_DIR / "args.gn" 172 args_gn_path.write_bytes(GN_ARGS) 173 174 try: 175 run_checked("gn", "gen", str(OUT_DIR), shell=True, env=GN_ENV) 176 except subprocess.CalledProcessError: 177 sys.stderr.buffer.write(b"`gn` failed. Is depot_tools in your PATH?\n") 178 exit(1) 179 180 p = run_checked( 181 "python3", 182 "scripts/export_targets.py", 183 str(OUT_DIR), 184 *ROOTS, 185 stdout=subprocess.PIPE, 186 shell=True, 187 env=GN_ENV, 188 ) 189 190 # - 191 192 print("\nProcessing graph") 193 libraries = json.loads(p.stdout.decode()) 194 195 # - 196 # HACKHACKHACK: Inject linux/mac sources instead of trying to merge graphs of different 197 # platforms. 198 # descs["//:angle_common"]["sources"] += 199 EXTRA_ANGLE_COMMON_SOURCES = [ 200 "//src/common/system_utils_apple.cpp", 201 "//src/common/system_utils_linux.cpp", 202 "//src/common/system_utils_mac.cpp", 203 "//src/common/system_utils_posix.cpp", 204 ] 205 206 angle_common = libraries["//:angle_common"] 207 angle_common["sources"] += EXTRA_ANGLE_COMMON_SOURCES 208 angle_common["sources"] = sorted(angle_common["sources"]) 209 210 # - 211 # Reuse our own zlib 212 213 del libraries["//third_party/zlib:zlib"] 214 215 # - 216 217 if CHECK_ONLY: 218 print("\n--check complete.") 219 exit(0) 220 221 # ------------------------------------------------------------------------------ 222 # Output to moz.builds 223 224 import vendor_from_git 225 226 print("") 227 vendor_from_git.record_cherry_picks(GECKO_ANGLE_DIR, GIT_REMOTE) 228 229 # -- 230 231 232 def sortedi(x): 233 return sorted(x, key=str.lower) 234 235 236 def append_arr(dest, name, vals, indent=0): 237 if not vals: 238 return 239 240 dest.append("{}{} += [".format(" " * 4 * indent, name)) 241 for x in sortedi(vals): 242 dest.append('{}"{}",'.format(" " * 4 * (indent + 1), x)) 243 dest.append("{}]".format(" " * 4 * indent)) 244 dest.append("") 245 return 246 247 248 REGISTERED_DEFINES = { 249 "ADLER32_SIMD_SSSE3": False, 250 "ANGLE_CAPTURE_ENABLED": True, 251 "ANGLE_DISABLE_POOL_ALLOC": True, 252 "ANGLE_EGL_LIBRARY_NAME": False, 253 "ANGLE_ENABLE_APPLE_WORKAROUNDS": True, 254 "ANGLE_ENABLE_D3D11": True, 255 "ANGLE_ENABLE_D3D11_COMPOSITOR_NATIVE_WINDOW": True, 256 "ANGLE_ENABLE_D3D9": True, 257 "ANGLE_ENABLE_DEBUG_ANNOTATIONS": True, 258 "ANGLE_ENABLE_NULL": False, 259 "ANGLE_ENABLE_OPENGL": False, 260 "ANGLE_ENABLE_OPENGL_NULL": False, 261 "ANGLE_ENABLE_ESSL": True, 262 "ANGLE_ENABLE_GLSL": True, 263 "ANGLE_ENABLE_HLSL": True, 264 "ANGLE_ENABLE_SHARE_CONTEXT_LOCK": True, 265 "ANGLE_GENERATE_SHADER_DEBUG_INFO": True, 266 "ANGLE_GLESV2_LIBRARY_NAME": True, 267 "ANGLE_HAS_ASTCENC": True, 268 "ANGLE_HAS_VULKAN_SYSTEM_INFO": False, 269 "ANGLE_IS_64_BIT_CPU": False, 270 "ANGLE_IS_WIN": False, 271 "ANGLE_PRELOADED_D3DCOMPILER_MODULE_NAMES": False, 272 "ANGLE_SHARED_LIBVULKAN": True, 273 "ANGLE_USE_CUSTOM_LIBVULKAN": True, 274 "ANGLE_USE_EGL_LOADER": True, 275 "ANGLE_VK_LAYERS_DIR": True, 276 "ANGLE_VK_MOCK_ICD_JSON": True, 277 "ANGLE_VMA_VERSION": True, 278 "ASTCENC_DECOMPRESS_ONLY": True, 279 "CERT_CHAIN_PARA_HAS_EXTRA_FIELDS": False, 280 "CHROMIUM_BUILD": False, 281 "COMPONENT_BUILD": False, 282 "CRC32_SIMD_SSE42_PCLMUL": False, 283 "DEFLATE_FILL_WINDOW_SSE2": False, 284 "DYNAMIC_ANNOTATIONS_ENABLED": True, 285 "EGL_EGL_PROTOTYPES": True, 286 "EGL_EGLEXT_PROTOTYPES": True, 287 "EGLAPI": True, 288 "FIELDTRIAL_TESTING_ENABLED": False, 289 "FULL_SAFE_BROWSING": False, 290 "GL_API": True, 291 "GL_APICALL": True, 292 "GL_GLES_PROTOTYPES": True, 293 "GL_GLEXT_PROTOTYPES": True, 294 "GPU_INFO_USE_SETUPAPI": True, 295 "INFLATE_CHUNK_READ_64LE": False, 296 "INFLATE_CHUNK_SIMD_SSE2": False, 297 "LIBANGLE_IMPLEMENTATION": True, 298 "LIBEGL_IMPLEMENTATION": True, 299 "LIBGLESV2_IMPLEMENTATION": True, 300 "NOMINMAX": True, 301 "NO_TCMALLOC": False, 302 # Else: gfx/angle/checkout/src/libANGLE/renderer/d3d/d3d11/win32/NativeWindow11Win32.cpp(89): error C2787: 'IDCompositionDevice': no GUID has been associated with this object 303 "NTDDI_VERSION": False, 304 "PSAPI_VERSION": False, 305 "SAFE_BROWSING_CSD": False, 306 "SAFE_BROWSING_DB_LOCAL": False, 307 "UNICODE": True, 308 "USE_AURA": False, 309 "V8_DEPRECATION_WARNINGS": False, 310 "VK_USE_PLATFORM_WIN32_KHR": False, 311 "WIN32": False, 312 "WIN32_LEAN_AND_MEAN": False, 313 "WINAPI_FAMILY": False, 314 "WINVER": True, 315 # Otherwise: 316 # gfx/angle/targets/libANGLE 317 # In file included from c:/dev/mozilla/gecko4/gfx/angle/checkout/src/libANGLE/renderer/d3d/d3d11/converged/CompositorNativeWindow11.cpp:10: 318 # In file included from c:/dev/mozilla/gecko4/gfx/angle/checkout/src\libANGLE/renderer/d3d/d3d11/converged/CompositorNativeWindow11.h:17: 319 # C:\Program Files (x86)\Windows Kits\10\include\10.0.17763.0\winrt\Windows.ui.composition.interop.h(103,20): error: unknown type name 'POINTER_INFO' 320 # _In_ const POINTER_INFO& pointerInfo 321 # ^ 322 "WTF_USE_DYNAMIC_ANNOTATIONS": False, 323 "X86_WINDOWS": False, 324 "ZLIB_IMPLEMENTATION": True, 325 "_ATL_NO_OPENGL": True, 326 "_CRT_NONSTDC_NO_DEPRECATE": True, 327 "_CRT_NONSTDC_NO_WARNINGS": True, 328 "_CRT_RAND_S": True, 329 "_CRT_SECURE_NO_DEPRECATE": True, 330 "_DEBUG": False, 331 "_HAS_EXCEPTIONS": True, 332 "_HAS_ITERATOR_DEBUGGING": False, 333 "_SCL_SECURE_NO_DEPRECATE": True, 334 "_SECURE_ATL": True, 335 "_UNICODE": True, 336 "_USING_V110_SDK71_": False, 337 "_WIN32_WINNT": False, 338 "_WINDOWS": False, 339 "_WINSOCK_DEPRECATED_NO_WARNINGS": True, 340 "__STD_C": False, 341 # clang specific 342 "CR_CLANG_REVISION": True, 343 "NDEBUG": False, 344 "NVALGRIND": False, 345 "_HAS_NODISCARD": False, 346 } 347 348 # - 349 350 print("\nRun actions") 351 required_files: Set[str] = set() 352 required_files.add("//LICENSE") 353 354 run_checked("ninja", "-C", str(OUT_DIR), ":angle_commit_id", shell=True) 355 required_files.add("//out/gen/angle/angle_commit.h") 356 357 # - 358 359 # Export our targets 360 print("\nExport targets") 361 362 # Clear our dest directories 363 targets_dir = pathlib.Path(GECKO_ANGLE_DIR, "targets") 364 checkout_dir = pathlib.Path(GECKO_ANGLE_DIR, "checkout") 365 366 shutil.rmtree(targets_dir, True) 367 shutil.rmtree(checkout_dir, True) 368 targets_dir.mkdir(exist_ok=True) 369 checkout_dir.mkdir(exist_ok=True) 370 371 # - 372 373 RE_TARGET_NAME = re.compile("//(.*):(.+)") 374 375 376 def export_target(target_full_name) -> Set[str]: 377 # print(' ', target_full_name) 378 descs = libraries 379 desc = descs[target_name] 380 flat = desc 381 382 m = RE_TARGET_NAME.match(target_name) 383 assert m, target_name 384 name = m.group(2) 385 386 required_files: Set[str] = set(flat["sources"]) 387 388 # Create our manifest lines 389 target_dir = targets_dir / name 390 target_dir.mkdir(exist_ok=True) 391 392 lines = list(COMMON_HEADER) 393 lines.append("") 394 395 for x in sorted(set(desc["defines"])): 396 try: 397 (k, v) = x.split("=", 1) 398 if v.startswith('"'): 399 v = f"'{v}'" 400 else: 401 v = f'"{v}"' 402 except ValueError: 403 (k, v) = (x, "True") 404 405 line = f'DEFINES["{k}"] = {v}' 406 try: 407 if REGISTERED_DEFINES[k] == False: 408 line = "# " + line 409 except KeyError: 410 print(f"[{name}] Unrecognized define: {k}") 411 line = "# Unrecognized: " + line 412 lines.append(line) 413 lines.append("") 414 415 cxxflags = set(desc["cflags"] + desc["cflags_cc"]) 416 417 def fixup_paths(listt): 418 for x in set(listt): 419 assert x.startswith("//"), x 420 yield "../../checkout/" + x[2:] 421 422 sources_by_config: Dict[str, List[str]] = {} 423 extras: Dict[str, str] = dict() 424 for x in fixup_paths(flat["sources"]): 425 # print(' '*5, x) 426 (b, e) = x.rsplit(".", 1) 427 if e in ["h", "hpp", "y", "l", "inc", "inl"]: 428 continue 429 elif e in ["cpp", "cc", "c"]: 430 if b.endswith("_win") or b.endswith("_win32"): 431 config = 'CONFIG["OS_ARCH"] == "WINNT"' 432 elif b.endswith("_linux"): 433 # Include these on BSDs too. 434 config = 'CONFIG["OS_ARCH"] not in ("Darwin", "WINNT")' 435 elif b.endswith("_apple") or b.endswith("_mac"): 436 config = 'CONFIG["OS_ARCH"] == "Darwin"' 437 elif b.endswith("_posix"): 438 config = 'CONFIG["OS_ARCH"] != "WINNT"' 439 else: 440 config = "" # None can't compare against str. 441 442 sources_by_config.setdefault(config, []).append(x) 443 continue 444 elif e == "rc": 445 assert "RCFILE" not in extras, (target_name, extras["RCFILE"], x) 446 extras["RCFILE"] = f'"{x}"' 447 continue 448 elif e == "def": 449 assert "DEFFILE" not in extras, (target_name, extras["DEFFILE"], x) 450 extras["DEFFILE"] = f'"{x}"' 451 continue 452 else: 453 assert False, ("Unhandled ext:", x) 454 455 ldflags = set(desc["ldflags"]) 456 DEF_PREFIX = "/DEF:" 457 for x in set(ldflags): 458 if x.startswith(DEF_PREFIX): 459 def_path = x[len(DEF_PREFIX) :] 460 required_files.add(def_path) 461 assert "DEFFILE" not in extras 462 ldflags.remove(x) 463 464 def_path = str(OUT_DIR) + "/" + def_path 465 def_path = "//" + collapse_dotdots(def_path) 466 467 def_rel_path = list(fixup_paths([def_path]))[0] 468 extras["DEFFILE"] = '"{}"'.format(def_rel_path) 469 elif x.startswith("/PDBSourcePath:"): 470 ldflags.remove(x) 471 472 os_libs = list(map(lambda x: x[: -len(".lib")], set(desc.get("libs", [])))) 473 474 def append_arr_commented(dest, name, src): 475 lines = [] 476 append_arr(lines, name, src) 477 478 def comment(x): 479 if x: 480 x = "# " + x 481 return x 482 483 lines = map(comment, lines) 484 dest += lines 485 486 append_arr(lines, "LOCAL_INCLUDES", fixup_paths(desc["include_dirs"])) 487 append_arr_commented(lines, "CXXFLAGS", cxxflags) 488 489 for config, v in sorted_items(sources_by_config): 490 indent = 0 491 if config: 492 lines.append("if {}:".format(config)) 493 indent = 1 494 append_arr(lines, "SOURCES", v, indent=indent) 495 496 dep_libs: Set[str] = set() 497 for dep_full_name in set(flat["dep_libs"]): 498 assert dep_full_name.startswith("//"), dep_name 499 (_, dep_name) = dep_full_name.split(":") 500 dep_libs.add(dep_name) 501 502 dep_dirs = set(dep_libs) 503 dep_dirs.discard("zlib") 504 505 # Those directories are added by gfx/angle/moz.build. 506 already_added_dirs = [ 507 "angle_common", 508 "translator", 509 "libEGL", 510 "libGLESv2", 511 ] 512 513 append_arr(lines, "USE_LIBS", dep_libs) 514 append_arr( 515 lines, "DIRS", ["../" + x for x in dep_dirs if x not in already_added_dirs] 516 ) 517 append_arr(lines, "OS_LIBS", os_libs) 518 append_arr_commented(lines, "LDFLAGS", ldflags) 519 520 for k, v in sorted(extras.items()): 521 lines.append("{} = {}".format(k, v)) 522 523 lib_type = desc["type"] 524 if lib_type == "shared_library": 525 lines.append(f'GeckoSharedLibrary("{name}")') 526 elif lib_type == "static_library": 527 lines.append(f'Library("{name}")') 528 else: 529 assert False, lib_type 530 531 lines.append("") 532 # EOF newline 533 534 # Write it out 535 536 mozbuild = target_dir / "moz.build" 537 print(" ", " ", f"Writing {mozbuild}") 538 data = b"\n".join((x.encode() for x in lines)) 539 mozbuild.write_bytes(data) 540 541 return required_files 542 543 544 # - 545 546 for target_name in libraries: 547 reqs = export_target(target_name) 548 required_files |= reqs 549 550 # Copy all the files 551 552 print("\nMigrate required files") 553 554 i = 0 555 for x in required_files: 556 i += 1 557 sys.stdout.write(f"\r Copying {i}/{len(required_files)}") 558 sys.stdout.flush() 559 assert x.startswith("//"), x 560 x = x[2:] 561 562 src = REPO_DIR / x 563 dest = checkout_dir / x 564 565 dest.parent.mkdir(parents=True, exist_ok=True) 566 data = src.read_bytes() 567 data = data.replace(b"\r\n", b"\n") 568 dest.write_bytes(data) 569 570 print("\n\nDone")