gn_processor.py (32727B)
1 # This Source Code Form is subject to the terms of the Mozilla Public 2 # License, v. 2.0. If a copy of the MPL was not distributed with this 3 # file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 5 import argparse 6 import json 7 import os 8 import subprocess 9 import sys 10 import tempfile 11 from collections import defaultdict, deque 12 from concurrent.futures import ProcessPoolExecutor, as_completed 13 from copy import deepcopy 14 from pathlib import Path 15 from shutil import which 16 17 import mozpack.path as mozpath 18 from mozbuild.bootstrap import bootstrap_toolchain 19 from mozbuild.dirutils import mkdir 20 from mozbuild.frontend.sandbox import alphabetical_sorted 21 from mozfile import json as mozfile_json 22 23 license_header = """# This Source Code Form is subject to the terms of the Mozilla Public 24 # License, v. 2.0. If a copy of the MPL was not distributed with this 25 # file, You can obtain one at http://mozilla.org/MPL/2.0/. 26 """ 27 28 generated_header = """ 29 ### This moz.build was AUTOMATICALLY GENERATED from a GN config, ### 30 ### DO NOT edit it by hand. ### 31 """ 32 33 34 class MozbuildWriter: 35 def __init__(self, fh): 36 self._fh = fh 37 self.indent = "" 38 self._indent_increment = 4 39 40 # We need to correlate a small amount of state here to figure out 41 # which library template to use ("Library()" or "SharedLibrary()") 42 self._library_name = None 43 self._shared_library = None 44 45 def mb_serialize(self, v): 46 if isinstance(v, list): 47 if len(v) <= 1: 48 return repr(v) 49 # Pretty print a list 50 raw = json.dumps(v, indent=self._indent_increment) 51 # Add the indent of the current indentation level 52 return raw.replace("\n", "\n" + self.indent) 53 if isinstance(v, bool): 54 return repr(v) 55 return f'"{v}"' 56 57 def finalize(self): 58 if self._library_name: 59 self.write("\n") 60 if self._shared_library: 61 self.write_ln(f"SharedLibrary({self.mb_serialize(self._library_name)})") 62 else: 63 self.write_ln(f"Library({self.mb_serialize(self._library_name)})") 64 65 def write(self, content): 66 self._fh.write(content) 67 68 def write_ln(self, line): 69 self.write(self.indent) 70 self.write(line) 71 self.write("\n") 72 73 def write_attrs(self, context_attrs): 74 for k in sorted(context_attrs.keys()): 75 v = context_attrs[k] 76 if isinstance(v, (list, set)): 77 self.write_mozbuild_list(k, v) 78 elif isinstance(v, dict): 79 self.write_mozbuild_dict(k, v) 80 else: 81 self.write_mozbuild_value(k, v) 82 83 def write_mozbuild_list(self, key, value): 84 if value: 85 self.write("\n") 86 self.write(self.indent + key) 87 self.write(" += [\n " + self.indent) 88 self.write( 89 (",\n " + self.indent).join( 90 alphabetical_sorted(self.mb_serialize(v) for v in value) 91 ) 92 ) 93 self.write("\n") 94 self.write_ln("]") 95 96 def write_mozbuild_value(self, key, value): 97 if value: 98 if key == "LIBRARY_NAME": 99 self._library_name = value 100 elif key == "FORCE_SHARED_LIB": 101 self._shared_library = True 102 else: 103 self.write("\n") 104 self.write_ln(f"{key} = {self.mb_serialize(value)}") 105 self.write("\n") 106 107 def write_mozbuild_dict(self, key, value): 108 # Templates we need to use instead of certain values. 109 replacements = ( 110 ( 111 ("COMPILE_FLAGS", '"WARNINGS_AS_ERRORS"', "[]"), 112 "AllowCompilerWarnings()", 113 ), 114 ) 115 if value: 116 self.write("\n") 117 if key == "GeneratedFile": 118 self.write_ln("GeneratedFile(") 119 self.indent += " " * self._indent_increment 120 for o in value["outputs"]: 121 self.write_ln(f"{self.mb_serialize(o)},") 122 for k, v in sorted(value.items()): 123 if k == "outputs": 124 continue 125 self.write_ln(f"{k}={self.mb_serialize(v)},") 126 self.indent = self.indent[self._indent_increment :] 127 self.write_ln(")") 128 return 129 for k in sorted(value.keys()): 130 v = value[k] 131 subst_vals = key, self.mb_serialize(k), self.mb_serialize(v) 132 wrote_ln = False 133 for flags, tmpl in replacements: 134 if subst_vals == flags: 135 self.write_ln(tmpl) 136 wrote_ln = True 137 138 if not wrote_ln: 139 self.write_ln( 140 f"{key}[{self.mb_serialize(k)}] = {self.mb_serialize(v)}" 141 ) 142 143 def write_condition(self, values): 144 def mk_condition(k, v): 145 if not v: 146 return f'not CONFIG["{k}"]' 147 return f'CONFIG["{k}"] == {self.mb_serialize(v)}' 148 149 self.write("\n") 150 self.write("if ") 151 self.write( 152 " and ".join(mk_condition(k, values[k]) for k in sorted(values.keys())) 153 ) 154 self.write(":\n") 155 self.indent += " " * self._indent_increment 156 157 def terminate_condition(self): 158 assert len(self.indent) >= self._indent_increment 159 self.indent = self.indent[self._indent_increment :] 160 161 162 def find_deps(all_targets, target): 163 all_deps = set() 164 queue = deque([target]) 165 while queue: 166 item = queue.popleft() 167 all_deps.add(item) 168 for dep in all_targets[item]["deps"]: 169 if dep not in all_deps: 170 queue.append(dep) 171 return all_deps 172 173 174 def filter_gn_config(path, gn_result, sandbox_vars, input_vars, gn_target): 175 gen_path = path / "gen" 176 # Translates the raw output of gn into just what we'll need to generate a 177 # mozbuild configuration. 178 gn_out = {"targets": {}, "sandbox_vars": sandbox_vars} 179 180 cpus = { 181 "arm64": "aarch64", 182 "x64": "x86_64", 183 "mipsel": "mips32", 184 "mips64el": "mips64", 185 "loong64": "loongarch64", 186 } 187 oses = { 188 "android": "Android", 189 "linux": "Linux", 190 "mac": "Darwin", 191 "openbsd": "OpenBSD", 192 "win": "WINNT", 193 } 194 195 mozbuild_args = { 196 "MOZ_DEBUG": "1" if input_vars.get("is_debug") else None, 197 "OS_TARGET": oses[input_vars["target_os"]], 198 "TARGET_CPU": cpus.get(input_vars["target_cpu"], input_vars["target_cpu"]), 199 } 200 if "ozone_platform_x11" in input_vars: 201 mozbuild_args["MOZ_X11"] = "1" if input_vars["ozone_platform_x11"] else None 202 203 gn_out["mozbuild_args"] = mozbuild_args 204 all_deps = find_deps(gn_result["targets"], gn_target) 205 206 for target_fullname in all_deps: 207 raw_spec = gn_result["targets"][target_fullname] 208 209 if raw_spec["type"] == "action": 210 # Special handling for the action type to avoid putting empty 211 # arrays of args, script and outputs on all other types in `spec`. 212 spec = {} 213 for spec_attr in ( 214 "type", 215 "args", 216 "script", 217 "outputs", 218 ): 219 spec[spec_attr] = raw_spec.get(spec_attr, []) 220 if spec_attr == "outputs": 221 # Rebase outputs from an absolute path in the temp dir to a 222 # path relative to the target dir. 223 spec[spec_attr] = [ 224 mozpath.relpath(d, path) for d in spec[spec_attr] 225 ] 226 gn_out["targets"][target_fullname] = spec 227 228 # TODO: 'executable' will need to be handled here at some point as well. 229 if raw_spec["type"] not in ("static_library", "shared_library", "source_set"): 230 continue 231 232 spec = {} 233 for spec_attr in ( 234 "type", 235 "sources", 236 "defines", 237 "include_dirs", 238 "cflags", 239 "cflags_c", 240 "cflags_cc", 241 "cflags_objc", 242 "cflags_objcc", 243 "deps", 244 "libs", 245 ): 246 spec[spec_attr] = raw_spec.get(spec_attr, []) 247 if spec_attr == "defines": 248 spec[spec_attr] = [ 249 d 250 for d in spec[spec_attr] 251 if "CR_XCODE_VERSION" not in d 252 and "CR_SYSROOT_HASH" not in d 253 and "CR_SYSROOT_KEY" not in d 254 and "_FORTIFY_SOURCE" not in d 255 ] 256 if spec_attr == "include_dirs": 257 # Rebase outputs from an absolute path in the temp dir to a path 258 # relative to the target dir. 259 spec[spec_attr] = [ 260 d if gen_path != Path(d) else "!//gen" for d in spec[spec_attr] 261 ] 262 263 gn_out["targets"][target_fullname] = spec 264 265 return gn_out 266 267 268 def process_gn_config( 269 gn_config, 270 topsrcdir, 271 srcdir, 272 non_unified_sources, 273 sandbox_vars, 274 mozilla_flags, 275 mozilla_add_override_dir, 276 ): 277 # Translates a json gn config into attributes that can be used to write out 278 # moz.build files for this configuration. 279 280 # Much of this code is based on similar functionality in `gyp_reader.py`. 281 282 mozbuild_attrs = {"mozbuild_args": gn_config.get("mozbuild_args", None), "dirs": {}} 283 284 targets = gn_config["targets"] 285 286 project_relsrcdir = mozpath.relpath(srcdir, topsrcdir) 287 288 non_unified_sources = set([mozpath.normpath(s) for s in non_unified_sources]) 289 290 def target_info(fullname): 291 path, name = target_fullname.split(":") 292 # Stripping '//' gives us a path relative to the project root, 293 # adding a suffix avoids name collisions with libraries already 294 # in the tree (like "webrtc"). 295 return path.lstrip("//"), name + "_gn" 296 297 def resolve_path(path): 298 # GN will have resolved all these paths relative to the root of the 299 # project indicated by "//". 300 if path.startswith("//"): 301 path = path[2:] 302 if not path.startswith("/"): 303 path = f"/{project_relsrcdir}/{path}" 304 return path 305 306 # Process all targets from the given gn project and its dependencies. 307 for target_fullname, spec in targets.items(): 308 target_path, target_name = target_info(target_fullname) 309 context_attrs = {} 310 311 # Remove leading 'lib' from the target_name if any, and use as 312 # library name. 313 name = target_name 314 if spec["type"] in ("static_library", "shared_library", "source_set", "action"): 315 if name.startswith("lib"): 316 name = name[3:] 317 context_attrs["LIBRARY_NAME"] = str(name) 318 else: 319 raise Exception( 320 "The following GN target type is not currently " 321 f'consumed by moz.build: "{spec["type"]}". It may need to be ' 322 "added, or you may need to re-run the " 323 "`GnConfigGen` step." 324 ) 325 326 if spec["type"] == "shared_library": 327 context_attrs["FORCE_SHARED_LIB"] = True 328 329 if spec["type"] == "action" and "script" in spec: 330 flags = [ 331 resolve_path(spec["script"]), 332 resolve_path(""), 333 ] + spec.get("args", []) 334 context_attrs["GeneratedFile"] = { 335 "script": "/python/mozbuild/mozbuild/action/file_generate_wrapper.py", 336 "entry_point": "action", 337 "outputs": [resolve_path(f) for f in spec["outputs"]], 338 "flags": flags, 339 } 340 341 sources = [] 342 unified_sources = [] 343 extensions = set() 344 use_defines_in_asflags = False 345 346 for f in [item.lstrip("//") for item in spec.get("sources", [])]: 347 ext = mozpath.splitext(f)[-1] 348 extensions.add(ext) 349 src = f"{project_relsrcdir}/{f}" 350 if ext in {".h", ".inc"}: 351 continue 352 elif ext == ".def": 353 context_attrs["SYMBOLS_FILE"] = src 354 elif ext != ".S" and src not in non_unified_sources: 355 unified_sources.append(f"/{src}") 356 else: 357 sources.append(f"/{src}") 358 # The Mozilla build system doesn't use DEFINES for building 359 # ASFILES. 360 if ext == ".s": 361 use_defines_in_asflags = True 362 363 context_attrs["SOURCES"] = sources 364 context_attrs["UNIFIED_SOURCES"] = unified_sources 365 366 context_attrs["DEFINES"] = {} 367 for define in spec.get("defines", []): 368 if "=" in define: 369 name, value = define.split("=", 1) 370 context_attrs["DEFINES"][name] = value 371 else: 372 context_attrs["DEFINES"][define] = True 373 374 context_attrs["LOCAL_INCLUDES"] = [] 375 for include in spec.get("include_dirs", []): 376 if include.startswith("!"): 377 include = "!" + resolve_path(include[1:]) 378 else: 379 include = resolve_path(include) 380 # moz.build expects all LOCAL_INCLUDES to exist, so ensure they do. 381 resolved = mozpath.abspath(mozpath.join(topsrcdir, include[1:])) 382 if not os.path.exists(resolved): 383 # GN files may refer to include dirs that are outside of the 384 # tree or we simply didn't vendor. Print a warning in this case. 385 if not resolved.endswith("gn-output/gen"): 386 print( 387 f"Included path: '{resolved}' does not exist, dropping include from GN " 388 "configuration.", 389 file=sys.stderr, 390 ) 391 continue 392 if include in context_attrs["LOCAL_INCLUDES"]: 393 continue 394 context_attrs["LOCAL_INCLUDES"] += [include] 395 396 context_attrs["ASFLAGS"] = spec.get("asflags_mozilla", []) 397 if use_defines_in_asflags and context_attrs["DEFINES"]: 398 context_attrs["ASFLAGS"] += ["-D" + d for d in context_attrs["DEFINES"]] 399 suffix_map = { 400 ".c": ("CFLAGS", ["cflags", "cflags_c"]), 401 ".cpp": ("CXXFLAGS", ["cflags", "cflags_cc"]), 402 ".cc": ("CXXFLAGS", ["cflags", "cflags_cc"]), 403 ".m": ("CMFLAGS", ["cflags", "cflags_objc"]), 404 ".mm": ("CMMFLAGS", ["cflags", "cflags_objcc"]), 405 } 406 variables = (suffix_map[e] for e in extensions if e in suffix_map) 407 for var, flag_keys in variables: 408 flags = [ 409 _f for _k in flag_keys for _f in spec.get(_k, []) if _f in mozilla_flags 410 ] 411 for f in flags: 412 # the result may be a string or a list. 413 if isinstance(f, str): 414 context_attrs.setdefault(var, []).append(f) 415 else: 416 context_attrs.setdefault(var, []).extend(f) 417 418 context_attrs["OS_LIBS"] = [] 419 for lib in spec.get("libs", []): 420 lib_name = os.path.splitext(lib)[0] 421 if lib.endswith(".framework"): 422 context_attrs["OS_LIBS"] += ["-framework " + lib_name] 423 else: 424 context_attrs["OS_LIBS"] += [lib_name] 425 426 # Add some features to all contexts. Put here in case LOCAL_INCLUDES 427 # order matters. 428 if mozilla_add_override_dir != "": 429 context_attrs["LOCAL_INCLUDES"] += [mozilla_add_override_dir] 430 431 context_attrs["LOCAL_INCLUDES"] += [ 432 "!/ipc/ipdl/_ipdlheaders", 433 "/ipc/chromium/src", 434 "/tools/profiler/public", 435 ] 436 # These get set via VC project file settings for normal GYP builds. 437 # TODO: Determine if these defines are needed for GN builds. 438 if gn_config["mozbuild_args"]["OS_TARGET"] == "WINNT": 439 context_attrs["DEFINES"]["UNICODE"] = True 440 context_attrs["DEFINES"]["_UNICODE"] = True 441 442 context_attrs["COMPILE_FLAGS"] = {"OS_INCLUDES": []} 443 444 for key, value in sandbox_vars.items(): 445 if context_attrs.get(key) and isinstance(context_attrs[key], list): 446 # If we have a key from sandbox_vars that's also been 447 # populated here we use the value from sandbox_vars as our 448 # basis rather than overriding outright. 449 context_attrs[key] = value + context_attrs[key] 450 elif context_attrs.get(key) and isinstance(context_attrs[key], dict): 451 context_attrs[key].update(value) 452 else: 453 context_attrs[key] = value 454 455 target_relsrcdir = mozpath.join(project_relsrcdir, target_path, target_name) 456 mozbuild_attrs["dirs"][target_relsrcdir] = context_attrs 457 458 return mozbuild_attrs 459 460 461 def find_common_attrs(config_attributes): 462 # Returns the intersection of the given configs and prunes the inputs 463 # to no longer contain these common attributes. 464 465 common_attrs = deepcopy(config_attributes[0]) 466 467 def make_intersection(reference, input_attrs): 468 # Modifies `reference` so that after calling this function it only 469 # contains parts it had in common with in `input_attrs`. 470 471 for k, input_value in input_attrs.items(): 472 # Anything in `input_attrs` must match what's already in 473 # `reference`. 474 common_value = reference.get(k) 475 if common_value: 476 if isinstance(input_value, list): 477 reference[k] = [ 478 i 479 for i in common_value 480 if input_value.count(i) == common_value.count(i) 481 ] 482 elif isinstance(input_value, dict): 483 reference[k] = { 484 key: value 485 for key, value in common_value.items() 486 if key in input_value and value == input_value[key] 487 } 488 elif input_value != common_value: 489 del reference[k] 490 elif k in reference: 491 del reference[k] 492 493 # Additionally, any keys in `reference` that aren't in `input_attrs` 494 # must be deleted. 495 for k in set(reference.keys()) - set(input_attrs.keys()): 496 del reference[k] 497 498 def make_difference(reference, input_attrs): 499 # Modifies `input_attrs` so that after calling this function it contains 500 # no parts it has in common with in `reference`. 501 for k, input_value in list(input_attrs.items()): 502 common_value = reference.get(k) 503 if common_value: 504 if isinstance(input_value, list): 505 input_attrs[k] = [ 506 i 507 for i in input_value 508 if common_value.count(i) != input_value.count(i) 509 ] 510 elif isinstance(input_value, dict): 511 input_attrs[k] = { 512 key: value 513 for key, value in input_value.items() 514 if key not in common_value 515 } 516 else: 517 del input_attrs[k] 518 519 for config_attr_set in config_attributes[1:]: 520 make_intersection(common_attrs, config_attr_set) 521 522 for config_attr_set in config_attributes: 523 make_difference(common_attrs, config_attr_set) 524 525 return common_attrs 526 527 528 def write_mozbuild(topsrcdir, write_mozbuild_variables, relsrcdir, configs): 529 target_srcdir = mozpath.join(topsrcdir, relsrcdir) 530 mkdir(target_srcdir) 531 532 target_mozbuild = mozpath.join(target_srcdir, "moz.build") 533 with open(target_mozbuild, "w") as fh: 534 mb = MozbuildWriter(fh) 535 mb.write(license_header) 536 mb.write("\n") 537 mb.write(generated_header) 538 539 try: 540 if relsrcdir in write_mozbuild_variables["INCLUDE_TK_CFLAGS_DIRS"]: 541 mb.write('if CONFIG["MOZ_WIDGET_TOOLKIT"] == "gtk":\n') 542 mb.write(' CXXFLAGS += CONFIG["MOZ_GTK3_CFLAGS"]\n') 543 except KeyError: 544 pass 545 try: 546 if relsrcdir in write_mozbuild_variables["INCLUDE_SYSTEM_GBM_HANDLING"]: 547 mb.write('CXXFLAGS += CONFIG["MOZ_GBM_CFLAGS"]\n') 548 mb.write('if not CONFIG["MOZ_SYSTEM_GBM"]:\n') 549 mb.write(' LOCAL_INCLUDES += [ "/third_party/gbm/gbm/" ]\n') 550 except KeyError: 551 pass 552 try: 553 if relsrcdir in write_mozbuild_variables["INCLUDE_SYSTEM_LIBDRM_HANDLING"]: 554 mb.write('CXXFLAGS += CONFIG["MOZ_LIBDRM_CFLAGS"]\n') 555 mb.write('if not CONFIG["MOZ_SYSTEM_LIBDRM"]:\n') 556 mb.write(' LOCAL_INCLUDES += [ "/third_party/drm/drm/",\n') 557 mb.write(' "/third_party/drm/drm/include/",\n') 558 mb.write( 559 ' "/third_party/drm/drm/include/libdrm" ]\n' 560 ) 561 except KeyError: 562 pass 563 try: 564 if ( 565 relsrcdir 566 in write_mozbuild_variables["INCLUDE_SYSTEM_PIPEWIRE_HANDLING"] 567 ): 568 mb.write('CXXFLAGS += CONFIG["MOZ_PIPEWIRE_CFLAGS"]\n') 569 mb.write('if not CONFIG["MOZ_SYSTEM_PIPEWIRE"]:\n') 570 mb.write(' LOCAL_INCLUDES += [ "/third_party/pipewire/" ]\n') 571 except KeyError: 572 pass 573 try: 574 if relsrcdir in write_mozbuild_variables["INCLUDE_SYSTEM_LIBVPX_HANDLING"]: 575 mb.write('if not CONFIG["MOZ_SYSTEM_LIBVPX"]:\n') 576 mb.write(' LOCAL_INCLUDES += [ "/media/libvpx/libvpx/" ]\n') 577 mb.write(' CXXFLAGS += CONFIG["MOZ_LIBVPX_CFLAGS"]\n') 578 except KeyError: 579 pass 580 try: 581 if relsrcdir in write_mozbuild_variables["INCLUDE_SYSTEM_DAV1D_HANDLING"]: 582 mb.write('if CONFIG["MOZ_SYSTEM_AV1"]:\n') 583 mb.write(' CXXFLAGS += CONFIG["MOZ_SYSTEM_DAV1D_CFLAGS"]\n') 584 mb.write(' CXXFLAGS += CONFIG["MOZ_SYSTEM_LIBAOM_CFLAGS"]\n') 585 except KeyError: 586 pass 587 588 all_args = [args for args, _ in configs] 589 590 # Start with attributes that will be a part of the mozconfig 591 # for every configuration, then factor by other potentially useful 592 # combinations. 593 # FIXME: this is a time-bomb. See bug 1775202. 594 for attrs in ( 595 (), 596 ("MOZ_DEBUG",), 597 ("OS_TARGET",), 598 ("TARGET_CPU",), 599 ("MOZ_DEBUG", "OS_TARGET"), 600 ("OS_TARGET", "MOZ_X11"), 601 ("OS_TARGET", "TARGET_CPU"), 602 ("OS_TARGET", "TARGET_CPU", "MOZ_X11"), 603 ("OS_TARGET", "TARGET_CPU", "MOZ_DEBUG"), 604 ("OS_TARGET", "TARGET_CPU", "MOZ_DEBUG", "MOZ_X11"), 605 ): 606 conditions = set() 607 for args in all_args: 608 cond = tuple((k, args.get(k) or "") for k in attrs) 609 conditions.add(cond) 610 611 for cond in sorted(conditions): 612 common_attrs = find_common_attrs([ 613 attrs 614 for args, attrs in configs 615 if all((args.get(k) or "") == v for k, v in cond) 616 ]) 617 if any(common_attrs.values()): 618 if cond: 619 mb.write_condition(dict(cond)) 620 mb.write_attrs(common_attrs) 621 if cond: 622 mb.terminate_condition() 623 624 mb.finalize() 625 return target_mozbuild 626 627 628 def write_mozbuild_files( 629 topsrcdir, 630 srcdir, 631 all_mozbuild_results, 632 write_mozbuild_variables, 633 ): 634 # Translate {config -> {dirs -> build info}} into 635 # {dirs -> [(config, build_info)]} 636 configs_by_dir = defaultdict(list) 637 for config_attrs in all_mozbuild_results: 638 mozbuild_args = config_attrs["mozbuild_args"] 639 dirs = config_attrs["dirs"] 640 for d, build_data in dirs.items(): 641 configs_by_dir[d].append((mozbuild_args, build_data)) 642 643 mozbuilds = set() 644 # threading this section did not produce noticeable speed gains 645 for relsrcdir, configs in sorted(configs_by_dir.items()): 646 mozbuilds.add( 647 write_mozbuild(topsrcdir, write_mozbuild_variables, relsrcdir, configs) 648 ) 649 650 # write the project moz.build file 651 dirs_mozbuild = mozpath.join(srcdir, "moz.build") 652 mozbuilds.add(dirs_mozbuild) 653 with open(dirs_mozbuild, "w") as fh: 654 mb = MozbuildWriter(fh) 655 mb.write(license_header) 656 mb.write("\n") 657 mb.write(generated_header) 658 659 # Not every srcdir is present for every config, which needs to be 660 # reflected in the generated root moz.build. 661 dirs_by_config = { 662 tuple(v["mozbuild_args"].items()): set(v["dirs"].keys()) 663 for v in all_mozbuild_results 664 } 665 666 for attrs in ( 667 (), 668 ("OS_TARGET",), 669 ("OS_TARGET", "TARGET_CPU"), 670 ("OS_TARGET", "TARGET_CPU", "MOZ_X11"), 671 ): 672 conditions = set() 673 for args in dirs_by_config.keys(): 674 cond = tuple((k, dict(args).get(k) or "") for k in attrs) 675 conditions.add(cond) 676 677 for cond in sorted(conditions): 678 common_dirs = None 679 for args, dir_set in dirs_by_config.items(): 680 if all((dict(args).get(k) or "") == v for k, v in cond): 681 if common_dirs is None: 682 common_dirs = deepcopy(dir_set) 683 else: 684 common_dirs &= dir_set 685 686 for args, dir_set in dirs_by_config.items(): 687 if all(dict(args).get(k) == v for k, v in cond): 688 dir_set -= common_dirs 689 690 if common_dirs: 691 if cond: 692 mb.write_condition(dict(cond)) 693 mb.write_mozbuild_list("DIRS", [f"/{d}" for d in common_dirs]) 694 if cond: 695 mb.terminate_condition() 696 697 # Remove possibly stale moz.builds 698 for root, dirs, files in os.walk(srcdir): 699 if "moz.build" in files: 700 file = os.path.join(root, "moz.build") 701 if file not in mozbuilds: 702 os.unlink(file) 703 704 705 def generate_gn_config( 706 topsrcdir, 707 build_root_dir, 708 target_dir, 709 gn_binary, 710 input_variables, 711 sandbox_variables, 712 gn_target, 713 moz_build_flag, 714 non_unified_sources, 715 mozilla_flags, 716 mozilla_add_override_dir, 717 ): 718 def str_for_arg(v): 719 if v in (True, False): 720 return str(v).lower() 721 return f'"{v}"' 722 723 build_root_dir = topsrcdir / build_root_dir 724 srcdir = build_root_dir / target_dir 725 726 input_variables = input_variables.copy() 727 input_variables.update({ 728 f"{moz_build_flag}": True, 729 "concurrent_links": 1, 730 "action_pool_depth": 1, 731 }) 732 733 if input_variables["target_os"] == "win": 734 input_variables.update({ 735 "visual_studio_path": "/", 736 "visual_studio_version": 2015, 737 "wdk_path": "/", 738 "windows_sdk_version": "n/a", 739 }) 740 if input_variables["target_os"] == "mac": 741 input_variables.update({ 742 "mac_sdk_path": "/", 743 }) 744 745 gn_args = f"--args={' '.join([f'{k}={str_for_arg(v)}' for k, v in input_variables.items()])}" 746 with tempfile.TemporaryDirectory() as tempdir: 747 # On Mac, `tempdir` starts with /var which is a symlink to /private/var. 748 # We resolve the symlinks in `tempdir` here so later usage with 749 # relpath() does not lead to unexpected results, should it be used 750 # together with another path that has symlinks resolved. 751 resolved_tempdir = Path(tempdir).resolve() 752 gen_args = [ 753 gn_binary, 754 "gen", 755 str(resolved_tempdir), 756 gn_args, 757 "--ide=json", 758 "--root=./", # must find the google build directory in this directory 759 f"--dotfile={target_dir}/.gn", 760 ] 761 print(f'Running "{" ".join(gen_args)}"', file=sys.stderr) 762 subprocess.check_call(gen_args, cwd=build_root_dir, stderr=subprocess.STDOUT) 763 764 gn_config_file = resolved_tempdir / "project.json" 765 with open(gn_config_file) as fh: 766 raw_json = fh.read() 767 raw_json = raw_json.replace(f"{target_dir}/", "") 768 raw_json = raw_json.replace(f"{target_dir}:", ":") 769 gn_config = mozfile_json.loads(raw_json) 770 gn_config = filter_gn_config( 771 resolved_tempdir, 772 gn_config, 773 sandbox_variables, 774 input_variables, 775 gn_target, 776 ) 777 gn_config = process_gn_config( 778 gn_config, 779 topsrcdir, 780 srcdir, 781 non_unified_sources, 782 gn_config["sandbox_vars"], 783 mozilla_flags, 784 mozilla_add_override_dir, 785 ) 786 return gn_config 787 788 789 def main(): 790 parser = argparse.ArgumentParser() 791 parser.add_argument("config", help="configuration in json format") 792 args = parser.parse_args() 793 794 gn_binary = bootstrap_toolchain("gn/gn") or which("gn") 795 if not gn_binary: 796 raise Exception("The GN program must be present to generate GN configs.") 797 798 with open(args.config) as fh: 799 config = mozfile_json.load(fh) 800 801 topsrcdir = Path(__file__).parent.parent.resolve() 802 803 vars_set = [] 804 for is_debug in (True, False): 805 for target_os in ("android", "linux", "mac", "openbsd", "win"): 806 target_cpus = ["x64"] 807 if target_os in ("android", "linux", "mac", "win", "openbsd"): 808 target_cpus.append("arm64") 809 if target_os in ("android", "linux"): 810 target_cpus.append("arm") 811 if target_os in ("android", "linux", "win"): 812 target_cpus.append("x86") 813 if target_os in ("linux", "openbsd"): 814 target_cpus.append("riscv64") 815 if target_os == "linux": 816 target_cpus.extend(["loong64", "ppc64", "mipsel", "mips64el"]) 817 for target_cpu in target_cpus: 818 vars = { 819 "host_cpu": "x64", 820 "is_debug": is_debug, 821 "target_cpu": target_cpu, 822 "target_os": target_os, 823 } 824 if target_os == "linux": 825 for enable_x11 in (True, False): 826 vars["ozone_platform_x11"] = enable_x11 827 vars_set.append(vars.copy()) 828 else: 829 if target_os == "openbsd": 830 vars["ozone_platform_x11"] = True 831 vars_set.append(vars) 832 833 gn_configs = [] 834 NUM_WORKERS = 5 835 with ProcessPoolExecutor(max_workers=NUM_WORKERS) as executor: 836 # Submit tasks to the executor 837 futures = { 838 executor.submit( 839 generate_gn_config, 840 topsrcdir, 841 config["build_root_dir"], 842 config["target_dir"], 843 gn_binary, 844 vars, 845 config["gn_sandbox_variables"], 846 config["gn_target"], 847 config["moz_build_flag"], 848 config["non_unified_sources"], 849 config["mozilla_flags"], 850 config["mozilla_add_override_dir"], 851 ): vars 852 for vars in vars_set 853 } 854 855 # Process completed tasks as they finish 856 error_generating_configs = False 857 for future in as_completed(futures): 858 try: 859 gn_configs.append(future.result()) 860 except Exception as e: 861 print(f"[Task] Task failed with exception: {e}") 862 error_generating_configs = True 863 864 if error_generating_configs: 865 print("\nGenerating configs failed. See errors above.\n", file=sys.stderr) 866 sys.exit(1) 867 868 print("All generation tasks have been processed.") 869 870 print("Writing moz.build files") 871 write_mozbuild_files( 872 topsrcdir, 873 topsrcdir / config["build_root_dir"] / config["target_dir"], 874 gn_configs, 875 config["write_mozbuild_variables"], 876 ) 877 878 879 if __name__ == "__main__": 880 main()