emitter.py (80363B)
1 # This Source Code Form is subject to the terms of the Mozilla Public 2 # License, v. 2.0. If a copy of the MPL was not distributed with this 3 # file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 5 import logging 6 import os 7 import sys 8 import time 9 import traceback 10 from collections import OrderedDict, defaultdict 11 12 import mozinfo 13 import mozpack.path as mozpath 14 import toml 15 from mach.mixin.logging import LoggingMixin 16 from mozpack.chrome.manifest import Manifest 17 18 from mozbuild.base import ExecutionSummary 19 from mozbuild.util import HierarchicalStringList, memoize 20 21 from ..testing import REFTEST_FLAVORS, TEST_MANIFESTS, SupportFilesConverter 22 from .context import Context, ObjDirPath, Path, SourcePath, SubContext 23 from .data import ( 24 BaseRustProgram, 25 ChromeManifestEntry, 26 ComputedFlags, 27 ConfigFileSubstitution, 28 Defines, 29 DirectoryTraversal, 30 Exports, 31 ExternalSharedLibrary, 32 ExternalStaticLibrary, 33 FinalTargetFiles, 34 FinalTargetPreprocessedFiles, 35 GeneratedFile, 36 HostDefines, 37 HostLibrary, 38 HostProgram, 39 HostRustLibrary, 40 HostRustProgram, 41 HostSharedLibrary, 42 HostSimpleProgram, 43 HostSources, 44 InstallationTarget, 45 IPDLCollection, 46 JARManifest, 47 LegacyRunTests, 48 Library, 49 Linkable, 50 LocalInclude, 51 LocalizedFiles, 52 LocalizedPreprocessedFiles, 53 MozSrcFiles, 54 ObjdirFiles, 55 ObjdirPreprocessedFiles, 56 PerSourceFlag, 57 Program, 58 RustLibrary, 59 RustProgram, 60 RustTests, 61 SandboxedWasmLibrary, 62 SharedLibrary, 63 SimpleProgram, 64 Sources, 65 StaticLibrary, 66 TestHarnessFiles, 67 TestManifest, 68 UnifiedSources, 69 VariablePassthru, 70 WasmDefines, 71 WasmSources, 72 WebIDLCollection, 73 XPCOMComponentManifests, 74 XPIDLModule, 75 ) 76 from .reader import SandboxValidationError 77 78 79 class TreeMetadataEmitter(LoggingMixin): 80 """Converts the executed mozbuild files into data structures. 81 82 This is a bridge between reader.py and data.py. It takes what was read by 83 reader.BuildReader and converts it into the classes defined in the data 84 module. 85 """ 86 87 def __init__(self, config): 88 self.populate_logger() 89 90 self.config = config 91 92 mozinfo.find_and_update_from_json(config.topobjdir) 93 94 self.info = dict(mozinfo.info) 95 96 self._libs = defaultdict(list) 97 self._binaries = OrderedDict() 98 self._compile_dirs = set() 99 self._host_compile_dirs = set() 100 self._wasm_compile_dirs = set() 101 self._asm_compile_dirs = set() 102 self._compile_flags = dict() 103 self._compile_as_flags = dict() 104 self._linkage = [] 105 self._static_linking_shared = set() 106 self._crate_verified_local = set() 107 self._crate_directories = dict() 108 self._idls = defaultdict(set) 109 110 # Keep track of external paths (third party build systems), starting 111 # from what we run a subconfigure in. We'll eliminate some directories 112 # as we traverse them with moz.build (e.g. js/src). 113 subconfigures = os.path.join(self.config.topobjdir, "subconfigures") 114 paths = [] 115 if os.path.exists(subconfigures): 116 paths = open(subconfigures).read().splitlines() 117 self._external_paths = set(mozpath.normsep(d) for d in paths) 118 119 self._emitter_time = 0.0 120 self._object_count = 0 121 self._test_files_converter = SupportFilesConverter() 122 123 def summary(self): 124 return ExecutionSummary( 125 "Processed into {object_count:d} build config descriptors in " 126 "{execution_time:.2f}s", 127 execution_time=self._emitter_time, 128 object_count=self._object_count, 129 ) 130 131 def emit(self, output, emitfn=None): 132 """Convert the BuildReader output into data structures. 133 134 The return value from BuildReader.read_topsrcdir() (a generator) is 135 typically fed into this function. 136 """ 137 contexts = {} 138 emitfn = emitfn or self.emit_from_context 139 140 def emit_objs(objs): 141 for o in objs: 142 self._object_count += 1 143 yield o 144 145 for out in output: 146 # Nothing in sub-contexts is currently of interest to us. Filter 147 # them all out. 148 if isinstance(out, SubContext): 149 continue 150 151 if isinstance(out, Context): 152 # Keep all contexts around, we will need them later. 153 contexts[os.path.normcase(out.objdir)] = out 154 155 start = time.monotonic() 156 # We need to expand the generator for the timings to work. 157 objs = list(emitfn(out)) 158 self._emitter_time += time.monotonic() - start 159 160 for o in emit_objs(objs): 161 yield o 162 163 else: 164 raise Exception("Unhandled output type: %s" % type(out)) 165 166 # Don't emit Linkable objects when COMPILE_ENVIRONMENT is not set 167 if self.config.substs.get("COMPILE_ENVIRONMENT"): 168 start = time.monotonic() 169 objs = list(self._emit_libs_derived(contexts)) 170 self._emitter_time += time.monotonic() - start 171 172 for o in emit_objs(objs): 173 yield o 174 175 def _emit_libs_derived(self, contexts): 176 # First aggregate idl sources. 177 webidl_attrs = [ 178 ("GENERATED_EVENTS_WEBIDL_FILES", lambda c: c.generated_events_sources), 179 ("GENERATED_WEBIDL_FILES", lambda c: c.generated_sources), 180 ("PREPROCESSED_TEST_WEBIDL_FILES", lambda c: c.preprocessed_test_sources), 181 ("PREPROCESSED_WEBIDL_FILES", lambda c: c.preprocessed_sources), 182 ("TEST_WEBIDL_FILES", lambda c: c.test_sources), 183 ("WEBIDL_FILES", lambda c: c.sources), 184 ("WEBIDL_EXAMPLE_INTERFACES", lambda c: c.example_interfaces), 185 ] 186 ipdl_attrs = [ 187 ("IPDL_SOURCES", lambda c: c.sources), 188 ("PREPROCESSED_IPDL_SOURCES", lambda c: c.preprocessed_sources), 189 ] 190 xpcom_attrs = [("XPCOM_MANIFESTS", lambda c: c.manifests)] 191 192 idl_sources = {} 193 for root, cls, attrs in ( 194 (self.config.substs.get("WEBIDL_ROOT"), WebIDLCollection, webidl_attrs), 195 (self.config.substs.get("IPDL_ROOT"), IPDLCollection, ipdl_attrs), 196 ( 197 self.config.substs.get("XPCOM_ROOT"), 198 XPCOMComponentManifests, 199 xpcom_attrs, 200 ), 201 ): 202 if root: 203 collection = cls(contexts[os.path.normcase(root)]) 204 for var, src_getter in attrs: 205 src_getter(collection).update(self._idls[var]) 206 207 idl_sources[root] = collection.all_source_files() 208 if isinstance(collection, WebIDLCollection): 209 # Test webidl sources are added here as a somewhat special 210 # case. 211 idl_sources[mozpath.join(root, "test")] = [ 212 s for s in collection.all_test_cpp_basenames() 213 ] 214 215 yield collection 216 217 # Next do FINAL_LIBRARY linkage. 218 for lib in (l for libs in self._libs.values() for l in libs): 219 if not isinstance(lib, (StaticLibrary, RustLibrary)) or not lib.link_into: 220 continue 221 if lib.link_into not in self._libs: 222 raise SandboxValidationError( 223 'FINAL_LIBRARY ("%s") does not match any LIBRARY_NAME' 224 % lib.link_into, 225 contexts[os.path.normcase(lib.objdir)], 226 ) 227 candidates = self._libs[lib.link_into] 228 229 # When there are multiple candidates, but all are in the same 230 # directory and have a different type, we want all of them to 231 # have the library linked. The typical usecase is when building 232 # both a static and a shared library in a directory, and having 233 # that as a FINAL_LIBRARY. 234 if ( 235 len(set(type(l) for l in candidates)) == len(candidates) 236 and len(set(l.objdir for l in candidates)) == 1 237 ): 238 for c in candidates: 239 c.link_library(lib) 240 else: 241 raise SandboxValidationError( 242 'FINAL_LIBRARY ("%s") matches a LIBRARY_NAME defined in ' 243 "multiple places:\n %s" 244 % (lib.link_into, "\n ".join(l.objdir for l in candidates)), 245 contexts[os.path.normcase(lib.objdir)], 246 ) 247 248 # ...and USE_LIBS linkage. 249 for context, obj, variable in self._linkage: 250 self._link_libraries(context, obj, variable, idl_sources) 251 252 def recurse_refs(lib): 253 for o in lib.refs: 254 yield o 255 if isinstance(o, StaticLibrary): 256 yield from recurse_refs(o) 257 258 # Check that all static libraries refering shared libraries in 259 # USE_LIBS are linked into a shared library or program. 260 for lib in self._static_linking_shared: 261 if all(isinstance(o, StaticLibrary) for o in recurse_refs(lib)): 262 shared_libs = sorted( 263 l.basename 264 for l in lib.linked_libraries 265 if isinstance(l, SharedLibrary) 266 ) 267 raise SandboxValidationError( 268 'The static "%s" library is not used in a shared library ' 269 "or a program, but USE_LIBS contains the following shared " 270 "library names:\n %s\n\nMaybe you can remove the " 271 'static "%s" library?' 272 % (lib.basename, "\n ".join(shared_libs), lib.basename), 273 contexts[os.path.normcase(lib.objdir)], 274 ) 275 276 @memoize 277 def rust_libraries(obj): 278 libs = [] 279 for o in obj.linked_libraries: 280 if isinstance(o, (HostRustLibrary, RustLibrary)): 281 libs.append(o) 282 elif isinstance(o, (HostLibrary, StaticLibrary, SandboxedWasmLibrary)): 283 libs.extend(rust_libraries(o)) 284 return libs 285 286 def check_rust_libraries(obj): 287 rust_libs = set(rust_libraries(obj)) 288 if len(rust_libs) <= 1: 289 return 290 if isinstance(obj, (Library, HostLibrary)): 291 what = '"%s" library' % obj.basename 292 else: 293 what = '"%s" program' % obj.name 294 raise SandboxValidationError( 295 "Cannot link the following Rust libraries into the %s:\n" 296 "%s\nOnly one is allowed." 297 % ( 298 what, 299 "\n".join( 300 " - %s" % r.basename 301 for r in sorted(rust_libs, key=lambda r: r.basename) 302 ), 303 ), 304 contexts[os.path.normcase(obj.objdir)], 305 ) 306 307 # Propagate LIBRARY_DEFINES to all child libraries recursively. 308 def propagate_defines(outerlib, defines): 309 outerlib.lib_defines.update(defines) 310 for lib in outerlib.linked_libraries: 311 # Propagate defines only along FINAL_LIBRARY paths, not USE_LIBS 312 # paths. 313 if ( 314 isinstance(lib, StaticLibrary) 315 and lib.link_into == outerlib.basename 316 ): 317 propagate_defines(lib, defines) 318 319 for lib in (l for libs in self._libs.values() for l in libs): 320 if isinstance(lib, Library): 321 propagate_defines(lib, lib.lib_defines) 322 check_rust_libraries(lib) 323 yield lib 324 325 for lib in (l for libs in self._libs.values() for l in libs): 326 lib_defines = list(lib.lib_defines.get_defines()) 327 if lib_defines: 328 objdir_flags = self._compile_flags[lib.objdir] 329 objdir_flags.resolve_flags("LIBRARY_DEFINES", lib_defines) 330 331 objdir_flags = self._compile_as_flags.get(lib.objdir) 332 if objdir_flags: 333 objdir_flags.resolve_flags("LIBRARY_DEFINES", lib_defines) 334 335 for flags_obj in self._compile_flags.values(): 336 yield flags_obj 337 338 for flags_obj in self._compile_as_flags.values(): 339 yield flags_obj 340 341 for obj in self._binaries.values(): 342 if isinstance(obj, Linkable): 343 check_rust_libraries(obj) 344 yield obj 345 346 LIBRARY_NAME_VAR = { 347 "host": "HOST_LIBRARY_NAME", 348 "target": "LIBRARY_NAME", 349 "wasm": "SANDBOXED_WASM_LIBRARY_NAME", 350 } 351 352 ARCH_VAR = {"host": "HOST_OS_ARCH", "target": "OS_TARGET"} 353 354 STDCXXCOMPAT_NAME = {"host": "host_stdc++compat", "target": "stdc++compat"} 355 356 def _link_libraries(self, context, obj, variable, extra_sources): 357 """Add linkage declarations to a given object.""" 358 assert isinstance(obj, Linkable) 359 360 if context.objdir in extra_sources: 361 # All "extra sources" are .cpp for the moment, and happen to come 362 # first in order. 363 obj.sources[".cpp"] = extra_sources[context.objdir] + obj.sources[".cpp"] 364 365 for path in context.get(variable, []): 366 self._link_library(context, obj, variable, path) 367 368 # Link system libraries from OS_LIBS/HOST_OS_LIBS. 369 for lib in context.get(variable.replace("USE", "OS"), []): 370 obj.link_system_library(lib) 371 372 # We have to wait for all the self._link_library calls above to have 373 # happened for obj.cxx_link to be final. 374 # FIXME: Theoretically, HostSharedLibrary shouldn't be here (bug 375 # 1474022). 376 if ( 377 not isinstance( 378 obj, (StaticLibrary, HostLibrary, HostSharedLibrary, BaseRustProgram) 379 ) 380 and obj.cxx_link 381 ): 382 if ( 383 context.config.substs.get("MOZ_STDCXX_COMPAT") 384 and context.config.substs.get(self.ARCH_VAR.get(obj.KIND)) == "Linux" 385 ): 386 self._link_library( 387 context, obj, variable, self.STDCXXCOMPAT_NAME[obj.KIND] 388 ) 389 if obj.KIND == "target": 390 if "pure_virtual" in self._libs: 391 self._link_library(context, obj, variable, "pure_virtual") 392 for lib in context.config.substs.get("STLPORT_LIBS", []): 393 obj.link_system_library(lib) 394 395 def _link_library(self, context, obj, variable, path): 396 force_static = path.startswith("static:") and obj.KIND == "target" 397 if force_static: 398 path = path[7:] 399 name = mozpath.basename(path) 400 dir = mozpath.dirname(path) 401 candidates = [l for l in self._libs[name] if l.KIND == obj.KIND] 402 if dir: 403 if dir.startswith("/"): 404 dir = mozpath.normpath(mozpath.join(obj.topobjdir, dir[1:])) 405 else: 406 dir = mozpath.normpath(mozpath.join(obj.objdir, dir)) 407 dir = mozpath.relpath(dir, obj.topobjdir) 408 candidates = [l for l in candidates if l.relobjdir == dir] 409 if not candidates: 410 # If the given directory is under one of the external 411 # (third party) paths, use a fake library reference to 412 # there. 413 for d in self._external_paths: 414 if dir.startswith("%s/" % d): 415 candidates = [ 416 self._get_external_library(dir, name, force_static) 417 ] 418 break 419 420 if not candidates: 421 raise SandboxValidationError( 422 '%s contains "%s", but there is no "%s" %s in %s.' 423 % (variable, path, name, self.LIBRARY_NAME_VAR[obj.KIND], dir), 424 context, 425 ) 426 427 if len(candidates) > 1: 428 # If there's more than one remaining candidate, it could be 429 # that there are instances for the same library, in static and 430 # shared form. 431 libs = {} 432 for l in candidates: 433 key = mozpath.join(l.relobjdir, l.basename) 434 if force_static: 435 if isinstance(l, StaticLibrary): 436 libs[key] = l 437 else: 438 if key in libs and isinstance(l, SharedLibrary): 439 libs[key] = l 440 if key not in libs: 441 libs[key] = l 442 candidates = list(libs.values()) 443 if force_static and not candidates: 444 if dir: 445 raise SandboxValidationError( 446 '%s contains "static:%s", but there is no static ' 447 '"%s" %s in %s.' 448 % (variable, path, name, self.LIBRARY_NAME_VAR[obj.KIND], dir), 449 context, 450 ) 451 raise SandboxValidationError( 452 '%s contains "static:%s", but there is no static "%s" ' 453 "%s in the tree" 454 % (variable, name, name, self.LIBRARY_NAME_VAR[obj.KIND]), 455 context, 456 ) 457 458 if not candidates: 459 raise SandboxValidationError( 460 '%s contains "%s", which does not match any %s in the tree.' 461 % (variable, path, self.LIBRARY_NAME_VAR[obj.KIND]), 462 context, 463 ) 464 465 elif len(candidates) > 1: 466 paths = (mozpath.join(l.relsrcdir, "moz.build") for l in candidates) 467 raise SandboxValidationError( 468 '%s contains "%s", which matches a %s defined in multiple ' 469 "places:\n %s" 470 % ( 471 variable, 472 path, 473 self.LIBRARY_NAME_VAR[obj.KIND], 474 "\n ".join(paths), 475 ), 476 context, 477 ) 478 479 elif force_static and not isinstance(candidates[0], StaticLibrary): 480 raise SandboxValidationError( 481 '%s contains "static:%s", but there is only a shared "%s" ' 482 "in %s. You may want to add FORCE_STATIC_LIB=True in " 483 '%s/moz.build, or remove "static:".' 484 % ( 485 variable, 486 path, 487 name, 488 candidates[0].relobjdir, 489 candidates[0].relobjdir, 490 ), 491 context, 492 ) 493 494 elif isinstance(obj, StaticLibrary) and isinstance( 495 candidates[0], SharedLibrary 496 ): 497 self._static_linking_shared.add(obj) 498 obj.link_library(candidates[0]) 499 500 @memoize 501 def _get_external_library(self, dir, name, force_static): 502 # Create ExternalStaticLibrary or ExternalSharedLibrary object with a 503 # context more or less truthful about where the external library is. 504 context = Context(config=self.config) 505 context.add_source(mozpath.join(self.config.topsrcdir, dir, "dummy")) 506 if force_static: 507 return ExternalStaticLibrary(context, name) 508 else: 509 return ExternalSharedLibrary(context, name) 510 511 def _parse_and_check_cargo_file(self, context): 512 """Parse the Cargo.toml file in context and return a Python object 513 representation of it. Raise a SandboxValidationError if the Cargo.toml 514 file does not exist. Return a tuple of (config, cargo_file).""" 515 cargo_file = mozpath.join(context.srcdir, "Cargo.toml") 516 if not os.path.exists(cargo_file): 517 raise SandboxValidationError( 518 "No Cargo.toml file found in %s" % cargo_file, context 519 ) 520 with open(cargo_file) as f: 521 content = toml.load(f) 522 523 crate_name = content.get("package", {}).get("name") 524 if not crate_name: 525 raise SandboxValidationError( 526 f"{cargo_file} doesn't contain a crate name?!?", context 527 ) 528 529 hack_name = "mozilla-central-workspace-hack" 530 dep = f'{hack_name} = {{ version = "0.1", features = ["{crate_name}"], optional = true }}' 531 dep_dict = toml.loads(dep)[hack_name] 532 hint = ( 533 "\n\nYou may also need to adjust the build/workspace-hack/Cargo.toml" 534 f" file to add the {crate_name} feature." 535 ) 536 537 workspace_hack = content.get("dependencies", {}).get(hack_name) 538 if not workspace_hack: 539 raise SandboxValidationError( 540 f"{cargo_file} doesn't contain the workspace hack.\n\n" 541 f"Add the following to dependencies:\n{dep}{hint}", 542 context, 543 ) 544 545 if workspace_hack != dep_dict: 546 raise SandboxValidationError( 547 f"{cargo_file} needs an update to its {hack_name} dependency.\n\n" 548 f"Adjust the dependency to:\n{dep}{hint}", 549 context, 550 ) 551 552 return content, cargo_file 553 554 def _verify_deps( 555 self, context, crate_dir, crate_name, dependencies, description="Dependency" 556 ): 557 """Verify that a crate's dependencies all specify local paths.""" 558 for dep_crate_name, values in dependencies.items(): 559 # A simple version number. 560 if isinstance(values, (bytes, str)): 561 raise SandboxValidationError( 562 "%s %s of crate %s does not list a path" 563 % (description, dep_crate_name, crate_name), 564 context, 565 ) 566 567 dep_path = values.get("path", None) 568 if not dep_path: 569 raise SandboxValidationError( 570 "%s %s of crate %s does not list a path" 571 % (description, dep_crate_name, crate_name), 572 context, 573 ) 574 575 # Try to catch the case where somebody listed a 576 # local path for development. 577 if os.path.isabs(dep_path): 578 raise SandboxValidationError( 579 "%s %s of crate %s has a non-relative path" 580 % (description, dep_crate_name, crate_name), 581 context, 582 ) 583 584 if not os.path.exists( 585 mozpath.join(context.config.topsrcdir, crate_dir, dep_path) 586 ): 587 raise SandboxValidationError( 588 "%s %s of crate %s refers to a non-existent path" 589 % (description, dep_crate_name, crate_name), 590 context, 591 ) 592 593 def _rust_library( 594 self, context, libname, static_args, is_gkrust=False, cls=RustLibrary 595 ): 596 # We need to note any Rust library for linking purposes. 597 config, cargo_file = self._parse_and_check_cargo_file(context) 598 crate_name = config["package"]["name"] 599 600 if crate_name != libname: 601 raise SandboxValidationError( 602 "library %s does not match Cargo.toml-defined package %s" 603 % (libname, crate_name), 604 context, 605 ) 606 607 # Check that the [lib.crate-type] field is correct 608 lib_section = config.get("lib", None) 609 if not lib_section: 610 raise SandboxValidationError( 611 "Cargo.toml for %s has no [lib] section" % libname, context 612 ) 613 614 crate_type = lib_section.get("crate-type", None) 615 if not crate_type: 616 raise SandboxValidationError( 617 "Can't determine a crate-type for %s from Cargo.toml" % libname, context 618 ) 619 620 crate_type = crate_type[0] 621 if crate_type != "staticlib": 622 raise SandboxValidationError( 623 "crate-type %s is not permitted for %s" % (crate_type, libname), context 624 ) 625 626 dependencies = set(config.get("dependencies", {}).keys()) 627 628 features = context.get(cls.FEATURES_VAR, []) 629 unique_features = set(features) 630 if len(features) != len(unique_features): 631 raise SandboxValidationError( 632 "features for %s should not contain duplicates: %s" 633 % (libname, features), 634 context, 635 ) 636 637 return cls( 638 context, 639 libname, 640 cargo_file, 641 crate_type, 642 dependencies, 643 features, 644 is_gkrust, 645 **static_args, 646 ) 647 648 def _handle_linkables(self, context, passthru, generated_files): 649 linkables = [] 650 host_linkables = [] 651 wasm_linkables = [] 652 653 def add_program(prog, var): 654 if var.startswith("HOST_"): 655 host_linkables.append(prog) 656 else: 657 linkables.append(prog) 658 659 def check_unique_binary(program, kind): 660 if program in self._binaries: 661 raise SandboxValidationError( 662 'Cannot use "%s" as %s name, ' 663 "because it is already used in %s" 664 % (program, kind, self._binaries[program].relsrcdir), 665 context, 666 ) 667 668 for kind, cls in [("PROGRAM", Program), ("HOST_PROGRAM", HostProgram)]: 669 program = context.get(kind) 670 if program: 671 check_unique_binary(program, kind) 672 self._binaries[program] = cls(context, program) 673 self._linkage.append(( 674 context, 675 self._binaries[program], 676 kind.replace("PROGRAM", "USE_LIBS"), 677 )) 678 add_program(self._binaries[program], kind) 679 680 all_rust_programs = [] 681 for kind, cls in [ 682 ("RUST_PROGRAMS", RustProgram), 683 ("HOST_RUST_PROGRAMS", HostRustProgram), 684 ]: 685 programs = context[kind] 686 if not programs: 687 continue 688 689 all_rust_programs.append((programs, kind, cls)) 690 691 # Verify Rust program definitions. 692 if all_rust_programs: 693 config, cargo_file = self._parse_and_check_cargo_file(context) 694 bin_section = config.get("bin", None) 695 if not bin_section: 696 raise SandboxValidationError( 697 "Cargo.toml in %s has no [bin] section" % context.srcdir, context 698 ) 699 700 defined_binaries = {b["name"] for b in bin_section} 701 702 for programs, kind, cls in all_rust_programs: 703 for program in programs: 704 if program not in defined_binaries: 705 raise SandboxValidationError( 706 "Cannot find Cargo.toml definition for %s" % program, 707 context, 708 ) 709 710 check_unique_binary(program, kind) 711 self._binaries[program] = cls(context, program, cargo_file) 712 self._linkage.append(( 713 context, 714 self._binaries[program], 715 kind.replace("RUST_PROGRAMS", "USE_LIBS"), 716 )) 717 add_program(self._binaries[program], kind) 718 719 for kind, cls in [ 720 ("SIMPLE_PROGRAMS", SimpleProgram), 721 ("CPP_UNIT_TESTS", SimpleProgram), 722 ("HOST_SIMPLE_PROGRAMS", HostSimpleProgram), 723 ]: 724 for program in context[kind]: 725 if program in self._binaries: 726 raise SandboxValidationError( 727 'Cannot use "%s" in %s, ' 728 "because it is already used in %s" 729 % (program, kind, self._binaries[program].relsrcdir), 730 context, 731 ) 732 self._binaries[program] = cls( 733 context, program, is_unit_test=kind == "CPP_UNIT_TESTS" 734 ) 735 self._linkage.append(( 736 context, 737 self._binaries[program], 738 ("HOST_USE_LIBS" if kind == "HOST_SIMPLE_PROGRAMS" else "USE_LIBS"), 739 )) 740 add_program(self._binaries[program], kind) 741 742 host_libname = context.get("HOST_LIBRARY_NAME") 743 libname = context.get("LIBRARY_NAME") 744 745 if host_libname: 746 if host_libname == libname: 747 raise SandboxValidationError( 748 "LIBRARY_NAME and HOST_LIBRARY_NAME must have a different value", 749 context, 750 ) 751 752 is_rust_library = context.get("IS_RUST_LIBRARY") 753 if is_rust_library: 754 lib = self._rust_library(context, host_libname, {}, cls=HostRustLibrary) 755 elif context.get("FORCE_SHARED_LIB"): 756 lib = HostSharedLibrary(context, host_libname) 757 else: 758 lib = HostLibrary(context, host_libname) 759 self._libs[host_libname].append(lib) 760 self._linkage.append((context, lib, "HOST_USE_LIBS")) 761 host_linkables.append(lib) 762 763 final_lib = context.get("FINAL_LIBRARY") 764 if not libname and final_lib: 765 # If no LIBRARY_NAME is given, create one. 766 libname = context.relsrcdir.replace("/", "_") 767 768 static_lib = context.get("FORCE_STATIC_LIB") 769 shared_lib = context.get("FORCE_SHARED_LIB") 770 771 static_name = context.get("STATIC_LIBRARY_NAME") 772 shared_name = context.get("SHARED_LIBRARY_NAME") 773 774 is_framework = context.get("IS_FRAMEWORK") 775 776 soname = context.get("SONAME") 777 778 lib_defines = context.get("LIBRARY_DEFINES") 779 780 wasm_lib = context.get("SANDBOXED_WASM_LIBRARY_NAME") 781 782 shared_args = {} 783 static_args = {} 784 785 if final_lib: 786 if static_lib: 787 raise SandboxValidationError( 788 "FINAL_LIBRARY implies FORCE_STATIC_LIB. Please remove the latter.", 789 context, 790 ) 791 if shared_lib: 792 raise SandboxValidationError( 793 "FINAL_LIBRARY conflicts with FORCE_SHARED_LIB. Please remove one.", 794 context, 795 ) 796 if is_framework: 797 raise SandboxValidationError( 798 "FINAL_LIBRARY conflicts with IS_FRAMEWORK. Please remove one.", 799 context, 800 ) 801 static_args["link_into"] = final_lib 802 static_lib = True 803 804 if libname: 805 if is_framework: 806 if soname: 807 raise SandboxValidationError( 808 "IS_FRAMEWORK conflicts with SONAME. Please remove one.", 809 context, 810 ) 811 shared_lib = True 812 shared_args["variant"] = SharedLibrary.FRAMEWORK 813 814 if not static_lib and not shared_lib: 815 static_lib = True 816 817 if static_name: 818 if not static_lib: 819 raise SandboxValidationError( 820 "STATIC_LIBRARY_NAME requires FORCE_STATIC_LIB", context 821 ) 822 static_args["real_name"] = static_name 823 824 if shared_name: 825 if not shared_lib: 826 raise SandboxValidationError( 827 "SHARED_LIBRARY_NAME requires FORCE_SHARED_LIB", context 828 ) 829 shared_args["real_name"] = shared_name 830 831 if soname: 832 if not shared_lib: 833 raise SandboxValidationError( 834 "SONAME requires FORCE_SHARED_LIB", context 835 ) 836 shared_args["soname"] = soname 837 838 if context.get("NO_EXPAND_LIBS"): 839 if not static_lib: 840 raise SandboxValidationError( 841 "NO_EXPAND_LIBS can only be set for static libraries.", context 842 ) 843 static_args["no_expand_lib"] = True 844 845 if shared_lib and static_lib: 846 if not static_name and not shared_name: 847 raise SandboxValidationError( 848 "Both FORCE_STATIC_LIB and FORCE_SHARED_LIB are True, " 849 "but neither STATIC_LIBRARY_NAME or " 850 "SHARED_LIBRARY_NAME is set. At least one is required.", 851 context, 852 ) 853 if static_name and not shared_name and static_name == libname: 854 raise SandboxValidationError( 855 "Both FORCE_STATIC_LIB and FORCE_SHARED_LIB are True, " 856 "but STATIC_LIBRARY_NAME is the same as LIBRARY_NAME, " 857 "and SHARED_LIBRARY_NAME is unset. Please either " 858 "change STATIC_LIBRARY_NAME or LIBRARY_NAME, or set " 859 "SHARED_LIBRARY_NAME.", 860 context, 861 ) 862 if shared_name and not static_name and shared_name == libname: 863 raise SandboxValidationError( 864 "Both FORCE_STATIC_LIB and FORCE_SHARED_LIB are True, " 865 "but SHARED_LIBRARY_NAME is the same as LIBRARY_NAME, " 866 "and STATIC_LIBRARY_NAME is unset. Please either " 867 "change SHARED_LIBRARY_NAME or LIBRARY_NAME, or set " 868 "STATIC_LIBRARY_NAME.", 869 context, 870 ) 871 if shared_name and static_name and shared_name == static_name: 872 raise SandboxValidationError( 873 "Both FORCE_STATIC_LIB and FORCE_SHARED_LIB are True, " 874 "but SHARED_LIBRARY_NAME is the same as " 875 "STATIC_LIBRARY_NAME. Please change one of them.", 876 context, 877 ) 878 879 symbols_file = context.get("SYMBOLS_FILE") 880 if symbols_file: 881 if not shared_lib: 882 raise SandboxValidationError( 883 "SYMBOLS_FILE can only be used with a SHARED_LIBRARY.", context 884 ) 885 if context.get("DEFFILE"): 886 raise SandboxValidationError( 887 "SYMBOLS_FILE cannot be used along DEFFILE.", context 888 ) 889 if isinstance(symbols_file, SourcePath): 890 if not os.path.exists(symbols_file.full_path): 891 raise SandboxValidationError( 892 "Path specified in SYMBOLS_FILE does not exist: %s " 893 "(resolved to %s)" % (symbols_file, symbols_file.full_path), 894 context, 895 ) 896 shared_args["symbols_file"] = True 897 else: 898 if symbols_file.target_basename not in generated_files: 899 raise SandboxValidationError( 900 ( 901 "Objdir file specified in SYMBOLS_FILE not in " 902 + "GENERATED_FILES: %s" 903 ) 904 % (symbols_file,), 905 context, 906 ) 907 shared_args["symbols_file"] = symbols_file.target_basename 908 909 if shared_lib: 910 lib = SharedLibrary(context, libname, **shared_args) 911 self._libs[libname].append(lib) 912 self._linkage.append((context, lib, "USE_LIBS")) 913 linkables.append(lib) 914 if not lib.installed: 915 generated_files.add(lib.lib_name) 916 if symbols_file and isinstance(symbols_file, SourcePath): 917 script = mozpath.join( 918 mozpath.dirname(mozpath.dirname(__file__)), 919 "action", 920 "generate_symbols_file.py", 921 ) 922 defines = () 923 if lib.defines: 924 defines = lib.defines.get_defines() 925 yield GeneratedFile( 926 context, 927 script, 928 "generate_symbols_file", 929 lib.symbols_file, 930 [symbols_file], 931 defines, 932 required_during_compile=[lib.symbols_file], 933 ) 934 if static_lib: 935 is_rust_library = context.get("IS_RUST_LIBRARY") 936 if is_rust_library: 937 lib = self._rust_library( 938 context, 939 libname, 940 static_args, 941 is_gkrust=bool(context.get("IS_GKRUST")), 942 ) 943 else: 944 lib = StaticLibrary(context, libname, **static_args) 945 self._libs[libname].append(lib) 946 self._linkage.append((context, lib, "USE_LIBS")) 947 linkables.append(lib) 948 949 if lib_defines: 950 if not libname: 951 raise SandboxValidationError( 952 "LIBRARY_DEFINES needs a LIBRARY_NAME to take effect", 953 context, 954 ) 955 lib.lib_defines.update(lib_defines) 956 957 if wasm_lib: 958 if wasm_lib == libname: 959 raise SandboxValidationError( 960 "SANDBOXED_WASM_LIBRARY_NAME and LIBRARY_NAME must have a " 961 "different value.", 962 context, 963 ) 964 if wasm_lib == host_libname: 965 raise SandboxValidationError( 966 "SANDBOXED_WASM_LIBRARY_NAME and HOST_LIBRARY_NAME must " 967 "have a different value.", 968 context, 969 ) 970 if wasm_lib == shared_name: 971 raise SandboxValidationError( 972 "SANDBOXED_WASM_LIBRARY_NAME and SHARED_NAME must have a " 973 "different value.", 974 context, 975 ) 976 if wasm_lib == static_name: 977 raise SandboxValidationError( 978 "SANDBOXED_WASM_LIBRARY_NAME and STATIC_NAME must have a " 979 "different value.", 980 context, 981 ) 982 lib = SandboxedWasmLibrary(context, wasm_lib) 983 self._libs[libname].append(lib) 984 wasm_linkables.append(lib) 985 self._wasm_compile_dirs.add(context.objdir) 986 987 seen = {} 988 for symbol in ("SOURCES", "UNIFIED_SOURCES"): 989 for src in context.get(symbol, []): 990 basename = os.path.splitext(os.path.basename(src))[0] 991 if basename in seen: 992 other_src, where = seen[basename] 993 extra = "" 994 if "UNIFIED_SOURCES" in (symbol, where): 995 extra = " in non-unified builds" 996 raise SandboxValidationError( 997 f"{src} from {symbol} would have the same object name " 998 f"as {other_src} from {where}{extra}.", 999 context, 1000 ) 1001 seen[basename] = (src, symbol) 1002 1003 # Only emit sources if we have linkables defined in the same context. 1004 # Note the linkables are not emitted in this function, but much later, 1005 # after aggregation (because of e.g. USE_LIBS processing). 1006 if not (linkables or host_linkables or wasm_linkables): 1007 return 1008 1009 # TODO: objdirs with only host things in them shouldn't need target 1010 # flags, but there's at least one Makefile.in (in 1011 # build/unix/elfhack) that relies on the value of LDFLAGS being 1012 # passed to one-off rules. 1013 self._compile_dirs.add(context.objdir) 1014 1015 if host_linkables or any( 1016 isinstance(l, (RustLibrary, RustProgram)) for l in linkables 1017 ): 1018 self._host_compile_dirs.add(context.objdir) 1019 1020 sources = defaultdict(list) 1021 gen_sources = defaultdict(list) 1022 all_flags = {} 1023 for symbol in ("SOURCES", "HOST_SOURCES", "UNIFIED_SOURCES", "WASM_SOURCES"): 1024 srcs = sources[symbol] 1025 gen_srcs = gen_sources[symbol] 1026 context_srcs = context.get(symbol, []) 1027 seen_sources = set() 1028 for f in context_srcs: 1029 if f in seen_sources: 1030 raise SandboxValidationError( 1031 "Source file should only be added to %s once: %s" % (symbol, f), 1032 context, 1033 ) 1034 seen_sources.add(f) 1035 full_path = f.full_path 1036 if isinstance(f, SourcePath): 1037 srcs.append(full_path) 1038 else: 1039 assert isinstance(f, Path) 1040 gen_srcs.append(full_path) 1041 if symbol == "SOURCES": 1042 context_flags = context_srcs[f] 1043 if context_flags: 1044 all_flags[full_path] = context_flags 1045 1046 if isinstance(f, SourcePath) and not os.path.exists(full_path): 1047 raise SandboxValidationError( 1048 "File listed in %s does not exist: '%s'" % (symbol, full_path), 1049 context, 1050 ) 1051 1052 # Process the .cpp files generated by IPDL as generated sources within 1053 # the context which declared the IPDL_SOURCES attribute. 1054 ipdl_root = self.config.substs.get("IPDL_ROOT") 1055 for symbol in ("IPDL_SOURCES", "PREPROCESSED_IPDL_SOURCES"): 1056 context_srcs = context.get(symbol, []) 1057 for f in context_srcs: 1058 root, ext = mozpath.splitext(mozpath.basename(f)) 1059 1060 suffix_map = { 1061 ".ipdlh": [".cpp"], 1062 ".ipdl": [".cpp", "Child.cpp", "Parent.cpp"], 1063 } 1064 if ext not in suffix_map: 1065 raise SandboxValidationError( 1066 "Unexpected extension for IPDL source %s" % ext 1067 ) 1068 1069 gen_sources["UNIFIED_SOURCES"].extend( 1070 mozpath.join(ipdl_root, root + suffix) for suffix in suffix_map[ext] 1071 ) 1072 1073 no_pgo = context.get("NO_PGO") 1074 no_pgo_sources = [f for f, flags in all_flags.items() if flags.no_pgo] 1075 if no_pgo: 1076 if no_pgo_sources: 1077 raise SandboxValidationError( 1078 "NO_PGO and SOURCES[...].no_pgo cannot be set at the same time", 1079 context, 1080 ) 1081 passthru.variables["NO_PROFILE_GUIDED_OPTIMIZE"] = no_pgo 1082 if no_pgo_sources: 1083 passthru.variables["NO_PROFILE_GUIDED_OPTIMIZE"] = no_pgo_sources 1084 1085 # A map from "canonical suffixes" for a particular source file 1086 # language to the range of suffixes associated with that language. 1087 # 1088 # We deliberately don't list the canonical suffix in the suffix list 1089 # in the definition; we'll add it in programmatically after defining 1090 # things. 1091 suffix_map = { 1092 ".s": set([".asm"]), 1093 ".c": set(), 1094 ".m": set(), 1095 ".mm": set(), 1096 ".cpp": set([".cc", ".cxx"]), 1097 ".S": set(), 1098 } 1099 1100 # The inverse of the above, mapping suffixes to their canonical suffix. 1101 canonicalized_suffix_map = {} 1102 for suffix, alternatives in suffix_map.items(): 1103 alternatives.add(suffix) 1104 for a in alternatives: 1105 canonicalized_suffix_map[a] = suffix 1106 1107 # A map from moz.build variables to the canonical suffixes of file 1108 # kinds that can be listed therein. 1109 all_suffixes = list(suffix_map.keys()) 1110 varmap = dict( 1111 SOURCES=(Sources, all_suffixes), 1112 HOST_SOURCES=(HostSources, [".c", ".cpp"]), 1113 UNIFIED_SOURCES=(UnifiedSources, [".c", ".mm", ".m", ".cpp"]), 1114 ) 1115 # Only include a WasmSources context if there are any WASM_SOURCES. 1116 # (This is going to matter later because we inject an extra .c file to 1117 # compile with the wasm compiler if, and only if, there are any WASM 1118 # sources.) 1119 if sources["WASM_SOURCES"] or gen_sources["WASM_SOURCES"]: 1120 varmap["WASM_SOURCES"] = (WasmSources, [".c", ".cpp"]) 1121 # Track whether there are any C++ source files. 1122 # Technically this won't do the right thing for SIMPLE_PROGRAMS in 1123 # a directory with mixed C and C++ source, but it's not that important. 1124 cxx_sources = defaultdict(bool) 1125 1126 # Source files to track for linkables associated with this context. 1127 ctxt_sources = defaultdict(lambda: defaultdict(list)) 1128 1129 for variable, (klass, suffixes) in varmap.items(): 1130 # Group static and generated files by their canonical suffixes, and 1131 # ensure we haven't been given filetypes that we don't recognize. 1132 by_canonical_suffix = defaultdict(lambda: {"static": [], "generated": []}) 1133 for srcs, key in ( 1134 (sources[variable], "static"), 1135 (gen_sources[variable], "generated"), 1136 ): 1137 for f in srcs: 1138 canonical_suffix = canonicalized_suffix_map.get( 1139 mozpath.splitext(f)[1] 1140 ) 1141 if canonical_suffix not in suffixes: 1142 raise SandboxValidationError( 1143 "%s has an unknown file type." % f, context 1144 ) 1145 by_canonical_suffix[canonical_suffix][key].append(f) 1146 1147 # Yield an object for each canonical suffix, grouping generated and 1148 # static sources together to allow them to be unified together. 1149 for canonical_suffix in sorted(by_canonical_suffix.keys()): 1150 if canonical_suffix in (".cpp", ".mm"): 1151 cxx_sources[variable] = True 1152 elif canonical_suffix in (".s", ".S"): 1153 self._asm_compile_dirs.add(context.objdir) 1154 src_group = by_canonical_suffix[canonical_suffix] 1155 obj = klass( 1156 context, 1157 src_group["static"], 1158 src_group["generated"], 1159 canonical_suffix, 1160 ) 1161 srcs = list(obj.files) 1162 if isinstance(obj, UnifiedSources) and obj.have_unified_mapping: 1163 srcs = sorted(dict(obj.unified_source_mapping).keys()) 1164 ctxt_sources[variable][canonical_suffix] += srcs 1165 yield obj 1166 1167 if ctxt_sources: 1168 for linkable in linkables: 1169 for target_var in ("SOURCES", "UNIFIED_SOURCES"): 1170 for suffix, srcs in ctxt_sources[target_var].items(): 1171 linkable.sources[suffix] += srcs 1172 for host_linkable in host_linkables: 1173 for suffix, srcs in ctxt_sources["HOST_SOURCES"].items(): 1174 host_linkable.sources[suffix] += srcs 1175 for wasm_linkable in wasm_linkables: 1176 for suffix, srcs in ctxt_sources["WASM_SOURCES"].items(): 1177 wasm_linkable.sources[suffix] += srcs 1178 1179 for f, flags in sorted(all_flags.items()): 1180 if flags.flags: 1181 ext = mozpath.splitext(f)[1] 1182 yield PerSourceFlag(context, f, flags.flags) 1183 1184 # If there are any C++ sources, set all the linkables defined here 1185 # to require the C++ linker. 1186 for vars, linkable_items in ( 1187 (("SOURCES", "UNIFIED_SOURCES"), linkables), 1188 (("HOST_SOURCES",), host_linkables), 1189 ): 1190 for var in vars: 1191 if cxx_sources[var]: 1192 for l in linkable_items: 1193 l.cxx_link = True 1194 break 1195 1196 def emit_from_context(self, context): 1197 """Convert a Context to tree metadata objects. 1198 1199 This is a generator of mozbuild.frontend.data.ContextDerived instances. 1200 """ 1201 1202 # We only want to emit an InstallationTarget if one of the consulted 1203 # variables is defined. Later on, we look up FINAL_TARGET, which has 1204 # the side-effect of populating it. So, we need to do this lookup 1205 # early. 1206 if any(k in context for k in ("FINAL_TARGET", "XPI_NAME", "DIST_SUBDIR")): 1207 yield InstallationTarget(context) 1208 1209 # We always emit a directory traversal descriptor. This is needed by 1210 # the recursive make backend. 1211 yield from self._emit_directory_traversal_from_context(context) 1212 1213 for obj in self._process_xpidl(context): 1214 yield obj 1215 1216 computed_flags = ComputedFlags(context, context["COMPILE_FLAGS"]) 1217 computed_link_flags = ComputedFlags(context, context["LINK_FLAGS"]) 1218 computed_host_flags = ComputedFlags(context, context["HOST_COMPILE_FLAGS"]) 1219 computed_host_link_flags = ComputedFlags(context, context["HOST_LINK_FLAGS"]) 1220 computed_as_flags = ComputedFlags(context, context["ASM_FLAGS"]) 1221 computed_wasm_flags = ComputedFlags(context, context["WASM_FLAGS"]) 1222 1223 # Proxy some variables as-is until we have richer classes to represent 1224 # them. We should aim to keep this set small because it violates the 1225 # desired abstraction of the build definition away from makefiles. 1226 passthru = VariablePassthru(context) 1227 varlist = [ 1228 "DUMP_SYMBOLS_FLAGS", 1229 "EXTRA_DSO_LDOPTS", 1230 "RCFILE", 1231 "RCINCLUDE", 1232 "WIN32_EXE_LDFLAGS", 1233 "USE_EXTENSION_MANIFEST", 1234 "WASM_LIBS", 1235 "XPI_PKGNAME", 1236 "XPI_TESTDIR", 1237 ] 1238 for v in varlist: 1239 if v in context and context[v]: 1240 passthru.variables[v] = context[v] 1241 1242 if "XPI_TESTDIR" in context and "XPI_PKGNAME" not in context: 1243 raise SandboxValidationError("XPI_TESTDIR set but XPI_PKGNAME not set") 1244 1245 if ( 1246 context.config.substs.get("OS_TARGET") == "WINNT" 1247 and context["DELAYLOAD_DLLS"] 1248 ): 1249 if context.config.substs.get("CC_TYPE") != "clang": 1250 context["LDFLAGS"].extend([ 1251 ("-DELAYLOAD:%s" % dll) for dll in context["DELAYLOAD_DLLS"] 1252 ]) 1253 else: 1254 context["LDFLAGS"].extend([ 1255 ("-Wl,-Xlink=-DELAYLOAD:%s" % dll) 1256 for dll in context["DELAYLOAD_DLLS"] 1257 ]) 1258 context["OS_LIBS"].append("delayimp") 1259 1260 for v in ["CMFLAGS", "CMMFLAGS"]: 1261 if v in context and context[v]: 1262 passthru.variables["MOZBUILD_" + v] = context[v] 1263 1264 for v in ["CXXFLAGS", "CFLAGS"]: 1265 if v in context and context[v]: 1266 computed_flags.resolve_flags("MOZBUILD_%s" % v, context[v]) 1267 1268 for v in ["WASM_CFLAGS", "WASM_CXXFLAGS"]: 1269 if v in context and context[v]: 1270 computed_wasm_flags.resolve_flags("MOZBUILD_%s" % v, context[v]) 1271 1272 for v in ["HOST_CXXFLAGS", "HOST_CFLAGS"]: 1273 if v in context and context[v]: 1274 computed_host_flags.resolve_flags("MOZBUILD_%s" % v, context[v]) 1275 1276 if "LDFLAGS" in context and context["LDFLAGS"]: 1277 computed_link_flags.resolve_flags("MOZBUILD", context["LDFLAGS"]) 1278 1279 if "HOST_LDFLAGS" in context and context["HOST_LDFLAGS"]: 1280 computed_host_link_flags.resolve_flags("MOZBUILD", context["HOST_LDFLAGS"]) 1281 1282 # Set link flags according to whether we want a console. 1283 if context.config.substs.get("TARGET_OS") == "WINNT": 1284 if context.get("WINCONSOLE", True): 1285 context["WIN32_EXE_LDFLAGS"] += context.config.substs.get( 1286 "WIN32_CONSOLE_EXE_LDFLAGS", [] 1287 ) 1288 else: 1289 context["WIN32_EXE_LDFLAGS"] += context.config.substs.get( 1290 "WIN32_GUI_EXE_LDFLAGS", [] 1291 ) 1292 1293 deffile = context.get("DEFFILE") 1294 if deffile and context.config.substs.get("OS_TARGET") == "WINNT": 1295 if isinstance(deffile, SourcePath): 1296 if not os.path.exists(deffile.full_path): 1297 raise SandboxValidationError( 1298 "Path specified in DEFFILE does not exist: %s " 1299 "(resolved to %s)" % (deffile, deffile.full_path), 1300 context, 1301 ) 1302 path = mozpath.relpath(deffile.full_path, context.objdir) 1303 else: 1304 path = deffile.target_basename 1305 1306 if context.config.substs.get("CC_TYPE") == "clang-cl": 1307 computed_link_flags.resolve_flags("DEFFILE", ["-DEF:" + path]) 1308 else: 1309 computed_link_flags.resolve_flags("DEFFILE", [path]) 1310 1311 dist_install = context["DIST_INSTALL"] 1312 if dist_install is True: 1313 passthru.variables["DIST_INSTALL"] = True 1314 elif dist_install is False: 1315 passthru.variables["NO_DIST_INSTALL"] = True 1316 1317 # Ideally, this should be done in templates, but this is difficult at 1318 # the moment because USE_STATIC_MSVCRT can be set after a template 1319 # returns. Eventually, with context-based templates, it will be 1320 # possible. 1321 if ( 1322 context.config.substs.get("OS_ARCH") == "WINNT" 1323 and context.config.substs.get("CC_TYPE") == "clang-cl" 1324 ): 1325 use_static_msvcrt = context.get( 1326 "USE_STATIC_MSVCRT" 1327 ) and not context.config.substs.get("MOZ_ASAN") 1328 rtl_flag = "-MT" if use_static_msvcrt else "-MD" 1329 if context.config.substs.get("MOZ_DEBUG") and not context.config.substs.get( 1330 "MOZ_NO_DEBUG_RTL" 1331 ): 1332 rtl_flag += "d" 1333 computed_flags.resolve_flags("RTL", [rtl_flag]) 1334 if not context.config.substs.get("CROSS_COMPILE"): 1335 computed_host_flags.resolve_flags("RTL", [rtl_flag]) 1336 1337 generated_files = set() 1338 localized_generated_files = set() 1339 for obj in self._process_generated_files(context): 1340 for f in obj.outputs: 1341 generated_files.add(f) 1342 if obj.localized: 1343 localized_generated_files.add(f) 1344 yield obj 1345 1346 for path in context["CONFIGURE_SUBST_FILES"]: 1347 sub = self._create_substitution(ConfigFileSubstitution, context, path) 1348 generated_files.add(str(sub.relpath)) 1349 yield sub 1350 1351 for defines_var, cls, backend_flags in ( 1352 ("DEFINES", Defines, (computed_flags, computed_as_flags)), 1353 ("HOST_DEFINES", HostDefines, (computed_host_flags,)), 1354 ("WASM_DEFINES", WasmDefines, (computed_wasm_flags,)), 1355 ): 1356 defines = context.get(defines_var) 1357 if defines: 1358 defines_obj = cls(context, defines) 1359 if isinstance(defines_obj, Defines): 1360 # DEFINES have consumers outside the compile command line, 1361 # HOST_DEFINES do not. 1362 yield defines_obj 1363 else: 1364 # If we don't have explicitly set defines we need to make sure 1365 # initialized values if present end up in computed flags. 1366 defines_obj = cls(context, context[defines_var]) 1367 1368 defines_from_obj = list(defines_obj.get_defines()) 1369 if defines_from_obj: 1370 for flags in backend_flags: 1371 flags.resolve_flags(defines_var, defines_from_obj) 1372 1373 idl_vars = ( 1374 "GENERATED_EVENTS_WEBIDL_FILES", 1375 "GENERATED_WEBIDL_FILES", 1376 "PREPROCESSED_TEST_WEBIDL_FILES", 1377 "PREPROCESSED_WEBIDL_FILES", 1378 "TEST_WEBIDL_FILES", 1379 "WEBIDL_FILES", 1380 "IPDL_SOURCES", 1381 "PREPROCESSED_IPDL_SOURCES", 1382 "XPCOM_MANIFESTS", 1383 ) 1384 for context_var in idl_vars: 1385 for name in context.get(context_var, []): 1386 self._idls[context_var].add(mozpath.join(context.srcdir, name)) 1387 # WEBIDL_EXAMPLE_INTERFACES do not correspond to files. 1388 for name in context.get("WEBIDL_EXAMPLE_INTERFACES", []): 1389 self._idls["WEBIDL_EXAMPLE_INTERFACES"].add(name) 1390 1391 local_includes = [] 1392 for local_include in context.get("LOCAL_INCLUDES", []): 1393 full_path = local_include.full_path 1394 if not isinstance(local_include, ObjDirPath): 1395 if not os.path.exists(full_path): 1396 raise SandboxValidationError( 1397 "Path specified in LOCAL_INCLUDES does not exist: %s (resolved to %s)" 1398 % (local_include, full_path), 1399 context, 1400 ) 1401 if not os.path.isdir(full_path): 1402 raise SandboxValidationError( 1403 "Path specified in LOCAL_INCLUDES " 1404 "is a filename, but a directory is required: %s " 1405 "(resolved to %s)" % (local_include, full_path), 1406 context, 1407 ) 1408 if full_path in {context.config.topsrcdir, context.config.topobjdir}: 1409 raise SandboxValidationError( 1410 "Path specified in LOCAL_INCLUDES " 1411 "(%s) resolves to the topsrcdir or topobjdir (%s), which is " 1412 "not allowed" % (local_include, full_path), 1413 context, 1414 ) 1415 include_obj = LocalInclude(context, local_include) 1416 local_includes.append(include_obj.path.full_path) 1417 yield include_obj 1418 1419 computed_flags.resolve_flags( 1420 "LOCAL_INCLUDES", ["-I%s" % p for p in local_includes] 1421 ) 1422 computed_as_flags.resolve_flags( 1423 "LOCAL_INCLUDES", ["-I%s" % p for p in local_includes] 1424 ) 1425 computed_host_flags.resolve_flags( 1426 "LOCAL_INCLUDES", ["-I%s" % p for p in local_includes] 1427 ) 1428 computed_wasm_flags.resolve_flags( 1429 "LOCAL_INCLUDES", ["-I%s" % p for p in local_includes] 1430 ) 1431 1432 for obj in self._handle_linkables(context, passthru, generated_files): 1433 yield obj 1434 1435 generated_files.update([ 1436 "%s%s" % (k, self.config.substs.get("BIN_SUFFIX", "")) 1437 for k in self._binaries.keys() 1438 ]) 1439 1440 processed_moz_src_files = None 1441 if "MOZ_SRC_FILES" in context: 1442 # We process MOZ_SRC_FILES separately such that any items in the 1443 # list that are in subdirectories automatically get installed to 1444 # the same subdirectories in the objdir. 1445 processed_moz_src_files = HierarchicalStringList() 1446 for file in context.get("MOZ_SRC_FILES"): 1447 if "/" in file: 1448 processed_moz_src_files[mozpath.dirname(file)] += [file] 1449 else: 1450 processed_moz_src_files += [file] 1451 1452 components = [] 1453 for var, cls in ( 1454 ("EXPORTS", Exports), 1455 ("FINAL_TARGET_FILES", FinalTargetFiles), 1456 ("FINAL_TARGET_PP_FILES", FinalTargetPreprocessedFiles), 1457 ("LOCALIZED_FILES", LocalizedFiles), 1458 ("LOCALIZED_PP_FILES", LocalizedPreprocessedFiles), 1459 ("MOZ_SRC_FILES", MozSrcFiles), 1460 ("OBJDIR_FILES", ObjdirFiles), 1461 ("OBJDIR_PP_FILES", ObjdirPreprocessedFiles), 1462 ("TEST_HARNESS_FILES", TestHarnessFiles), 1463 ): 1464 if var == "MOZ_SRC_FILES": 1465 all_files = processed_moz_src_files 1466 else: 1467 all_files = context.get(var) 1468 if not all_files: 1469 continue 1470 if dist_install is False and var != "TEST_HARNESS_FILES": 1471 raise SandboxValidationError( 1472 "%s cannot be used with DIST_INSTALL = False" % var, context 1473 ) 1474 has_prefs = False 1475 has_resources = False 1476 for base, files in all_files.walk(): 1477 if var == "TEST_HARNESS_FILES" and not base: 1478 raise SandboxValidationError( 1479 "Cannot install files to the root of TEST_HARNESS_FILES", 1480 context, 1481 ) 1482 if base == "components": 1483 components.extend(files) 1484 if base == "defaults/pref": 1485 has_prefs = True 1486 if mozpath.split(base)[0] == "res": 1487 has_resources = True 1488 for f in files: 1489 if var in ( 1490 "FINAL_TARGET_PP_FILES", 1491 "OBJDIR_PP_FILES", 1492 "LOCALIZED_PP_FILES", 1493 ) and not isinstance(f, SourcePath): 1494 raise SandboxValidationError( 1495 ("Only source directory paths allowed in " + "%s: %s") 1496 % (var, f), 1497 context, 1498 ) 1499 if var.startswith("LOCALIZED_"): 1500 if isinstance(f, SourcePath): 1501 if f.startswith("en-US/"): 1502 pass 1503 elif "locales/en-US/" in f: 1504 pass 1505 else: 1506 raise SandboxValidationError( 1507 "%s paths must start with `en-US/` or " 1508 "contain `locales/en-US/`: %s" % (var, f), 1509 context, 1510 ) 1511 1512 if not isinstance(f, ObjDirPath): 1513 path = f.full_path 1514 if "*" not in path and not os.path.exists(path): 1515 raise SandboxValidationError( 1516 "File listed in %s does not exist: %s" % (var, path), 1517 context, 1518 ) 1519 else: 1520 # TODO: Bug 1254682 - The '/' check is to allow 1521 # installing files generated from other directories, 1522 # which is done occasionally for tests. However, it 1523 # means we don't fail early if the file isn't actually 1524 # created by the other moz.build file. 1525 if f.target_basename not in generated_files and "/" not in f: 1526 raise SandboxValidationError( 1527 ( 1528 "Objdir file listed in %s not in " 1529 + "GENERATED_FILES: %s" 1530 ) 1531 % (var, f), 1532 context, 1533 ) 1534 1535 if var.startswith("LOCALIZED_"): 1536 # Further require that LOCALIZED_FILES are from 1537 # LOCALIZED_GENERATED_FILES. 1538 if f.target_basename not in localized_generated_files: 1539 raise SandboxValidationError( 1540 ( 1541 "Objdir file listed in %s not in " 1542 + "LOCALIZED_GENERATED_FILES: %s" 1543 ) 1544 % (var, f), 1545 context, 1546 ) 1547 # Additionally, don't allow LOCALIZED_GENERATED_FILES to be used 1548 # in anything *but* LOCALIZED_FILES. 1549 elif f.target_basename in localized_generated_files: 1550 raise SandboxValidationError( 1551 ( 1552 "Outputs of LOCALIZED_GENERATED_FILES cannot " 1553 "be used in %s: %s" 1554 ) 1555 % (var, f), 1556 context, 1557 ) 1558 1559 # Addons (when XPI_NAME is defined) and Applications (when 1560 # DIST_SUBDIR is defined) use a different preferences directory 1561 # (default/preferences) from the one the GRE uses (defaults/pref). 1562 # Hence, we move the files from the latter to the former in that 1563 # case. 1564 if has_prefs and (context.get("XPI_NAME") or context.get("DIST_SUBDIR")): 1565 all_files.defaults.preferences += all_files.defaults.pref 1566 del all_files.defaults._children["pref"] 1567 1568 if has_resources and ( 1569 context.get("DIST_SUBDIR") or context.get("XPI_NAME") 1570 ): 1571 raise SandboxValidationError( 1572 "RESOURCES_FILES cannot be used with DIST_SUBDIR or XPI_NAME.", 1573 context, 1574 ) 1575 1576 yield cls(context, all_files) 1577 1578 for c in components: 1579 if c.endswith(".manifest"): 1580 yield ChromeManifestEntry( 1581 context, 1582 "chrome.manifest", 1583 Manifest("components", mozpath.basename(c)), 1584 ) 1585 1586 if run_tests := context.get("LEGACY_RUN_TESTS", []): 1587 yield LegacyRunTests(context, run_tests) 1588 1589 rust_tests = context.get("RUST_TESTS", []) 1590 if rust_tests: 1591 # TODO: more sophisticated checking of the declared name vs. 1592 # contents of the Cargo.toml file. 1593 features = context.get("RUST_TEST_FEATURES", []) 1594 1595 yield RustTests(context, rust_tests, features) 1596 1597 for obj in self._process_test_manifests(context): 1598 yield obj 1599 1600 for obj in self._process_jar_manifests(context): 1601 yield obj 1602 1603 computed_as_flags.resolve_flags("MOZBUILD", context.get("ASFLAGS")) 1604 1605 if context.get("USE_NASM") is True: 1606 nasm = context.config.substs.get("NASM") 1607 if not nasm: 1608 raise SandboxValidationError("nasm is not available", context) 1609 passthru.variables["AS"] = nasm 1610 passthru.variables["AS_DASH_C_FLAG"] = "" 1611 passthru.variables["ASOUTOPTION"] = "-o " 1612 computed_as_flags.resolve_flags( 1613 "OS", context.config.substs.get("NASM_ASFLAGS", []) 1614 ) 1615 1616 if context.get("USE_INTEGRATED_CLANGCL_AS") is True: 1617 if context.config.substs.get("CC_TYPE") != "clang-cl": 1618 raise SandboxValidationError("clang-cl is not available", context) 1619 passthru.variables["AS"] = context.config.substs.get("CC") 1620 passthru.variables["AS_DASH_C_FLAG"] = "-c" 1621 passthru.variables["ASOUTOPTION"] = "-o " 1622 1623 if passthru.variables: 1624 yield passthru 1625 1626 if context.objdir in self._compile_dirs: 1627 self._compile_flags[context.objdir] = computed_flags 1628 yield computed_link_flags 1629 1630 if context.objdir in self._asm_compile_dirs: 1631 self._compile_as_flags[context.objdir] = computed_as_flags 1632 1633 if context.objdir in self._host_compile_dirs: 1634 yield computed_host_flags 1635 yield computed_host_link_flags 1636 1637 if context.objdir in self._wasm_compile_dirs: 1638 yield computed_wasm_flags 1639 1640 def _create_substitution(self, cls, context, path): 1641 sub = cls(context) 1642 sub.input_path = "%s.in" % path.full_path 1643 sub.output_path = path.translated 1644 sub.relpath = path 1645 1646 return sub 1647 1648 def _process_xpidl(self, context): 1649 # XPIDL source files get processed and turned into .h and .xpt files. 1650 # If there are multiple XPIDL files in a directory, they get linked 1651 # together into a final .xpt, which has the name defined by 1652 # XPIDL_MODULE. 1653 xpidl_module = context["XPIDL_MODULE"] 1654 1655 if not xpidl_module: 1656 if context["XPIDL_SOURCES"]: 1657 raise SandboxValidationError( 1658 "XPIDL_MODULE must be defined if XPIDL_SOURCES is defined.", 1659 context, 1660 ) 1661 return 1662 1663 if not context["XPIDL_SOURCES"]: 1664 raise SandboxValidationError( 1665 "XPIDL_MODULE cannot be defined unless there are XPIDL_SOURCES", 1666 context, 1667 ) 1668 1669 if context["DIST_INSTALL"] is False: 1670 self.log( 1671 logging.WARN, 1672 "mozbuild_warning", 1673 dict(path=context.main_path), 1674 "{path}: DIST_INSTALL = False has no effect on XPIDL_SOURCES.", 1675 ) 1676 1677 for idl in context["XPIDL_SOURCES"]: 1678 if not os.path.exists(idl.full_path): 1679 raise SandboxValidationError( 1680 "File %s from XPIDL_SOURCES does not exist" % idl.full_path, 1681 context, 1682 ) 1683 1684 yield XPIDLModule(context, xpidl_module, context["XPIDL_SOURCES"]) 1685 1686 def _process_generated_files(self, context): 1687 for path in context["CONFIGURE_DEFINE_FILES"]: 1688 script = mozpath.join( 1689 mozpath.dirname(mozpath.dirname(__file__)), 1690 "action", 1691 "process_define_files.py", 1692 ) 1693 yield GeneratedFile( 1694 context, 1695 script, 1696 "process_define_file", 1697 str(path), 1698 [Path(context, path + ".in")], 1699 ) 1700 1701 generated_files = context.get("GENERATED_FILES") or [] 1702 localized_generated_files = context.get("LOCALIZED_GENERATED_FILES") or [] 1703 if not (generated_files or localized_generated_files): 1704 return 1705 1706 for localized, gen in ( 1707 (False, generated_files), 1708 (True, localized_generated_files), 1709 ): 1710 for f in gen: 1711 flags = gen[f] 1712 outputs = f 1713 inputs = [] 1714 if flags.script: 1715 method = "main" 1716 script = SourcePath(context, flags.script).full_path 1717 1718 # Deal with cases like "C:\\path\\to\\script.py:function". 1719 if ".py:" in script: 1720 script, method = script.rsplit(".py:", 1) 1721 script += ".py" 1722 1723 if not os.path.exists(script): 1724 raise SandboxValidationError( 1725 "Script for generating %s does not exist: %s" % (f, script), 1726 context, 1727 ) 1728 if os.path.splitext(script)[1] != ".py": 1729 raise SandboxValidationError( 1730 "Script for generating %s does not end in .py: %s" 1731 % (f, script), 1732 context, 1733 ) 1734 else: 1735 script = None 1736 method = None 1737 1738 for i in flags.inputs: 1739 p = Path(context, i) 1740 if isinstance(p, SourcePath) and not os.path.exists(p.full_path): 1741 raise SandboxValidationError( 1742 "Input for generating %s does not exist: %s" 1743 % (f, p.full_path), 1744 context, 1745 ) 1746 inputs.append(p) 1747 1748 yield GeneratedFile( 1749 context, 1750 script, 1751 method, 1752 outputs, 1753 inputs, 1754 flags.flags, 1755 localized=localized, 1756 force=flags.force, 1757 ) 1758 1759 def _process_test_manifests(self, context): 1760 for prefix, info in TEST_MANIFESTS.items(): 1761 for path, manifest in context.get("%s_MANIFESTS" % prefix, []): 1762 for obj in self._process_test_manifest(context, info, path, manifest): 1763 yield obj 1764 1765 for flavor in REFTEST_FLAVORS: 1766 for path, manifest in context.get("%s_MANIFESTS" % flavor.upper(), []): 1767 for obj in self._process_reftest_manifest( 1768 context, flavor, path, manifest 1769 ): 1770 yield obj 1771 1772 def _process_test_manifest(self, context, info, manifest_path, mpmanifest): 1773 flavor, install_root, install_subdir, package_tests = info 1774 1775 path = manifest_path.full_path 1776 manifest_dir = mozpath.dirname(path) 1777 manifest_reldir = mozpath.dirname( 1778 mozpath.relpath(path, context.config.topsrcdir) 1779 ) 1780 manifest_sources = [ 1781 mozpath.relpath(pth, context.config.topsrcdir) 1782 for pth in mpmanifest.source_files 1783 ] 1784 install_prefix = mozpath.join(install_root, install_subdir) 1785 1786 try: 1787 if not mpmanifest.tests: 1788 raise SandboxValidationError("Empty test manifest: %s" % path, context) 1789 1790 defaults = mpmanifest.manifest_defaults[os.path.normpath(path)] 1791 obj = TestManifest( 1792 context, 1793 path, 1794 mpmanifest, 1795 flavor=flavor, 1796 install_prefix=install_prefix, 1797 relpath=mozpath.join(manifest_reldir, mozpath.basename(path)), 1798 sources=manifest_sources, 1799 dupe_manifest="dupe-manifest" in defaults, 1800 ) 1801 1802 filtered = mpmanifest.tests 1803 1804 missing = [t["name"] for t in filtered if not os.path.exists(t["path"])] 1805 if missing: 1806 raise SandboxValidationError( 1807 "Test manifest (%s) lists " 1808 "test that does not exist: %s" % (path, ", ".join(missing)), 1809 context, 1810 ) 1811 1812 out_dir = mozpath.join(install_prefix, manifest_reldir) 1813 1814 def process_support_files(test): 1815 install_info = self._test_files_converter.convert_support_files( 1816 test, install_root, manifest_dir, out_dir 1817 ) 1818 1819 obj.pattern_installs.extend(install_info.pattern_installs) 1820 for source, dest in install_info.installs: 1821 obj.installs[source] = (dest, False) 1822 obj.external_installs |= install_info.external_installs 1823 for install_path in install_info.deferred_installs: 1824 if all([ 1825 "*" not in install_path, 1826 not os.path.isfile( 1827 mozpath.join(context.config.topsrcdir, install_path[2:]) 1828 ), 1829 install_path not in install_info.external_installs, 1830 ]): 1831 raise SandboxValidationError( 1832 "Error processing test " 1833 "manifest %s: entry in support-files not present " 1834 "in the srcdir: %s" % (path, install_path), 1835 context, 1836 ) 1837 1838 obj.deferred_installs |= install_info.deferred_installs 1839 1840 for test in filtered: 1841 obj.tests.append(test) 1842 1843 # Some test files are compiled and should not be copied into the 1844 # test package. They function as identifiers rather than files. 1845 if package_tests: 1846 manifest_relpath = mozpath.relpath( 1847 test["path"], mozpath.dirname(test["manifest"]) 1848 ) 1849 obj.installs[mozpath.normpath(test["path"])] = ( 1850 (mozpath.join(out_dir, manifest_relpath)), 1851 True, 1852 ) 1853 1854 process_support_files(test) 1855 1856 for path, m_defaults in mpmanifest.manifest_defaults.items(): 1857 process_support_files(m_defaults) 1858 1859 # We also copy manifests into the output directory, 1860 # including manifests from [include:foo] directives. 1861 for mpath in mpmanifest.manifests(): 1862 mpath = mozpath.normpath(mpath) 1863 out_path = mozpath.join(out_dir, mozpath.basename(mpath)) 1864 obj.installs[mpath] = (out_path, False) 1865 1866 # Some manifests reference files that are auto generated as 1867 # part of the build or shouldn't be installed for some 1868 # reason. Here, we prune those files from the install set. 1869 # FUTURE we should be able to detect autogenerated files from 1870 # other build metadata. Once we do that, we can get rid of this. 1871 for f in defaults.get("generated-files", "").split(): 1872 # We re-raise otherwise the stack trace isn't informative. 1873 try: 1874 del obj.installs[mozpath.join(manifest_dir, f)] 1875 except KeyError: 1876 raise SandboxValidationError( 1877 "Error processing test " 1878 "manifest %s: entry in generated-files not present " 1879 "elsewhere in manifest: %s" % (path, f), 1880 context, 1881 ) 1882 1883 yield obj 1884 except (AssertionError, Exception): 1885 raise SandboxValidationError( 1886 "Error processing test " 1887 "manifest file %s: %s" 1888 % (path, "\n".join(traceback.format_exception(*sys.exc_info()))), 1889 context, 1890 ) 1891 1892 def _process_reftest_manifest(self, context, flavor, manifest_path, manifest): 1893 manifest_full_path = manifest_path.full_path 1894 manifest_reldir = mozpath.dirname( 1895 mozpath.relpath(manifest_full_path, context.config.topsrcdir) 1896 ) 1897 1898 # reftest manifests don't come from manifest parser. But they are 1899 # similar enough that we can use the same emitted objects. Note 1900 # that we don't perform any installs for reftests. 1901 obj = TestManifest( 1902 context, 1903 manifest_full_path, 1904 manifest, 1905 flavor=flavor, 1906 install_prefix="%s/" % flavor, 1907 relpath=mozpath.join(manifest_reldir, mozpath.basename(manifest_path)), 1908 ) 1909 obj.tests = list(sorted(manifest.tests, key=lambda t: t["path"])) 1910 1911 yield obj 1912 1913 def _process_jar_manifests(self, context): 1914 jar_manifests = context.get("JAR_MANIFESTS", []) 1915 if len(jar_manifests) > 1: 1916 raise SandboxValidationError( 1917 "While JAR_MANIFESTS is a list, it is currently limited to one value.", 1918 context, 1919 ) 1920 1921 for path in jar_manifests: 1922 yield JARManifest(context, path) 1923 1924 # Temporary test to look for jar.mn files that creep in without using 1925 # the new declaration. Before, we didn't require jar.mn files to 1926 # declared anywhere (they were discovered). This will detect people 1927 # relying on the old behavior. 1928 if os.path.exists(os.path.join(context.srcdir, "jar.mn")): 1929 if "jar.mn" not in jar_manifests: 1930 raise SandboxValidationError( 1931 "A jar.mn exists but it " 1932 "is not referenced in the moz.build file. " 1933 "Please define JAR_MANIFESTS.", 1934 context, 1935 ) 1936 1937 def _emit_directory_traversal_from_context(self, context): 1938 o = DirectoryTraversal(context) 1939 o.dirs = context.get("DIRS", []) 1940 1941 # Some paths have a subconfigure, yet also have a moz.build. Those 1942 # shouldn't end up in self._external_paths. 1943 if o.objdir: 1944 self._external_paths -= {o.relobjdir} 1945 1946 yield o