swiftc.py (22680B)
1 # Copyright 2023 The Chromium Authors 2 # Use of this source code is governed by a BSD-style license that can be 3 # found in the LICENSE file. 4 5 import argparse 6 import collections 7 import contextlib 8 import hashlib 9 import io 10 import json 11 import multiprocessing 12 import os 13 import re 14 import shutil 15 import subprocess 16 import sys 17 import tempfile 18 19 20 class ArgumentForwarder(object): 21 """Class used to abstract forwarding arguments from to the swiftc compiler. 22 23 Arguments: 24 - arg_name: string corresponding to the argument to pass to the compiler 25 - arg_join: function taking the compiler name and returning whether the 26 argument value is attached to the argument or separated 27 - to_swift: function taking the argument value and returning whether it 28 must be passed to the swift compiler 29 - to_clang: function taking the argument value and returning whether it 30 must be passed to the clang compiler 31 """ 32 33 def __init__(self, arg_name, arg_join, to_swift, to_clang): 34 self._arg_name = arg_name 35 self._arg_join = arg_join 36 self._to_swift = to_swift 37 self._to_clang = to_clang 38 39 def forward(self, swiftc_args, values, target_triple): 40 if not values: 41 return 42 43 is_catalyst = target_triple.endswith('macabi') 44 for value in values: 45 if self._to_swift(value): 46 if self._arg_join('swift'): 47 swiftc_args.append(f'{self._arg_name}{value}') 48 else: 49 swiftc_args.append(self._arg_name) 50 swiftc_args.append(value) 51 52 if self._to_clang(value) and not is_catalyst: 53 if self._arg_join('clang'): 54 swiftc_args.append('-Xcc') 55 swiftc_args.append(f'{self._arg_name}{value}') 56 else: 57 swiftc_args.append('-Xcc') 58 swiftc_args.append(self._arg_name) 59 swiftc_args.append('-Xcc') 60 swiftc_args.append(value) 61 62 63 class IncludeArgumentForwarder(ArgumentForwarder): 64 """Argument forwarder for -I and -isystem.""" 65 66 def __init__(self, arg_name): 67 ArgumentForwarder.__init__(self, 68 arg_name, 69 arg_join=lambda _: len(arg_name) == 1, 70 to_swift=lambda _: arg_name != '-isystem', 71 to_clang=lambda _: True) 72 73 74 class FrameworkArgumentForwarder(ArgumentForwarder): 75 """Argument forwarder for -F and -Fsystem.""" 76 77 def __init__(self, arg_name): 78 ArgumentForwarder.__init__(self, 79 arg_name, 80 arg_join=lambda _: len(arg_name) == 1, 81 to_swift=lambda _: True, 82 to_clang=lambda _: True) 83 84 85 class DefineArgumentForwarder(ArgumentForwarder): 86 """Argument forwarder for -D.""" 87 88 def __init__(self, arg_name): 89 ArgumentForwarder.__init__(self, 90 arg_name, 91 arg_join=lambda _: _ == 'clang', 92 to_swift=lambda _: '=' not in _, 93 to_clang=lambda _: True) 94 95 96 # Dictionary mapping argument names to their ArgumentForwarder. 97 ARGUMENT_FORWARDER_FOR_ATTR = ( 98 ('include_dirs', IncludeArgumentForwarder('-I')), 99 ('system_include_dirs', IncludeArgumentForwarder('-isystem')), 100 ('framework_dirs', FrameworkArgumentForwarder('-F')), 101 ('system_framework_dirs', FrameworkArgumentForwarder('-Fsystem')), 102 ('defines', DefineArgumentForwarder('-D')), 103 ) 104 105 # Regexp used to parse #import lines. 106 IMPORT_LINE_REGEXP = re.compile('#import "([^"]*)"') 107 108 109 class FileWriter(contextlib.AbstractContextManager): 110 """ 111 FileWriter is a file-like object that only write data to disk if changed. 112 113 This object implements the context manager protocols and thus can be used 114 in a with-clause. The data is written to disk when the context is exited, 115 and only if the content is different from current file content. 116 117 with FileWriter(path) as stream: 118 stream.write('...') 119 120 If the with-clause ends with an exception, no data is written to the disk 121 and any existing file is left untouched. 122 """ 123 124 def __init__(self, filepath, encoding='utf8'): 125 self._stringio = io.StringIO() 126 self._filepath = filepath 127 self._encoding = encoding 128 129 def __exit__(self, exc_type, exc_value, traceback): 130 if exc_type or exc_value or traceback: 131 return 132 133 new_content = self._stringio.getvalue() 134 if os.path.exists(self._filepath): 135 with open(self._filepath, encoding=self._encoding) as stream: 136 old_content = stream.read() 137 138 if old_content == new_content: 139 return 140 141 with open(self._filepath, 'w', encoding=self._encoding) as stream: 142 stream.write(new_content) 143 144 def write(self, data): 145 self._stringio.write(data) 146 147 148 @contextlib.contextmanager 149 def existing_directory(path): 150 """Returns a context manager wrapping an existing directory.""" 151 yield path 152 153 154 def create_stamp_file(path): 155 """Writes an empty stamp file at path.""" 156 with FileWriter(path) as stream: 157 stream.write('') 158 159 160 def create_build_cache_dir(args, build_signature): 161 """Creates the build cache directory according to `args`. 162 163 This function returns an object that implements the context manager 164 protocol and thus can be used in a with-clause. If -derived-data-dir 165 argument is not used, the returned directory is a temporary directory 166 that will be deleted when the with-clause is exited. 167 """ 168 if not args.derived_data_dir: 169 return tempfile.TemporaryDirectory() 170 171 # The derived data cache can be quite large, so delete any obsolete 172 # files or directories. 173 stamp_name = f'{args.module_name}.stamp' 174 if os.path.isdir(args.derived_data_dir): 175 for name in os.listdir(args.derived_data_dir): 176 if name not in (build_signature, stamp_name): 177 path = os.path.join(args.derived_data_dir, name) 178 if os.path.isdir(path): 179 shutil.rmtree(path) 180 else: 181 os.unlink(path) 182 183 ensure_directory(args.derived_data_dir) 184 create_stamp_file(os.path.join(args.derived_data_dir, stamp_name)) 185 186 return existing_directory( 187 ensure_directory(os.path.join(args.derived_data_dir, build_signature))) 188 189 190 def ensure_directory(path): 191 """Creates directory at `path` if it does not exists.""" 192 if not os.path.isdir(path): 193 os.makedirs(path) 194 return os.path.abspath(path) 195 196 197 def build_signature(env, args): 198 """Generates the build signature from `env` and `args`. 199 200 This allow re-using the derived data dir between builds while still 201 forcing the data to be recreated from scratch in case of significant 202 changes to the build settings (different arguments or tool versions). 203 """ 204 m = hashlib.sha1() 205 for key in sorted(env): 206 if key.endswith('_VERSION') or key == 'DEVELOPER_DIR': 207 m.update(f'{key}={env[key]}'.encode('utf8')) 208 for i, arg in enumerate(args): 209 m.update(f'{i}={arg}'.encode('utf8')) 210 return m.hexdigest() 211 212 213 def generate_source_output_file_map_fragment(args, filename): 214 """Generates source OutputFileMap.json fragment according to `args`. 215 216 Create the fragment for a single .swift source file for OutputFileMap. 217 The output depends on whether -whole-module-optimization argument is 218 used or not. 219 """ 220 assert os.path.splitext(filename)[1] == '.swift', filename 221 basename = os.path.splitext(os.path.basename(filename))[0] 222 rel_name = os.path.join(args.target_out_dir, basename) 223 out_name = rel_name 224 225 fragment = { 226 'index-unit-output-path': f'/{rel_name}.o', 227 'object': f'{out_name}.o', 228 } 229 230 if not args.whole_module_optimization: 231 fragment.update({ 232 'const-values': f'{out_name}.swiftconstvalues', 233 'dependencies': f'{out_name}.d', 234 'diagnostics': f'{out_name}.dia', 235 'swift-dependencies': f'{out_name}.swiftdeps', 236 }) 237 238 return fragment 239 240 241 def generate_module_output_file_map_fragment(args): 242 """Generates module OutputFileMap.json fragment according to `args`. 243 244 Create the fragment for the module itself for OutputFileMap. The output 245 depends on whether -whole-module-optimization argument is used or not. 246 """ 247 out_name = os.path.join(args.target_out_dir, args.module_name) 248 249 if args.whole_module_optimization: 250 fragment = { 251 'const-values': f'{out_name}.swiftconstvalues', 252 'dependencies': f'{out_name}.d', 253 'diagnostics': f'{out_name}.dia', 254 'swift-dependencies': f'{out_name}.swiftdeps', 255 } 256 else: 257 fragment = { 258 'emit-module-dependencies': f'{out_name}.d', 259 'emit-module-diagnostics': f'{out_name}.dia', 260 'swift-dependencies': f'{out_name}.swiftdeps', 261 } 262 263 return fragment 264 265 266 def generate_output_file_map(args): 267 """Generates OutputFileMap.json according to `args`. 268 269 Returns the mapping as a python dictionary that can be serialized to 270 disk as JSON. 271 """ 272 output_file_map = {'': generate_module_output_file_map_fragment(args)} 273 for filename in args.sources: 274 fragment = generate_source_output_file_map_fragment(args, filename) 275 output_file_map[filename] = fragment 276 return output_file_map 277 278 279 def fix_generated_header(header_path, output_path, src_dir, gen_dir): 280 """Fix the Objective-C header generated by the Swift compiler. 281 282 The Swift compiler assumes that the generated Objective-C header will be 283 imported from code compiled with module support enabled (-fmodules). The 284 generated code thus uses @import and provides no fallback if modules are 285 not enabled. 286 287 The Swift compiler also uses absolute path when including the bridging 288 header or another module's generated header. This causes issues with the 289 distributed compiler (i.e. reclient or siso) who expects all paths to be 290 relative to the build directory 291 292 This method fix the generated header to use relative path for #import 293 and to use #import instead of @import when using system frameworks. 294 295 The header is read at `header_path` and written to `output_path`. 296 """ 297 298 header_contents = [] 299 with open(header_path, 'r', encoding='utf8') as header_file: 300 301 imports_section = None 302 for line in header_file: 303 # Handle #import lines. 304 match = IMPORT_LINE_REGEXP.match(line) 305 if match: 306 import_path = match.group(1) 307 for root in (gen_dir, src_dir): 308 if import_path.startswith(root): 309 import_path = os.path.relpath(import_path, root) 310 if import_path != match.group(1): 311 span = match.span(1) 312 line = line[:span[0]] + import_path + line[span[1]:] 313 314 # Handle @import lines. 315 if line.startswith('#if __has_feature(objc_modules)'): 316 assert imports_section is None 317 imports_section = (len(header_contents) + 1, 1) 318 elif imports_section: 319 section_start, nesting_level = imports_section 320 if line.startswith('#if'): 321 imports_section = (section_start, nesting_level + 1) 322 elif line.startswith('#endif'): 323 if nesting_level > 1: 324 imports_section = (section_start, nesting_level - 1) 325 else: 326 imports_section = None 327 section_end = len(header_contents) 328 header_contents.append('#else\n') 329 for index in range(section_start, section_end): 330 l = header_contents[index] 331 if l.startswith('@import'): 332 name = l.split()[1].split(';')[0] 333 if name != 'ObjectiveC': 334 header_contents.append(f'#import <{name}/{name}.h>\n') 335 else: 336 header_contents.append(l) 337 338 header_contents.append(line) 339 340 with FileWriter(output_path) as header_file: 341 for line in header_contents: 342 header_file.write(line) 343 344 345 def invoke_swift_compiler(args, extras_args, build_cache_dir, output_file_map): 346 """Invokes Swift compiler to compile module according to `args`. 347 348 The `build_cache_dir` and `output_file_map` should be path to existing 349 directory to use for writing intermediate build artifact (optionally 350 a temporary directory) and path to $module-OutputFileMap.json file that 351 lists the outputs to generate for the module and each source file. 352 353 If -fix-module-imports argument is passed, the generated header for the 354 module is written to a temporary location and then modified to replace 355 @import by corresponding #import. 356 """ 357 358 # Write the $module.SwiftFileList file. 359 swift_file_list_path = os.path.join(args.target_out_dir, 360 f'{args.module_name}.SwiftFileList') 361 362 with FileWriter(swift_file_list_path) as stream: 363 for filename in sorted(args.sources): 364 stream.write(f'"{filename}"\n') 365 366 header_path = args.header_path 367 if args.fix_generated_header: 368 header_path = os.path.join(build_cache_dir, os.path.basename(header_path)) 369 370 swiftc_args = [ 371 '-parse-as-library', 372 '-module-name', 373 args.module_name, 374 f'@{swift_file_list_path}', 375 '-sdk', 376 args.sdk_path, 377 '-target', 378 args.target_triple, 379 '-swift-version', 380 args.swift_version, 381 '-c', 382 '-output-file-map', 383 output_file_map, 384 '-save-temps', 385 '-no-color-diagnostics', 386 '-serialize-diagnostics', 387 '-emit-dependencies', 388 '-emit-module', 389 '-emit-module-path', 390 os.path.join(args.target_out_dir, f'{args.module_name}.swiftmodule'), 391 '-emit-objc-header', 392 '-emit-objc-header-path', 393 header_path, 394 '-working-directory', 395 os.getcwd(), 396 '-index-store-path', 397 ensure_directory(os.path.join(build_cache_dir, 'Index.noindex')), 398 '-module-cache-path', 399 ensure_directory(os.path.join(build_cache_dir, 'ModuleCache.noindex')), 400 '-pch-output-dir', 401 ensure_directory(os.path.join(build_cache_dir, 'PrecompiledHeaders')), 402 ] 403 404 # Handle optional -bridge-header flag. 405 if args.bridge_header: 406 swiftc_args.extend(('-import-objc-header', args.bridge_header)) 407 408 # Handle swift const values extraction. 409 swiftc_args.extend(['-emit-const-values']) 410 swiftc_args.extend([ 411 '-Xfrontend', 412 '-const-gather-protocols-file', 413 '-Xfrontend', 414 args.const_gather_protocols_file, 415 ]) 416 417 # Handle -I, -F, -isystem, -Fsystem and -D arguments. 418 for (attr_name, forwarder) in ARGUMENT_FORWARDER_FOR_ATTR: 419 forwarder.forward(swiftc_args, getattr(args, attr_name), args.target_triple) 420 421 # Handle -whole-module-optimization flag. 422 num_threads = max(1, multiprocessing.cpu_count() // 2) 423 if args.whole_module_optimization: 424 swiftc_args.extend([ 425 '-whole-module-optimization', 426 '-no-emit-module-separately-wmo', 427 '-num-threads', 428 f'{num_threads}', 429 ]) 430 else: 431 swiftc_args.extend([ 432 '-enable-batch-mode', 433 '-incremental', 434 '-experimental-emit-module-separately', 435 '-disable-cmo', 436 f'-j{num_threads}', 437 ]) 438 439 # Handle -file-prefix-map flag unless --swift-keep-intermediate-files is set. 440 if args.file_prefix_map and not args.swift_keep_intermediate_files: 441 swiftc_args.extend([ 442 '-file-prefix-map', 443 args.file_prefix_map, 444 ]) 445 446 swift_toolchain_path = args.swift_toolchain_path 447 if not swift_toolchain_path: 448 swift_toolchain_path = os.path.join(os.path.dirname(args.sdk_path), 449 'XcodeDefault.xctoolchain') 450 if not os.path.isdir(swift_toolchain_path): 451 swift_toolchain_path = '' 452 453 command = [f'{swift_toolchain_path}/usr/bin/swiftc'] + swiftc_args 454 if extras_args: 455 command.extend(extras_args) 456 457 process = subprocess.Popen(command) 458 process.communicate() 459 460 if process.returncode: 461 sys.exit(process.returncode) 462 463 if args.fix_generated_header: 464 fix_generated_header(header_path, 465 args.header_path, 466 src_dir=os.path.abspath(args.src_dir) + os.path.sep, 467 gen_dir=os.path.abspath(args.gen_dir) + os.path.sep) 468 469 470 def generate_depfile(args, output_file_map): 471 """Generates compilation depfile according to `args`. 472 473 Parses all intermediate depfile generated by the Swift compiler and 474 replaces absolute path by relative paths (since ninja compares paths 475 as strings and does not resolve relative paths to absolute). 476 477 Converts path to the SDK and toolchain files to the sdk/xcode_link 478 symlinks if possible and available. 479 """ 480 xcode_paths = {} 481 if os.path.islink(args.sdk_path): 482 xcode_links = os.path.dirname(args.sdk_path) 483 for link_name in os.listdir(xcode_links): 484 link_path = os.path.join(xcode_links, link_name) 485 if os.path.islink(link_path): 486 xcode_paths[os.path.realpath(link_path) + os.sep] = link_path + os.sep 487 488 out_dir = os.getcwd() + os.path.sep 489 src_dir = os.path.abspath(args.src_dir) + os.path.sep 490 491 depfile_content = collections.defaultdict(set) 492 for value in output_file_map.values(): 493 partial_depfile_path = value.get('dependencies', None) 494 if partial_depfile_path: 495 with open(partial_depfile_path, encoding='utf8') as stream: 496 for line in stream: 497 output, inputs = line.split(' : ', 2) 498 output = os.path.relpath(output, out_dir) 499 500 # The depfile format uses '\' to quote space in filename. Split the 501 # list of file while respecting this convention. 502 for path in re.split(r'(?<!\\) ', inputs): 503 for xcode_path in xcode_paths: 504 if path.startswith(xcode_path): 505 path = xcode_paths[xcode_path] + path[len(xcode_path):] 506 if path.startswith(src_dir) or path.startswith(out_dir): 507 path = os.path.relpath(path, out_dir) 508 depfile_content[output].add(path) 509 510 with FileWriter(args.depfile_path) as stream: 511 for output, inputs in sorted(depfile_content.items()): 512 stream.write(f'{output}: {" ".join(sorted(inputs))}\n') 513 514 515 def compile_module(args, extras_args, build_signature): 516 """Compiles Swift module according to `args`.""" 517 for path in (args.target_out_dir, os.path.dirname(args.header_path)): 518 ensure_directory(path) 519 520 # Write the $module-OutputFileMap.json file. 521 output_file_map = generate_output_file_map(args) 522 output_file_map_path = os.path.join(args.target_out_dir, 523 f'{args.module_name}-OutputFileMap.json') 524 525 with FileWriter(output_file_map_path) as stream: 526 json.dump(output_file_map, stream, indent=' ', sort_keys=True) 527 528 # Invoke Swift compiler. 529 with create_build_cache_dir(args, build_signature) as build_cache_dir: 530 invoke_swift_compiler(args, 531 extras_args, 532 build_cache_dir=build_cache_dir, 533 output_file_map=output_file_map_path) 534 535 # Generate the depfile. 536 generate_depfile(args, output_file_map) 537 538 539 def main(args): 540 parser = argparse.ArgumentParser(allow_abbrev=False, add_help=False) 541 542 # Required arguments. 543 parser.add_argument('--module-name', 544 required=True, 545 help='name of the Swift module') 546 547 parser.add_argument('--src-dir', 548 required=True, 549 help='path to the source directory') 550 551 parser.add_argument('--gen-dir', 552 required=True, 553 help='path to the gen directory root') 554 555 parser.add_argument('--target-out-dir', 556 required=True, 557 help='path to the object directory') 558 559 parser.add_argument('--header-path', 560 required=True, 561 help='path to the generated header file') 562 563 parser.add_argument('--bridge-header', 564 required=True, 565 help='path to the Objective-C bridge header file') 566 567 parser.add_argument('--depfile-path', 568 required=True, 569 help='path to the output dependency file') 570 571 parser.add_argument('--const-gather-protocols-file', 572 required=True, 573 help='path to file containing const values protocols') 574 575 # Optional arguments. 576 parser.add_argument('--derived-data-dir', 577 help='path to the derived data directory') 578 579 parser.add_argument('--fix-generated-header', 580 default=False, 581 action='store_true', 582 help='fix imports in generated header') 583 584 parser.add_argument('--swift-toolchain-path', 585 default='', 586 help='path to the Swift toolchain to use') 587 588 parser.add_argument('--whole-module-optimization', 589 default=False, 590 action='store_true', 591 help='enable whole module optimisation') 592 593 parser.add_argument('--swift-keep-intermediate-files', 594 default=False, 595 action='store_true', 596 help='keep intermediate files') 597 598 # Required arguments (forwarded to the Swift compiler). 599 parser.add_argument('-target', 600 required=True, 601 dest='target_triple', 602 help='generate code for the given target') 603 604 parser.add_argument('-sdk', 605 required=True, 606 dest='sdk_path', 607 help='path to the iOS SDK') 608 609 # Optional arguments (forwarded to the Swift compiler). 610 parser.add_argument('-I', 611 action='append', 612 dest='include_dirs', 613 help='add directory to header search path') 614 615 parser.add_argument('-isystem', 616 action='append', 617 dest='system_include_dirs', 618 help='add directory to system header search path') 619 620 parser.add_argument('-F', 621 action='append', 622 dest='framework_dirs', 623 help='add directory to framework search path') 624 625 parser.add_argument('-Fsystem', 626 action='append', 627 dest='system_framework_dirs', 628 help='add directory to system framework search path') 629 630 parser.add_argument('-D', 631 action='append', 632 dest='defines', 633 help='add preprocessor define') 634 635 parser.add_argument('-swift-version', 636 default='5', 637 help='version of the Swift language') 638 639 parser.add_argument( 640 '-file-prefix-map', 641 help='remap source paths in debug, coverage, and index info') 642 643 # Positional arguments. 644 parser.add_argument('sources', 645 nargs='+', 646 help='Swift source files to compile') 647 648 parsed, extras = parser.parse_known_args(args) 649 compile_module(parsed, extras, build_signature(os.environ, args)) 650 651 652 if __name__ == '__main__': 653 sys.exit(main(sys.argv[1:]))