dex.py (21986B)
1 #!/usr/bin/env python3 2 # 3 # Copyright 2013 The Chromium Authors 4 # Use of this source code is governed by a BSD-style license that can be 5 # found in the LICENSE file. 6 7 import argparse 8 import collections 9 import logging 10 import os 11 import re 12 import shutil 13 import shlex 14 import sys 15 import tempfile 16 import zipfile 17 18 from util import build_utils 19 from util import md5_check 20 import action_helpers # build_utils adds //build to sys.path. 21 import zip_helpers 22 23 24 _DEX_XMX = '2G' # Increase this when __final_dex OOMs. 25 26 DEFAULT_IGNORE_WARNINGS = ( 27 # Warning: Running R8 version main (build engineering), which cannot be 28 # represented as a semantic version. Using an artificial version newer than 29 # any known version for selecting Proguard configurations embedded under 30 # META-INF/. This means that all rules with a '-upto-' qualifier will be 31 # excluded and all rules with a -from- qualifier will be included. 32 r'Running R8 version main', 33 # https://issuetracker.google.com/327611582 34 r'The companion object Companion could not be found', 35 ) 36 37 _MERGE_SERVICE_ENTRIES = ( 38 # Uses ServiceLoader to find all implementing classes, so multiple are 39 # expected. 40 'META-INF/services/androidx.appsearch.app.AppSearchDocumentClassMap', 41 'META-INF/services/kotlinx.coroutines.CoroutineExceptionHandler', 42 'META-INF/services/kotlinx.coroutines.internal.MainDispatcherFactory', 43 ) 44 45 _IGNORE_SERVICE_ENTRIES = ( 46 # ServiceLoader call is used only for ProtoBuf full (non-lite). 47 # BaseGeneratedExtensionRegistryLite$Loader conflicts with 48 # ChromeGeneratedExtensionRegistryLite$Loader. 49 'META-INF/services/com.google.protobuf.GeneratedExtensionRegistryLoader', ) 50 51 INTERFACE_DESUGARING_WARNINGS = (r'default or static interface methods', ) 52 53 _SKIPPED_CLASS_FILE_NAMES = ( 54 'module-info.class', # Explicitly skipped by r8/utils/FileUtils#isClassFile 55 ) 56 57 58 def _ParseArgs(args): 59 args = build_utils.ExpandFileArgs(args) 60 parser = argparse.ArgumentParser() 61 62 action_helpers.add_depfile_arg(parser) 63 parser.add_argument('--output', required=True, help='Dex output path.') 64 parser.add_argument( 65 '--class-inputs', 66 action='append', 67 help='GN-list of .jars with .class files.') 68 parser.add_argument( 69 '--class-inputs-filearg', 70 action='append', 71 help='GN-list of .jars with .class files (added to depfile).') 72 parser.add_argument( 73 '--dex-inputs', action='append', help='GN-list of .jars with .dex files.') 74 parser.add_argument( 75 '--dex-inputs-filearg', 76 action='append', 77 help='GN-list of .jars with .dex files (added to depfile).') 78 parser.add_argument( 79 '--incremental-dir', 80 help='Path of directory to put intermediate dex files.') 81 parser.add_argument('--library', 82 action='store_true', 83 help='Allow numerous dex files within output.') 84 parser.add_argument('--r8-jar-path', required=True, help='Path to R8 jar.') 85 parser.add_argument('--skip-custom-d8', 86 action='store_true', 87 help='When rebuilding the CustomD8 jar, this may be ' 88 'necessary to avoid incompatibility with the new r8 ' 89 'jar.') 90 parser.add_argument('--custom-d8-jar-path', 91 required=True, 92 help='Path to our customized d8 jar.') 93 parser.add_argument('--desugar-dependencies', 94 help='Path to store desugar dependencies.') 95 parser.add_argument('--desugar', action='store_true') 96 parser.add_argument( 97 '--bootclasspath', 98 action='append', 99 help='GN-list of bootclasspath. Needed for --desugar') 100 parser.add_argument('--show-desugar-default-interface-warnings', 101 action='store_true', 102 help='Enable desugaring warnings.') 103 parser.add_argument( 104 '--classpath', 105 action='append', 106 help='GN-list of full classpath. Needed for --desugar') 107 parser.add_argument('--release', 108 action='store_true', 109 help='Run D8 in release mode.') 110 parser.add_argument( 111 '--min-api', help='Minimum Android API level compatibility.') 112 parser.add_argument('--force-enable-assertions', 113 action='store_true', 114 help='Forcefully enable javac generated assertion code.') 115 parser.add_argument('--assertion-handler', 116 help='The class name of the assertion handler class.') 117 parser.add_argument('--warnings-as-errors', 118 action='store_true', 119 help='Treat all warnings as errors.') 120 parser.add_argument('--dump-inputs', 121 action='store_true', 122 help='Use when filing D8 bugs to capture inputs.' 123 ' Stores inputs to d8inputs.zip') 124 options = parser.parse_args(args) 125 126 if options.force_enable_assertions and options.assertion_handler: 127 parser.error('Cannot use both --force-enable-assertions and ' 128 '--assertion-handler') 129 130 options.class_inputs = action_helpers.parse_gn_list(options.class_inputs) 131 options.class_inputs_filearg = action_helpers.parse_gn_list( 132 options.class_inputs_filearg) 133 options.bootclasspath = action_helpers.parse_gn_list(options.bootclasspath) 134 options.classpath = action_helpers.parse_gn_list(options.classpath) 135 options.dex_inputs = action_helpers.parse_gn_list(options.dex_inputs) 136 options.dex_inputs_filearg = action_helpers.parse_gn_list( 137 options.dex_inputs_filearg) 138 139 return options 140 141 142 def CreateStderrFilter(filters): 143 def filter_stderr(output): 144 # Set this when debugging R8 output. 145 if os.environ.get('R8_SHOW_ALL_OUTPUT', '0') != '0': 146 return output 147 148 # All missing definitions are logged as a single warning, but start on a 149 # new line like "Missing class ...". 150 warnings = re.split(r'^(?=Warning|Error|Missing (?:class|field|method))', 151 output, 152 flags=re.MULTILINE) 153 preamble, *warnings = warnings 154 155 combined_pattern = '|'.join(filters) 156 preamble = build_utils.FilterLines(preamble, combined_pattern) 157 158 compiled_re = re.compile(combined_pattern, re.DOTALL) 159 warnings = [w for w in warnings if not compiled_re.search(w)] 160 161 return preamble + ''.join(warnings) 162 163 return filter_stderr 164 165 166 def _RunD8(dex_cmd, input_paths, output_path, warnings_as_errors, 167 show_desugar_default_interface_warnings): 168 dex_cmd = dex_cmd + ['--output', output_path] + input_paths 169 170 # Missing deps can happen for prebuilts that are missing transitive deps 171 # and have set enable_bytecode_checks=false. 172 filters = list(DEFAULT_IGNORE_WARNINGS) 173 if not show_desugar_default_interface_warnings: 174 filters += INTERFACE_DESUGARING_WARNINGS 175 176 stderr_filter = CreateStderrFilter(filters) 177 178 is_debug = logging.getLogger().isEnabledFor(logging.DEBUG) 179 180 # Avoid deleting the flag file when DEX_DEBUG is set in case the flag file 181 # needs to be examined after the build. 182 with tempfile.NamedTemporaryFile(mode='w', delete=not is_debug) as flag_file: 183 # Chosen arbitrarily. Needed to avoid command-line length limits. 184 MAX_ARGS = 50 185 orig_dex_cmd = dex_cmd 186 if len(dex_cmd) > MAX_ARGS: 187 # Add all flags to D8 (anything after the first --) as well as all 188 # positional args at the end to the flag file. 189 for idx, cmd in enumerate(dex_cmd): 190 if cmd.startswith('--'): 191 flag_file.write('\n'.join(dex_cmd[idx:])) 192 flag_file.flush() 193 dex_cmd = dex_cmd[:idx] 194 dex_cmd.append('@' + flag_file.name) 195 break 196 197 # stdout sometimes spams with things like: 198 # Stripped invalid locals information from 1 method. 199 try: 200 build_utils.CheckOutput(dex_cmd, 201 stderr_filter=stderr_filter, 202 fail_on_output=warnings_as_errors) 203 except Exception as e: 204 if isinstance(e, build_utils.CalledProcessError): 205 output = e.output # pylint: disable=no-member 206 if "global synthetic for 'Record desugaring'" in output: 207 sys.stderr.write('Java records are not supported.\n') 208 sys.stderr.write( 209 'See https://chromium.googlesource.com/chromium/src/+/' + 210 'main/styleguide/java/java.md#Records\n') 211 sys.exit(1) 212 if orig_dex_cmd is not dex_cmd: 213 sys.stderr.write('Full command: ' + shlex.join(orig_dex_cmd) + '\n') 214 raise 215 216 217 def _ZipAligned(dex_files, output_path, services_map): 218 """Creates a .dex.jar with 4-byte aligned files. 219 220 Args: 221 dex_files: List of dex files. 222 output_path: The output file in which to write the zip. 223 services_map: map of path->data for META-INF/services 224 """ 225 with zipfile.ZipFile(output_path, 'w') as z: 226 for i, dex_file in enumerate(dex_files): 227 name = 'classes{}.dex'.format(i + 1 if i > 0 else '') 228 zip_helpers.add_to_zip_hermetic(z, name, src_path=dex_file, alignment=4) 229 for path, data in sorted(services_map.items()): 230 zip_helpers.add_to_zip_hermetic(z, path, data=data, alignment=4) 231 232 233 def _CreateServicesMap(service_jars): 234 ret = {} 235 origins = {} 236 for jar_path in service_jars: 237 with zipfile.ZipFile(jar_path, 'r') as z: 238 for n in z.namelist(): 239 if n.startswith('META-INF/services/') and not n.endswith('/'): 240 if n in _IGNORE_SERVICE_ENTRIES: 241 continue 242 old_lines = ret.get(n, '').splitlines() 243 new_lines = z.read(n).decode('utf8').splitlines() 244 old_lines.extend(l for l in new_lines if l not in old_lines) 245 data = '\n'.join(old_lines) + '\n' 246 if _MERGE_SERVICE_ENTRIES or ret.get(n, data) == data: 247 ret[n] = data 248 origins[n] = jar_path 249 else: 250 # We should arguably just concat the files here, but Chrome's own 251 # uses (via ServiceLoaderUtil) all assume only one entry. 252 raise Exception(f"""\ 253 Conflicting contents for: {n} 254 {origins[n]}: 255 {ret[n]} 256 {jar_path}: 257 {data} 258 259 If this entry can be safely ignored (because the ServiceLoader.load() call is \ 260 never hit), update _IGNORE_SERVICE_ENTRIES in dex.py. 261 262 If this service is meant to allow multiple implementations, update \ 263 _MERGE_SERVICE_ENTRIES in dex.py. 264 """) 265 return ret 266 267 268 def _CreateFinalDex(d8_inputs, 269 output, 270 tmp_dir, 271 dex_cmd, 272 options=None, 273 service_jars=None): 274 tmp_dex_output = os.path.join(tmp_dir, 'tmp_dex_output.zip') 275 needs_dexing = not all(f.endswith('.dex') for f in d8_inputs) 276 needs_dexmerge = output.endswith('.dex') or not (options and options.library) 277 services_map = _CreateServicesMap(service_jars or []) 278 if needs_dexing or needs_dexmerge: 279 tmp_dex_dir = os.path.join(tmp_dir, 'tmp_dex_dir') 280 os.mkdir(tmp_dex_dir) 281 282 _RunD8(dex_cmd, d8_inputs, tmp_dex_dir, 283 (not options or options.warnings_as_errors), 284 (options and options.show_desugar_default_interface_warnings)) 285 logging.debug('Performed dex merging') 286 287 dex_files = [os.path.join(tmp_dex_dir, f) for f in os.listdir(tmp_dex_dir)] 288 289 if output.endswith('.dex'): 290 if len(dex_files) > 1: 291 raise Exception('%d files created, expected 1' % len(dex_files)) 292 tmp_dex_output = dex_files[0] 293 else: 294 _ZipAligned(sorted(dex_files), tmp_dex_output, services_map) 295 else: 296 # Skip dexmerger. Just put all incrementals into the .jar individually. 297 _ZipAligned(sorted(d8_inputs), tmp_dex_output, services_map) 298 logging.debug('Quick-zipped %d files', len(d8_inputs)) 299 300 # The dex file is complete and can be moved out of tmp_dir. 301 shutil.move(tmp_dex_output, output) 302 303 304 def _IntermediateDexFilePathsFromInputJars(class_inputs, incremental_dir): 305 """Returns list of intermediate dex file paths, .jar files with services.""" 306 dex_files = [] 307 service_jars = set() 308 for jar in class_inputs: 309 with zipfile.ZipFile(jar, 'r') as z: 310 for subpath in z.namelist(): 311 if _IsClassFile(subpath): 312 subpath = subpath[:-5] + 'dex' 313 dex_files.append(os.path.join(incremental_dir, subpath)) 314 elif subpath.startswith('META-INF/services/'): 315 service_jars.add(jar) 316 return dex_files, sorted(service_jars) 317 318 319 def _DeleteStaleIncrementalDexFiles(dex_dir, dex_files): 320 """Deletes intermediate .dex files that are no longer needed.""" 321 all_files = build_utils.FindInDirectory(dex_dir) 322 desired_files = set(dex_files) 323 for path in all_files: 324 if path not in desired_files: 325 os.unlink(path) 326 327 328 def _ParseDesugarDeps(desugar_dependencies_file): 329 # pylint: disable=line-too-long 330 """Returns a dict of dependent/dependency mapping parsed from the file. 331 332 Example file format: 333 $ tail out/Debug/gen/base/base_java__dex.desugardeps 334 org/chromium/base/task/SingleThreadTaskRunnerImpl.class 335 <- org/chromium/base/task/SingleThreadTaskRunner.class 336 <- org/chromium/base/task/TaskRunnerImpl.class 337 org/chromium/base/task/TaskRunnerImpl.class 338 <- org/chromium/base/task/TaskRunner.class 339 org/chromium/base/task/TaskRunnerImplJni$1.class 340 <- obj/base/jni_java.turbine.jar:org/jni_zero/JniStaticTestMocker.class 341 org/chromium/base/task/TaskRunnerImplJni.class 342 <- org/chromium/base/task/TaskRunnerImpl$Natives.class 343 """ 344 # pylint: enable=line-too-long 345 dependents_from_dependency = collections.defaultdict(set) 346 if desugar_dependencies_file and os.path.exists(desugar_dependencies_file): 347 with open(desugar_dependencies_file, 'r') as f: 348 dependent = None 349 for line in f: 350 line = line.rstrip() 351 if line.startswith(' <- '): 352 dependency = line[len(' <- '):] 353 # Note that this is a reversed mapping from the one in CustomD8.java. 354 dependents_from_dependency[dependency].add(dependent) 355 else: 356 dependent = line 357 return dependents_from_dependency 358 359 360 def _ComputeRequiredDesugarClasses(changes, desugar_dependencies_file, 361 class_inputs, classpath): 362 dependents_from_dependency = _ParseDesugarDeps(desugar_dependencies_file) 363 required_classes = set() 364 # Gather classes that need to be re-desugared from changes in the classpath. 365 for jar in classpath: 366 for subpath in changes.IterChangedSubpaths(jar): 367 dependency = '{}:{}'.format(jar, subpath) 368 required_classes.update(dependents_from_dependency[dependency]) 369 370 for jar in class_inputs: 371 for subpath in changes.IterChangedSubpaths(jar): 372 required_classes.update(dependents_from_dependency[subpath]) 373 374 return required_classes 375 376 377 def _IsClassFile(path): 378 if os.path.basename(path) in _SKIPPED_CLASS_FILE_NAMES: 379 return False 380 return path.endswith('.class') 381 382 383 def _ExtractClassFiles(changes, tmp_dir, class_inputs, required_classes_set): 384 classes_list = [] 385 for jar in class_inputs: 386 if changes: 387 changed_class_list = (set(changes.IterChangedSubpaths(jar)) 388 | required_classes_set) 389 predicate = lambda x: x in changed_class_list and _IsClassFile(x) 390 else: 391 predicate = _IsClassFile 392 393 classes_list.extend( 394 build_utils.ExtractAll(jar, path=tmp_dir, predicate=predicate)) 395 return classes_list 396 397 398 def _CreateIntermediateDexFiles(changes, options, tmp_dir, dex_cmd): 399 # Create temporary directory for classes to be extracted to. 400 tmp_extract_dir = os.path.join(tmp_dir, 'tmp_extract_dir') 401 os.mkdir(tmp_extract_dir) 402 403 # Do a full rebuild when changes occur in non-input files. 404 allowed_changed = set(options.class_inputs) 405 allowed_changed.update(options.dex_inputs) 406 allowed_changed.update(options.classpath) 407 strings_changed = changes.HasStringChanges() 408 non_direct_input_changed = next( 409 (p for p in changes.IterChangedPaths() if p not in allowed_changed), None) 410 411 if strings_changed or non_direct_input_changed: 412 logging.debug('Full dex required: strings_changed=%s path_changed=%s', 413 strings_changed, non_direct_input_changed) 414 changes = None 415 416 if changes is None: 417 required_desugar_classes_set = set() 418 else: 419 required_desugar_classes_set = _ComputeRequiredDesugarClasses( 420 changes, options.desugar_dependencies, options.class_inputs, 421 options.classpath) 422 logging.debug('Class files needing re-desugar: %d', 423 len(required_desugar_classes_set)) 424 class_files = _ExtractClassFiles(changes, tmp_extract_dir, 425 options.class_inputs, 426 required_desugar_classes_set) 427 logging.debug('Extracted class files: %d', len(class_files)) 428 429 # If the only change is deleting a file, class_files will be empty. 430 if class_files: 431 # Dex necessary classes into intermediate dex files. 432 dex_cmd = dex_cmd + ['--intermediate', '--file-per-class-file'] 433 if options.desugar_dependencies and not options.skip_custom_d8: 434 # Adding os.sep to remove the entire prefix. 435 dex_cmd += ['--file-tmp-prefix', tmp_extract_dir + os.sep] 436 if changes is None and os.path.exists(options.desugar_dependencies): 437 # Since incremental dexing only ever adds to the desugar_dependencies 438 # file, whenever full dexes are required the .desugardeps files need to 439 # be manually removed. 440 os.unlink(options.desugar_dependencies) 441 _RunD8(dex_cmd, class_files, options.incremental_dir, 442 options.warnings_as_errors, 443 options.show_desugar_default_interface_warnings) 444 logging.debug('Dexed class files.') 445 446 447 def _OnStaleMd5(changes, options, final_dex_inputs, service_jars, dex_cmd): 448 logging.debug('_OnStaleMd5') 449 with build_utils.TempDir() as tmp_dir: 450 if options.incremental_dir: 451 # Create directory for all intermediate dex files. 452 if not os.path.exists(options.incremental_dir): 453 os.makedirs(options.incremental_dir) 454 455 _DeleteStaleIncrementalDexFiles(options.incremental_dir, final_dex_inputs) 456 logging.debug('Stale files deleted') 457 _CreateIntermediateDexFiles(changes, options, tmp_dir, dex_cmd) 458 459 _CreateFinalDex(final_dex_inputs, 460 options.output, 461 tmp_dir, 462 dex_cmd, 463 options=options, 464 service_jars=service_jars) 465 466 467 def MergeDexForIncrementalInstall(r8_jar_path, src_paths, dest_dex_jar, 468 min_api): 469 dex_cmd = build_utils.JavaCmd(xmx=_DEX_XMX) + [ 470 '-cp', 471 r8_jar_path, 472 'com.android.tools.r8.D8', 473 '--min-api', 474 min_api, 475 ] 476 with build_utils.TempDir() as tmp_dir: 477 _CreateFinalDex(src_paths, 478 dest_dex_jar, 479 tmp_dir, 480 dex_cmd, 481 service_jars=src_paths) 482 483 484 def main(args): 485 build_utils.InitLogging('DEX_DEBUG') 486 options = _ParseArgs(args) 487 488 options.class_inputs += options.class_inputs_filearg 489 options.dex_inputs += options.dex_inputs_filearg 490 491 input_paths = ([ 492 build_utils.JAVA_PATH_FOR_INPUTS, options.r8_jar_path, 493 options.custom_d8_jar_path 494 ] + options.class_inputs + options.dex_inputs) 495 496 depfile_deps = options.class_inputs_filearg + options.dex_inputs_filearg 497 498 output_paths = [options.output] 499 500 track_subpaths_allowlist = [] 501 if options.incremental_dir: 502 final_dex_inputs, service_jars = _IntermediateDexFilePathsFromInputJars( 503 options.class_inputs, options.incremental_dir) 504 output_paths += final_dex_inputs 505 track_subpaths_allowlist += options.class_inputs 506 else: 507 final_dex_inputs = list(options.class_inputs) 508 service_jars = final_dex_inputs 509 service_jars += options.dex_inputs 510 final_dex_inputs += options.dex_inputs 511 512 dex_cmd = build_utils.JavaCmd(xmx=_DEX_XMX) 513 514 if options.dump_inputs: 515 dex_cmd += ['-Dcom.android.tools.r8.dumpinputtofile=d8inputs.zip'] 516 517 if not options.skip_custom_d8: 518 dex_cmd += [ 519 '-cp', 520 '{}:{}'.format(options.r8_jar_path, options.custom_d8_jar_path), 521 'org.chromium.build.CustomD8', 522 ] 523 else: 524 dex_cmd += [ 525 '-cp', 526 options.r8_jar_path, 527 'com.android.tools.r8.D8', 528 ] 529 530 if options.release: 531 dex_cmd += ['--release'] 532 if options.min_api: 533 dex_cmd += ['--min-api', options.min_api] 534 535 if not options.desugar: 536 dex_cmd += ['--no-desugaring'] 537 elif options.classpath: 538 # The classpath is used by D8 to for interface desugaring. 539 if options.desugar_dependencies and not options.skip_custom_d8: 540 dex_cmd += ['--desugar-dependencies', options.desugar_dependencies] 541 if track_subpaths_allowlist: 542 track_subpaths_allowlist += options.classpath 543 depfile_deps += options.classpath 544 input_paths += options.classpath 545 # Still pass the entire classpath in case a new dependency is needed by 546 # desugar, so that desugar_dependencies will be updated for the next build. 547 for path in options.classpath: 548 dex_cmd += ['--classpath', path] 549 550 if options.classpath: 551 dex_cmd += ['--lib', build_utils.JAVA_HOME] 552 for path in options.bootclasspath: 553 dex_cmd += ['--lib', path] 554 depfile_deps += options.bootclasspath 555 input_paths += options.bootclasspath 556 557 558 if options.assertion_handler: 559 dex_cmd += ['--force-assertions-handler:' + options.assertion_handler] 560 if options.force_enable_assertions: 561 dex_cmd += ['--force-enable-assertions'] 562 563 # The changes feature from md5_check allows us to only re-dex the class files 564 # that have changed and the class files that need to be re-desugared by D8. 565 md5_check.CallAndWriteDepfileIfStale( 566 lambda changes: _OnStaleMd5(changes, options, final_dex_inputs, 567 service_jars, dex_cmd), 568 options, 569 input_paths=input_paths, 570 input_strings=dex_cmd + [str(bool(options.incremental_dir))], 571 output_paths=output_paths, 572 pass_changes=True, 573 track_subpaths_allowlist=track_subpaths_allowlist, 574 depfile_deps=depfile_deps) 575 576 577 if __name__ == '__main__': 578 sys.exit(main(sys.argv[1:]))