compile_resources.py (40139B)
1 #!/usr/bin/env python3 2 # 3 # Copyright 2012 The Chromium Authors 4 # Use of this source code is governed by a BSD-style license that can be 5 # found in the LICENSE file. 6 7 """Compile Android resources into an intermediate APK. 8 9 This can also generate an R.txt, and an .srcjar file containing the proper 10 final R.java class for all resource packages the APK depends on. 11 12 This will crunch images with aapt2. 13 """ 14 15 import argparse 16 import collections 17 import contextlib 18 import filecmp 19 import hashlib 20 import logging 21 import os 22 import pathlib 23 import re 24 import shutil 25 import subprocess 26 import sys 27 import textwrap 28 from xml.etree import ElementTree 29 30 from util import build_utils 31 from util import diff_utils 32 from util import manifest_utils 33 from util import parallel 34 from util import protoresources 35 from util import resource_utils 36 import action_helpers # build_utils adds //build to sys.path. 37 import zip_helpers 38 39 40 # Pngs that we shouldn't convert to webp. Please add rationale when updating. 41 _PNG_WEBP_EXCLUSION_PATTERN = re.compile('|'.join([ 42 # Android requires pngs for 9-patch images. 43 r'.*\.9\.png', 44 # Daydream requires pngs for icon files. 45 r'.*daydream_icon_.*\.png' 46 ])) 47 48 49 def _ParseArgs(args): 50 """Parses command line options. 51 52 Returns: 53 An options object as from argparse.ArgumentParser.parse_args() 54 """ 55 parser = argparse.ArgumentParser(description=__doc__) 56 57 input_opts = parser.add_argument_group('Input options') 58 output_opts = parser.add_argument_group('Output options') 59 60 input_opts.add_argument('--include-resources', 61 action='append', 62 required=True, 63 help='Paths to arsc resource files used to link ' 64 'against. Can be specified multiple times.') 65 input_opts.add_argument( 66 '--dependencies-res-zips', 67 default=[], 68 help='Resources zip archives from dependents. Required to ' 69 'resolve @type/foo references into dependent libraries.') 70 input_opts.add_argument( 71 '--extra-res-packages', 72 help='Additional package names to generate R.java files for.') 73 input_opts.add_argument( 74 '--aapt2-path', required=True, help='Path to the Android aapt2 tool.') 75 input_opts.add_argument( 76 '--android-manifest', required=True, help='AndroidManifest.xml path.') 77 input_opts.add_argument( 78 '--r-java-root-package-name', 79 default='base', 80 help='Short package name for this target\'s root R java file (ex. ' 81 'input of "base" would become gen.base_module). Defaults to "base".') 82 group = input_opts.add_mutually_exclusive_group() 83 group.add_argument( 84 '--shared-resources', 85 action='store_true', 86 help='Make all resources in R.java non-final and allow the resource IDs ' 87 'to be reset to a different package index when the apk is loaded by ' 88 'another application at runtime.') 89 group.add_argument( 90 '--app-as-shared-lib', 91 action='store_true', 92 help='Same as --shared-resources, but also ensures all resource IDs are ' 93 'directly usable from the APK loaded as an application.') 94 input_opts.add_argument( 95 '--package-id', 96 type=int, 97 help='Decimal integer representing custom package ID for resources ' 98 '(instead of 127==0x7f). Cannot be used with --shared-resources.') 99 input_opts.add_argument( 100 '--package-name', 101 help='Package name that will be used to create R class.') 102 input_opts.add_argument( 103 '--rename-manifest-package', help='Package name to force AAPT to use.') 104 input_opts.add_argument( 105 '--arsc-package-name', 106 help='Package name to set in manifest of resources.arsc file. This is ' 107 'only used for apks under test.') 108 input_opts.add_argument( 109 '--shared-resources-allowlist', 110 help='An R.txt file acting as a allowlist for resources that should be ' 111 'non-final and have their package ID changed at runtime in R.java. ' 112 'Implies and overrides --shared-resources.') 113 input_opts.add_argument( 114 '--shared-resources-allowlist-locales', 115 default='[]', 116 help='Optional GN-list of locales. If provided, all strings corresponding' 117 ' to this locale list will be kept in the final output for the ' 118 'resources identified through --shared-resources-allowlist, even ' 119 'if --locale-allowlist is being used.') 120 input_opts.add_argument( 121 '--use-resource-ids-path', 122 help='Use resource IDs generated by aapt --emit-ids.') 123 input_opts.add_argument( 124 '--debuggable', 125 action='store_true', 126 help='Whether to add android:debuggable="true".') 127 input_opts.add_argument('--static-library-version', 128 help='Version code for static library.') 129 input_opts.add_argument('--version-code', help='Version code for apk.') 130 input_opts.add_argument('--version-name', help='Version name for apk.') 131 input_opts.add_argument( 132 '--min-sdk-version', required=True, help='android:minSdkVersion for APK.') 133 input_opts.add_argument( 134 '--target-sdk-version', 135 required=True, 136 help="android:targetSdkVersion for APK.") 137 input_opts.add_argument( 138 '--max-sdk-version', 139 help="android:maxSdkVersion expected in AndroidManifest.xml.") 140 input_opts.add_argument( 141 '--manifest-package', help='Package name of the AndroidManifest.xml.') 142 input_opts.add_argument( 143 '--locale-allowlist', 144 default='[]', 145 help='GN list of languages to include. All other language configs will ' 146 'be stripped out. List may include a combination of Android locales ' 147 'or Chrome locales.') 148 input_opts.add_argument( 149 '--resource-exclusion-regex', 150 default='', 151 help='File-based filter for resources (applied before compiling)') 152 input_opts.add_argument( 153 '--resource-exclusion-exceptions', 154 default='[]', 155 help='GN list of globs that say which files to include even ' 156 'when --resource-exclusion-regex is set.') 157 input_opts.add_argument( 158 '--dependencies-res-zip-overlays', 159 help='GN list with subset of --dependencies-res-zips to use overlay ' 160 'semantics for.') 161 input_opts.add_argument( 162 '--values-filter-rules', 163 help='GN list of source_glob:regex for filtering resources after they ' 164 'are compiled. Use this to filter out entries within values/ files.') 165 input_opts.add_argument('--png-to-webp', action='store_true', 166 help='Convert png files to webp format.') 167 168 input_opts.add_argument('--webp-binary', default='', 169 help='Path to the cwebp binary.') 170 input_opts.add_argument( 171 '--webp-cache-dir', help='The directory to store webp image cache.') 172 input_opts.add_argument( 173 '--is-bundle-module', 174 action='store_true', 175 help='Whether resources are being generated for a bundle module.') 176 input_opts.add_argument( 177 '--uses-split', 178 help='Value to set uses-split to in the AndroidManifest.xml.') 179 input_opts.add_argument( 180 '--verification-version-code-offset', 181 help='Subtract this from versionCode for expectation files') 182 input_opts.add_argument( 183 '--verification-library-version-offset', 184 help='Subtract this from static-library version for expectation files') 185 186 action_helpers.add_depfile_arg(output_opts) 187 output_opts.add_argument('--arsc-path', help='Apk output for arsc format.') 188 output_opts.add_argument('--proto-path', help='Apk output for proto format.') 189 output_opts.add_argument( 190 '--info-path', help='Path to output info file for the partial apk.') 191 output_opts.add_argument( 192 '--srcjar-out', 193 help='Path to srcjar to contain generated R.java.') 194 output_opts.add_argument('--r-text-out', 195 help='Path to store the generated R.txt file.') 196 output_opts.add_argument( 197 '--proguard-file', help='Path to proguard.txt generated file.') 198 output_opts.add_argument( 199 '--emit-ids-out', help='Path to file produced by aapt2 --emit-ids.') 200 201 diff_utils.AddCommandLineFlags(parser) 202 options = parser.parse_args(args) 203 204 options.include_resources = action_helpers.parse_gn_list( 205 options.include_resources) 206 options.dependencies_res_zips = action_helpers.parse_gn_list( 207 options.dependencies_res_zips) 208 options.extra_res_packages = action_helpers.parse_gn_list( 209 options.extra_res_packages) 210 options.locale_allowlist = action_helpers.parse_gn_list( 211 options.locale_allowlist) 212 options.shared_resources_allowlist_locales = action_helpers.parse_gn_list( 213 options.shared_resources_allowlist_locales) 214 options.resource_exclusion_exceptions = action_helpers.parse_gn_list( 215 options.resource_exclusion_exceptions) 216 options.dependencies_res_zip_overlays = action_helpers.parse_gn_list( 217 options.dependencies_res_zip_overlays) 218 options.values_filter_rules = action_helpers.parse_gn_list( 219 options.values_filter_rules) 220 221 if not options.arsc_path and not options.proto_path: 222 parser.error('One of --arsc-path or --proto-path is required.') 223 224 if options.package_id and options.shared_resources: 225 parser.error('--package-id and --shared-resources are mutually exclusive') 226 227 if options.static_library_version and (options.static_library_version != 228 options.version_code): 229 assert options.static_library_version == options.version_code, ( 230 f'static_library_version={options.static_library_version} must equal ' 231 f'version_code={options.version_code}. Please verify the version code ' 232 'map for this target is defined correctly.') 233 234 return options 235 236 237 def _IterFiles(root_dir): 238 for root, _, files in os.walk(root_dir): 239 for f in files: 240 yield os.path.join(root, f) 241 242 243 def _RenameLocaleResourceDirs(resource_dirs, path_info): 244 """Rename locale resource directories into standard names when necessary. 245 246 This is necessary to deal with the fact that older Android releases only 247 support ISO 639-1 two-letter codes, and sometimes even obsolete versions 248 of them. 249 250 In practice it means: 251 * 3-letter ISO 639-2 qualifiers are renamed under a corresponding 252 2-letter one. E.g. for Filipino, strings under values-fil/ will be moved 253 to a new corresponding values-tl/ sub-directory. 254 255 * Modern ISO 639-1 codes will be renamed to their obsolete variant 256 for Indonesian, Hebrew and Yiddish (e.g. 'values-in/ -> values-id/). 257 258 * Norwegian macrolanguage strings will be renamed to Bokmal (main 259 Norway language). See http://crbug.com/920960. In practice this 260 means that 'values-no/ -> values-nb/' unless 'values-nb/' already 261 exists. 262 263 * BCP 47 langauge tags will be renamed to an equivalent ISO 639-1 264 locale qualifier if possible (e.g. 'values-b+en+US/ -> values-en-rUS'). 265 266 Args: 267 resource_dirs: list of top-level resource directories. 268 """ 269 for resource_dir in resource_dirs: 270 ignore_dirs = {} 271 for path in _IterFiles(resource_dir): 272 locale = resource_utils.FindLocaleInStringResourceFilePath(path) 273 if not locale: 274 continue 275 cr_locale = resource_utils.ToChromiumLocaleName(locale) 276 if not cr_locale: 277 continue # Unsupported Android locale qualifier!? 278 locale2 = resource_utils.ToAndroidLocaleName(cr_locale) 279 if locale != locale2: 280 path2 = path.replace('/values-%s/' % locale, '/values-%s/' % locale2) 281 if path == path2: 282 raise Exception('Could not substitute locale %s for %s in %s' % 283 (locale, locale2, path)) 284 285 # Ignore rather than rename when the destination resources config 286 # already exists. 287 # e.g. some libraries provide both values-nb/ and values-no/. 288 # e.g. material design provides: 289 # * res/values-rUS/values-rUS.xml 290 # * res/values-b+es+419/values-b+es+419.xml 291 config_dir = os.path.dirname(path2) 292 already_has_renamed_config = ignore_dirs.get(config_dir) 293 if already_has_renamed_config is None: 294 # Cache the result of the first time the directory is encountered 295 # since subsequent encounters will find the directory already exists 296 # (due to the rename). 297 already_has_renamed_config = os.path.exists(config_dir) 298 ignore_dirs[config_dir] = already_has_renamed_config 299 if already_has_renamed_config: 300 continue 301 302 build_utils.MakeDirectory(os.path.dirname(path2)) 303 shutil.move(path, path2) 304 path_info.RegisterRename( 305 os.path.relpath(path, resource_dir), 306 os.path.relpath(path2, resource_dir)) 307 308 309 def _ToAndroidLocales(locale_allowlist): 310 """Converts the list of Chrome locales to Android config locale qualifiers. 311 312 Args: 313 locale_allowlist: A list of Chromium locale names. 314 Returns: 315 A set of matching Android config locale qualifier names. 316 """ 317 ret = set() 318 for locale in locale_allowlist: 319 locale = resource_utils.ToAndroidLocaleName(locale) 320 if locale is None or ('-' in locale and '-r' not in locale): 321 raise Exception('Unsupported Chromium locale name: %s' % locale) 322 ret.add(locale) 323 # Always keep non-regional fall-backs. 324 language = locale.split('-')[0] 325 ret.add(language) 326 327 return ret 328 329 330 def _MoveImagesToNonMdpiFolders(res_root, path_info): 331 """Move images from drawable-*-mdpi-* folders to drawable-* folders. 332 333 Why? http://crbug.com/289843 334 """ 335 for src_dir_name in os.listdir(res_root): 336 src_components = src_dir_name.split('-') 337 if src_components[0] != 'drawable' or 'mdpi' not in src_components: 338 continue 339 src_dir = os.path.join(res_root, src_dir_name) 340 if not os.path.isdir(src_dir): 341 continue 342 dst_components = [c for c in src_components if c != 'mdpi'] 343 assert dst_components != src_components 344 dst_dir_name = '-'.join(dst_components) 345 dst_dir = os.path.join(res_root, dst_dir_name) 346 build_utils.MakeDirectory(dst_dir) 347 for src_file_name in os.listdir(src_dir): 348 src_file = os.path.join(src_dir, src_file_name) 349 dst_file = os.path.join(dst_dir, src_file_name) 350 assert not os.path.lexists(dst_file) 351 shutil.move(src_file, dst_file) 352 path_info.RegisterRename( 353 os.path.relpath(src_file, res_root), 354 os.path.relpath(dst_file, res_root)) 355 356 357 def _DeterminePlatformVersion(aapt2_path, jar_candidates): 358 def maybe_extract_version(j): 359 try: 360 return resource_utils.ExtractBinaryManifestValues(aapt2_path, j) 361 except build_utils.CalledProcessError: 362 return None 363 364 def is_sdk_jar(jar_name): 365 if jar_name in ('android.jar', 'android_system.jar'): 366 return True 367 # Robolectric jar looks a bit different. 368 return 'android-all' in jar_name and 'robolectric' in jar_name 369 370 android_sdk_jars = [ 371 j for j in jar_candidates if is_sdk_jar(os.path.basename(j)) 372 ] 373 extract_all = [maybe_extract_version(j) for j in android_sdk_jars] 374 extract_all = [x for x in extract_all if x] 375 if len(extract_all) == 0: 376 raise Exception( 377 'Unable to find android SDK jar among candidates: %s' 378 % ', '.join(android_sdk_jars)) 379 if len(extract_all) > 1: 380 raise Exception( 381 'Found multiple android SDK jars among candidates: %s' 382 % ', '.join(android_sdk_jars)) 383 platform_version_code, platform_version_name = extract_all.pop()[:2] 384 return platform_version_code, platform_version_name 385 386 387 def _FixManifest(options, temp_dir): 388 """Fix the APK's AndroidManifest.xml. 389 390 This adds any missing namespaces for 'android' and 'tools', and 391 sets certains elements like 'platformBuildVersionCode' or 392 'android:debuggable' depending on the content of |options|. 393 394 Args: 395 options: The command-line arguments tuple. 396 temp_dir: A temporary directory where the fixed manifest will be written to. 397 Returns: 398 Tuple of: 399 * Manifest path within |temp_dir|. 400 * Original package_name. 401 * Manifest package name. 402 """ 403 doc, manifest_node, app_node = manifest_utils.ParseManifest( 404 options.android_manifest) 405 406 # merge_manifest.py also sets package & <uses-sdk>. We may want to ensure 407 # manifest merger is always enabled and remove these command-line arguments. 408 manifest_utils.SetUsesSdk(manifest_node, options.target_sdk_version, 409 options.min_sdk_version, options.max_sdk_version) 410 orig_package = manifest_node.get('package') or options.manifest_package 411 fixed_package = (options.arsc_package_name or options.manifest_package 412 or orig_package) 413 manifest_node.set('package', fixed_package) 414 415 platform_version_code, platform_version_name = _DeterminePlatformVersion( 416 options.aapt2_path, options.include_resources) 417 manifest_node.set('platformBuildVersionCode', platform_version_code) 418 manifest_node.set('platformBuildVersionName', platform_version_name) 419 if options.version_code: 420 manifest_utils.NamespacedSet(manifest_node, 'versionCode', 421 options.version_code) 422 if options.version_name: 423 manifest_utils.NamespacedSet(manifest_node, 'versionName', 424 options.version_name) 425 if options.debuggable: 426 manifest_utils.NamespacedSet(app_node, 'debuggable', 'true') 427 428 if options.uses_split: 429 uses_split = ElementTree.SubElement(manifest_node, 'uses-split') 430 manifest_utils.NamespacedSet(uses_split, 'name', options.uses_split) 431 432 # Make sure the min-sdk condition is not less than the min-sdk of the bundle. 433 for min_sdk_node in manifest_node.iter('{%s}min-sdk' % 434 manifest_utils.DIST_NAMESPACE): 435 dist_value = '{%s}value' % manifest_utils.DIST_NAMESPACE 436 if int(min_sdk_node.get(dist_value)) < int(options.min_sdk_version): 437 min_sdk_node.set(dist_value, options.min_sdk_version) 438 439 debug_manifest_path = os.path.join(temp_dir, 'AndroidManifest.xml') 440 manifest_utils.SaveManifest(doc, debug_manifest_path) 441 return debug_manifest_path, orig_package, fixed_package 442 443 444 def _CreateKeepPredicate(resource_exclusion_regex, 445 resource_exclusion_exceptions): 446 """Return a predicate lambda to determine which resource files to keep. 447 448 Args: 449 resource_exclusion_regex: A regular expression describing all resources 450 to exclude, except if they are mip-maps, or if they are listed 451 in |resource_exclusion_exceptions|. 452 resource_exclusion_exceptions: A list of glob patterns corresponding 453 to exceptions to the |resource_exclusion_regex|. 454 Returns: 455 A lambda that takes a path, and returns true if the corresponding file 456 must be kept. 457 """ 458 predicate = lambda path: os.path.basename(path)[0] != '.' 459 if resource_exclusion_regex == '': 460 # Do not extract dotfiles (e.g. ".gitkeep"). aapt ignores them anyways. 461 return predicate 462 463 # A simple predicate that only removes (returns False for) paths covered by 464 # the exclusion regex or listed as exceptions. 465 return lambda path: ( 466 not re.search(resource_exclusion_regex, path) or 467 build_utils.MatchesGlob(path, resource_exclusion_exceptions)) 468 469 470 def _ComputeSha1(path): 471 with open(path, 'rb') as f: 472 data = f.read() 473 return hashlib.sha1(data).hexdigest() 474 475 476 def _ConvertToWebPSingle(png_path, cwebp_binary, cwebp_version, webp_cache_dir): 477 sha1_hash = _ComputeSha1(png_path) 478 479 # The set of arguments that will appear in the cache key. 480 quality_args = ['-m', '6', '-q', '100', '-lossless'] 481 482 webp_cache_path = os.path.join( 483 webp_cache_dir, '{}-{}-{}'.format(sha1_hash, cwebp_version, 484 ''.join(quality_args))) 485 # No need to add .webp. Android can load images fine without them. 486 webp_path = os.path.splitext(png_path)[0] 487 488 cache_hit = os.path.exists(webp_cache_path) 489 if cache_hit: 490 os.link(webp_cache_path, webp_path) 491 else: 492 # We place the generated webp image to webp_path, instead of in the 493 # webp_cache_dir to avoid concurrency issues. 494 args = [cwebp_binary, png_path, '-o', webp_path, '-quiet'] + quality_args 495 subprocess.check_call(args) 496 497 try: 498 os.link(webp_path, webp_cache_path) 499 except OSError: 500 # Because of concurrent run, a webp image may already exists in 501 # webp_cache_path. 502 pass 503 504 os.remove(png_path) 505 original_dir = os.path.dirname(os.path.dirname(png_path)) 506 rename_tuple = (os.path.relpath(png_path, original_dir), 507 os.path.relpath(webp_path, original_dir)) 508 return rename_tuple, cache_hit 509 510 511 def _ConvertToWebP(cwebp_binary, png_paths, path_info, webp_cache_dir): 512 cwebp_version = subprocess.check_output([cwebp_binary, '-version']).rstrip() 513 shard_args = [(f, ) for f in png_paths 514 if not _PNG_WEBP_EXCLUSION_PATTERN.match(f)] 515 516 build_utils.MakeDirectory(webp_cache_dir) 517 results = parallel.BulkForkAndCall(_ConvertToWebPSingle, 518 shard_args, 519 cwebp_binary=cwebp_binary, 520 cwebp_version=cwebp_version, 521 webp_cache_dir=webp_cache_dir) 522 total_cache_hits = 0 523 for rename_tuple, cache_hit in results: 524 path_info.RegisterRename(*rename_tuple) 525 total_cache_hits += int(cache_hit) 526 527 logging.debug('png->webp cache: %d/%d', total_cache_hits, len(shard_args)) 528 529 530 def _RemoveImageExtensions(directory, path_info): 531 """Remove extensions from image files in the passed directory. 532 533 This reduces binary size but does not affect android's ability to load the 534 images. 535 """ 536 for f in _IterFiles(directory): 537 if (f.endswith('.png') or f.endswith('.webp')) and not f.endswith('.9.png'): 538 path_with_extension = f 539 path_no_extension = os.path.splitext(path_with_extension)[0] 540 if path_no_extension != path_with_extension: 541 shutil.move(path_with_extension, path_no_extension) 542 path_info.RegisterRename( 543 os.path.relpath(path_with_extension, directory), 544 os.path.relpath(path_no_extension, directory)) 545 546 547 def _CompileSingleDep(index, dep_subdir, keep_predicate, aapt2_path, 548 partials_dir): 549 unique_name = '{}_{}'.format(index, os.path.basename(dep_subdir)) 550 partial_path = os.path.join(partials_dir, '{}.zip'.format(unique_name)) 551 552 compile_command = [ 553 aapt2_path, 554 'compile', 555 # TODO(wnwen): Turn this on once aapt2 forces 9-patch to be crunched. 556 # '--no-crunch', 557 '--dir', 558 dep_subdir, 559 '-o', 560 partial_path 561 ] 562 563 # There are resources targeting API-versions lower than our minapi. For 564 # various reasons it's easier to let aapt2 ignore these than for us to 565 # remove them from our build (e.g. it's from a 3rd party library). 566 build_utils.CheckOutput( 567 compile_command, 568 stderr_filter=lambda output: build_utils.FilterLines( 569 output, r'ignoring configuration .* for (styleable|attribute)')) 570 571 # Filtering these files is expensive, so only apply filters to the partials 572 # that have been explicitly targeted. 573 if keep_predicate: 574 logging.debug('Applying .arsc filtering to %s', dep_subdir) 575 protoresources.StripUnwantedResources(partial_path, keep_predicate) 576 return partial_path 577 578 579 def _CreateValuesKeepPredicate(exclusion_rules, dep_subdir): 580 patterns = [ 581 x[1] for x in exclusion_rules 582 if build_utils.MatchesGlob(dep_subdir, [x[0]]) 583 ] 584 if not patterns: 585 return None 586 587 regexes = [re.compile(p) for p in patterns] 588 return lambda x: not any(r.search(x) for r in regexes) 589 590 591 def _CompileDeps(aapt2_path, dep_subdirs, dep_subdir_overlay_set, temp_dir, 592 exclusion_rules): 593 partials_dir = os.path.join(temp_dir, 'partials') 594 build_utils.MakeDirectory(partials_dir) 595 596 job_params = [(i, dep_subdir, 597 _CreateValuesKeepPredicate(exclusion_rules, dep_subdir)) 598 for i, dep_subdir in enumerate(dep_subdirs)] 599 600 # Filtering is slow, so ensure jobs with keep_predicate are started first. 601 job_params.sort(key=lambda x: not x[2]) 602 partials = list( 603 parallel.BulkForkAndCall(_CompileSingleDep, 604 job_params, 605 aapt2_path=aapt2_path, 606 partials_dir=partials_dir)) 607 608 partials_cmd = list() 609 for i, partial in enumerate(partials): 610 dep_subdir = job_params[i][1] 611 if dep_subdir in dep_subdir_overlay_set: 612 partials_cmd += ['-R'] 613 partials_cmd += [partial] 614 return partials_cmd 615 616 617 def _CreateResourceInfoFile(path_info, info_path, dependencies_res_zips): 618 for zip_file in dependencies_res_zips: 619 zip_info_file_path = zip_file + '.info' 620 if os.path.exists(zip_info_file_path): 621 path_info.MergeInfoFile(zip_info_file_path) 622 path_info.Write(info_path) 623 624 625 def _RemoveUnwantedLocalizedStrings(dep_subdirs, options): 626 """Remove localized strings that should not go into the final output. 627 628 Args: 629 dep_subdirs: List of resource dependency directories. 630 options: Command-line options namespace. 631 """ 632 # Collect locale and file paths from the existing subdirs. 633 # The following variable maps Android locale names to 634 # sets of corresponding xml file paths. 635 locale_to_files_map = collections.defaultdict(set) 636 for directory in dep_subdirs: 637 for f in _IterFiles(directory): 638 locale = resource_utils.FindLocaleInStringResourceFilePath(f) 639 if locale: 640 locale_to_files_map[locale].add(f) 641 642 all_locales = set(locale_to_files_map) 643 644 # Set A: wanted locales, either all of them or the 645 # list provided by --locale-allowlist. 646 wanted_locales = all_locales 647 if options.locale_allowlist: 648 wanted_locales = _ToAndroidLocales(options.locale_allowlist) 649 650 # Set B: shared resources locales, which is either set A 651 # or the list provided by --shared-resources-allowlist-locales 652 shared_resources_locales = wanted_locales 653 shared_names_allowlist = set() 654 if options.shared_resources_allowlist_locales: 655 shared_names_allowlist = set( 656 resource_utils.GetRTxtStringResourceNames( 657 options.shared_resources_allowlist)) 658 659 shared_resources_locales = _ToAndroidLocales( 660 options.shared_resources_allowlist_locales) 661 662 # Remove any file that belongs to a locale not covered by 663 # either A or B. 664 removable_locales = (all_locales - wanted_locales - shared_resources_locales) 665 for locale in removable_locales: 666 for path in locale_to_files_map[locale]: 667 os.remove(path) 668 669 # For any locale in B but not in A, only keep the shared 670 # resource strings in each file. 671 for locale in shared_resources_locales - wanted_locales: 672 for path in locale_to_files_map[locale]: 673 resource_utils.FilterAndroidResourceStringsXml( 674 path, lambda x: x in shared_names_allowlist) 675 676 # For any locale in A but not in B, only keep the strings 677 # that are _not_ from shared resources in the file. 678 for locale in wanted_locales - shared_resources_locales: 679 for path in locale_to_files_map[locale]: 680 resource_utils.FilterAndroidResourceStringsXml( 681 path, lambda x: x not in shared_names_allowlist) 682 683 684 def _FilterResourceFiles(dep_subdirs, keep_predicate): 685 # Create a function that selects which resource files should be packaged 686 # into the final output. Any file that does not pass the predicate will 687 # be removed below. 688 png_paths = [] 689 for directory in dep_subdirs: 690 for f in _IterFiles(directory): 691 if not keep_predicate(f): 692 os.remove(f) 693 elif f.endswith('.png'): 694 png_paths.append(f) 695 696 return png_paths 697 698 699 def _PackageApk(options, build): 700 """Compile and link resources with aapt2. 701 702 Args: 703 options: The command-line options. 704 build: BuildContext object. 705 Returns: 706 The manifest package name for the APK. 707 """ 708 logging.debug('Extracting resource .zips') 709 dep_subdirs = [] 710 dep_subdir_overlay_set = set() 711 for dependency_res_zip in options.dependencies_res_zips: 712 extracted_dep_subdirs = resource_utils.ExtractDeps([dependency_res_zip], 713 build.deps_dir) 714 dep_subdirs += extracted_dep_subdirs 715 if dependency_res_zip in options.dependencies_res_zip_overlays: 716 dep_subdir_overlay_set.update(extracted_dep_subdirs) 717 718 logging.debug('Applying locale transformations') 719 path_info = resource_utils.ResourceInfoFile() 720 _RenameLocaleResourceDirs(dep_subdirs, path_info) 721 722 logging.debug('Applying file-based exclusions') 723 keep_predicate = _CreateKeepPredicate(options.resource_exclusion_regex, 724 options.resource_exclusion_exceptions) 725 png_paths = _FilterResourceFiles(dep_subdirs, keep_predicate) 726 727 if options.locale_allowlist or options.shared_resources_allowlist_locales: 728 logging.debug('Applying locale-based string exclusions') 729 _RemoveUnwantedLocalizedStrings(dep_subdirs, options) 730 731 if png_paths and options.png_to_webp: 732 logging.debug('Converting png->webp') 733 _ConvertToWebP(options.webp_binary, png_paths, path_info, 734 options.webp_cache_dir) 735 logging.debug('Applying drawable transformations') 736 for directory in dep_subdirs: 737 _MoveImagesToNonMdpiFolders(directory, path_info) 738 _RemoveImageExtensions(directory, path_info) 739 740 logging.debug('Running aapt2 compile') 741 exclusion_rules = [x.split(':', 1) for x in options.values_filter_rules] 742 partials = _CompileDeps(options.aapt2_path, dep_subdirs, 743 dep_subdir_overlay_set, build.temp_dir, 744 exclusion_rules) 745 746 link_command = [ 747 options.aapt2_path, 748 'link', 749 '--auto-add-overlay', 750 '--no-version-vectors', 751 '--no-xml-namespaces', 752 '--output-text-symbols', 753 build.r_txt_path, 754 ] 755 756 for j in options.include_resources: 757 link_command += ['-I', j] 758 if options.proguard_file: 759 link_command += ['--proguard', build.proguard_path] 760 link_command += ['--proguard-minimal-keep-rules'] 761 if options.emit_ids_out: 762 link_command += ['--emit-ids', build.emit_ids_path] 763 764 # Note: only one of --proto-format, --shared-lib or --app-as-shared-lib 765 # can be used with recent versions of aapt2. 766 if options.shared_resources: 767 link_command.append('--shared-lib') 768 769 if options.package_id: 770 link_command += [ 771 '--package-id', 772 '0x%02x' % options.package_id, 773 '--allow-reserved-package-id', 774 ] 775 776 fixed_manifest, desired_manifest_package_name, fixed_manifest_package = ( 777 _FixManifest(options, build.temp_dir)) 778 if options.rename_manifest_package: 779 desired_manifest_package_name = options.rename_manifest_package 780 781 link_command += [ 782 '--manifest', fixed_manifest, '--rename-manifest-package', 783 desired_manifest_package_name 784 ] 785 786 if options.package_id is not None: 787 package_id = options.package_id 788 elif options.shared_resources: 789 package_id = 0 790 else: 791 package_id = 0x7f 792 _CreateStableIdsFile(options.use_resource_ids_path, build.stable_ids_path, 793 fixed_manifest_package, package_id) 794 link_command += ['--stable-ids', build.stable_ids_path] 795 796 link_command += partials 797 798 # We always create a binary arsc file first, then convert to proto, so flags 799 # such as --shared-lib can be supported. 800 link_command += ['-o', build.arsc_path] 801 802 logging.debug('Starting: aapt2 link') 803 link_proc = subprocess.Popen(link_command) 804 805 # Create .res.info file in parallel. 806 if options.info_path: 807 logging.debug('Creating .res.info file') 808 _CreateResourceInfoFile(path_info, build.info_path, 809 options.dependencies_res_zips) 810 811 exit_code = link_proc.wait() 812 assert exit_code == 0, f'aapt2 link cmd failed with {exit_code=}' 813 logging.debug('Finished: aapt2 link') 814 815 if options.shared_resources: 816 logging.debug('Resolving styleables in R.txt') 817 # Need to resolve references because unused resource removal tool does not 818 # support references in R.txt files. 819 resource_utils.ResolveStyleableReferences(build.r_txt_path) 820 821 if exit_code: 822 raise subprocess.CalledProcessError(exit_code, link_command) 823 824 if options.proguard_file and (options.shared_resources 825 or options.app_as_shared_lib): 826 # Make sure the R class associated with the manifest package does not have 827 # its onResourcesLoaded method obfuscated or removed, so that the framework 828 # can call it in the case where the APK is being loaded as a library. 829 with open(build.proguard_path, 'a') as proguard_file: 830 keep_rule = ''' 831 -keep,allowoptimization class {package}.R {{ 832 public static void onResourcesLoaded(int); 833 }} 834 '''.format(package=desired_manifest_package_name) 835 proguard_file.write(textwrap.dedent(keep_rule)) 836 837 logging.debug('Running aapt2 convert') 838 build_utils.CheckOutput([ 839 options.aapt2_path, 'convert', '--output-format', 'proto', '-o', 840 build.proto_path, build.arsc_path 841 ]) 842 843 # Workaround for b/147674078. This is only needed for WebLayer and does not 844 # affect WebView usage, since WebView does not used dynamic attributes. 845 if options.shared_resources: 846 logging.debug('Hardcoding dynamic attributes') 847 protoresources.HardcodeSharedLibraryDynamicAttributes( 848 build.proto_path, options.is_bundle_module, 849 options.shared_resources_allowlist) 850 851 build_utils.CheckOutput([ 852 options.aapt2_path, 'convert', '--output-format', 'binary', '-o', 853 build.arsc_path, build.proto_path 854 ]) 855 856 # Sanity check that the created resources have the expected package ID. 857 logging.debug('Performing sanity check') 858 _, actual_package_id = resource_utils.ExtractArscPackage( 859 options.aapt2_path, 860 build.arsc_path if options.arsc_path else build.proto_path) 861 # When there are no resources, ExtractArscPackage returns (None, None), in 862 # this case there is no need to check for matching package ID. 863 if actual_package_id is not None and actual_package_id != package_id: 864 raise Exception('Invalid package ID 0x%x (expected 0x%x)' % 865 (actual_package_id, package_id)) 866 867 return desired_manifest_package_name 868 869 870 def _CreateStableIdsFile(in_path, out_path, package_name, package_id): 871 """Transforms a file generated by --emit-ids from another package. 872 873 --stable-ids is generally meant to be used by different versions of the same 874 package. To make it work for other packages, we need to transform the package 875 name references to match the package that resources are being generated for. 876 """ 877 if in_path: 878 data = pathlib.Path(in_path).read_text() 879 else: 880 # Force IDs to use 0x01 for the type byte in order to ensure they are 881 # different from IDs generated by other apps. https://crbug.com/1293336 882 data = 'pkg:id/fake_resource_id = 0x7f010000\n' 883 # Replace "pkg:" with correct package name. 884 data = re.sub(r'^.*?:', package_name + ':', data, flags=re.MULTILINE) 885 # Replace "0x7f" with correct package id. 886 data = re.sub(r'0x..', '0x%02x' % package_id, data) 887 pathlib.Path(out_path).write_text(data) 888 889 890 def _WriteOutputs(options, build): 891 possible_outputs = [ 892 (options.srcjar_out, build.srcjar_path), 893 (options.r_text_out, build.r_txt_path), 894 (options.arsc_path, build.arsc_path), 895 (options.proto_path, build.proto_path), 896 (options.proguard_file, build.proguard_path), 897 (options.emit_ids_out, build.emit_ids_path), 898 (options.info_path, build.info_path), 899 ] 900 901 for final, temp in possible_outputs: 902 # Write file only if it's changed. 903 if final and not (os.path.exists(final) and filecmp.cmp(final, temp)): 904 shutil.move(temp, final) 905 906 907 def _CreateNormalizedManifestForVerification(options): 908 with build_utils.TempDir() as tempdir: 909 fixed_manifest, _, _ = _FixManifest(options, tempdir) 910 with open(fixed_manifest) as f: 911 return manifest_utils.NormalizeManifest( 912 f.read(), options.verification_version_code_offset, 913 options.verification_library_version_offset) 914 915 916 def main(args): 917 build_utils.InitLogging('RESOURCE_DEBUG') 918 args = build_utils.ExpandFileArgs(args) 919 options = _ParseArgs(args) 920 921 if options.expected_file: 922 actual_data = _CreateNormalizedManifestForVerification(options) 923 diff_utils.CheckExpectations(actual_data, options) 924 if options.only_verify_expectations: 925 return 926 927 path = options.arsc_path or options.proto_path 928 debug_temp_resources_dir = os.environ.get('TEMP_RESOURCES_DIR') 929 if debug_temp_resources_dir: 930 path = os.path.join(debug_temp_resources_dir, os.path.basename(path)) 931 else: 932 # Use a deterministic temp directory since .pb files embed the absolute 933 # path of resources: crbug.com/939984 934 path = path + '.tmpdir' 935 build_utils.DeleteDirectory(path) 936 937 with resource_utils.BuildContext( 938 temp_dir=path, keep_files=bool(debug_temp_resources_dir)) as build: 939 940 manifest_package_name = _PackageApk(options, build) 941 942 # If --shared-resources-allowlist is used, all the resources listed in the 943 # corresponding R.txt file will be non-final, and an onResourcesLoaded() 944 # will be generated to adjust them at runtime. 945 # 946 # Otherwise, if --shared-resources is used, the all resources will be 947 # non-final, and an onResourcesLoaded() method will be generated too. 948 # 949 # Otherwise, all resources will be final, and no method will be generated. 950 # 951 rjava_build_options = resource_utils.RJavaBuildOptions() 952 if options.shared_resources_allowlist: 953 rjava_build_options.ExportSomeResources( 954 options.shared_resources_allowlist) 955 rjava_build_options.GenerateOnResourcesLoaded() 956 if options.shared_resources: 957 # The final resources will only be used in WebLayer, so hardcode the 958 # package ID to be what WebLayer expects. 959 rjava_build_options.SetFinalPackageId( 960 protoresources.SHARED_LIBRARY_HARDCODED_ID) 961 elif options.shared_resources or options.app_as_shared_lib: 962 rjava_build_options.ExportAllResources() 963 rjava_build_options.GenerateOnResourcesLoaded() 964 965 custom_root_package_name = options.r_java_root_package_name 966 grandparent_custom_package_name = None 967 968 # Always generate an R.java file for the package listed in 969 # AndroidManifest.xml because this is where Android framework looks to find 970 # onResourcesLoaded() for shared library apks. While not actually necessary 971 # for application apks, it also doesn't hurt. 972 apk_package_name = manifest_package_name 973 974 if options.package_name and not options.arsc_package_name: 975 # Feature modules have their own custom root package name and should 976 # inherit from the appropriate base module package. This behaviour should 977 # not be present for test apks with an apk under test. Thus, 978 # arsc_package_name is used as it is only defined for test apks with an 979 # apk under test. 980 custom_root_package_name = options.package_name 981 grandparent_custom_package_name = options.r_java_root_package_name 982 # Feature modules have the same manifest package as the base module but 983 # they should not create an R.java for said manifest package because it 984 # will be created in the base module. 985 apk_package_name = None 986 987 if options.srcjar_out: 988 logging.debug('Creating R.srcjar') 989 resource_utils.CreateRJavaFiles(build.srcjar_dir, apk_package_name, 990 build.r_txt_path, 991 options.extra_res_packages, 992 rjava_build_options, options.srcjar_out, 993 custom_root_package_name, 994 grandparent_custom_package_name) 995 with action_helpers.atomic_output(build.srcjar_path) as f: 996 zip_helpers.zip_directory(f, build.srcjar_dir) 997 998 logging.debug('Copying outputs') 999 _WriteOutputs(options, build) 1000 1001 if options.depfile: 1002 assert options.srcjar_out, 'Update first output below and remove assert.' 1003 depfile_deps = (options.dependencies_res_zips + 1004 options.dependencies_res_zip_overlays + 1005 options.include_resources) 1006 action_helpers.write_depfile(options.depfile, options.srcjar_out, 1007 depfile_deps) 1008 1009 1010 if __name__ == '__main__': 1011 main(sys.argv[1:])