apkbuilder.py (22603B)
1 #!/usr/bin/env python3 2 # 3 # Copyright 2015 The Chromium Authors 4 # Use of this source code is governed by a BSD-style license that can be 5 # found in the LICENSE file. 6 7 """Adds the code parts to a resource APK.""" 8 9 import argparse 10 import logging 11 import os 12 import posixpath 13 import shutil 14 import sys 15 import tempfile 16 import zipfile 17 import zlib 18 19 import finalize_apk 20 21 from util import build_utils 22 from util import diff_utils 23 import action_helpers # build_utils adds //build to sys.path. 24 import zip_helpers 25 26 27 # Taken from aapt's Package.cpp: 28 _NO_COMPRESS_EXTENSIONS = ('.jpg', '.jpeg', '.png', '.gif', '.wav', '.mp2', 29 '.mp3', '.ogg', '.aac', '.mpg', '.mpeg', '.mid', 30 '.midi', '.smf', '.jet', '.rtttl', '.imy', '.xmf', 31 '.mp4', '.m4a', '.m4v', '.3gp', '.3gpp', '.3g2', 32 '.3gpp2', '.amr', '.awb', '.wma', '.wmv', '.webm') 33 34 35 def _ParseArgs(args): 36 parser = argparse.ArgumentParser() 37 action_helpers.add_depfile_arg(parser) 38 parser.add_argument('--assets', 39 action='append', 40 help='GYP-list of files to add as assets in the form ' 41 '"srcPath:zipPath", where ":zipPath" is optional.') 42 parser.add_argument( 43 '--java-resources', help='GYP-list of java_resources JARs to include.') 44 parser.add_argument('--write-asset-list', 45 action='store_true', 46 help='Whether to create an assets/assets_list file.') 47 parser.add_argument( 48 '--uncompressed-assets', 49 help='Same as --assets, except disables compression.') 50 parser.add_argument('--resource-apk', 51 help='An .ap_ file built using aapt', 52 required=True) 53 parser.add_argument('--output-apk', 54 help='Path to the output file', 55 required=True) 56 parser.add_argument('--format', choices=['apk', 'bundle-module'], 57 default='apk', help='Specify output format.') 58 parser.add_argument('--dex-file', 59 help='Path to the classes.dex to use') 60 parser.add_argument('--uncompress-dex', action='store_true', 61 help='Store .dex files uncompressed in the APK') 62 parser.add_argument('--native-libs', 63 action='append', 64 help='GYP-list of native libraries to include. ' 65 'Can be specified multiple times.', 66 default=[]) 67 parser.add_argument('--secondary-native-libs', 68 action='append', 69 help='GYP-list of native libraries for secondary ' 70 'android-abi. Can be specified multiple times.', 71 default=[]) 72 parser.add_argument('--android-abi', 73 help='Android architecture to use for native libraries') 74 parser.add_argument('--secondary-android-abi', 75 help='The secondary Android architecture to use for' 76 'secondary native libraries') 77 parser.add_argument( 78 '--is-multi-abi', 79 action='store_true', 80 help='Will add a placeholder for the missing ABI if no native libs or ' 81 'placeholders are set for either the primary or secondary ABI. Can only ' 82 'be set if both --android-abi and --secondary-android-abi are set.') 83 parser.add_argument( 84 '--native-lib-placeholders', 85 help='GYP-list of native library placeholders to add.') 86 parser.add_argument( 87 '--secondary-native-lib-placeholders', 88 help='GYP-list of native library placeholders to add ' 89 'for the secondary ABI') 90 parser.add_argument('--uncompress-shared-libraries', default='False', 91 choices=['true', 'True', 'false', 'False'], 92 help='Whether to uncompress native shared libraries. Argument must be ' 93 'a boolean value.') 94 parser.add_argument( 95 '--apksigner-jar', help='Path to the apksigner executable.') 96 parser.add_argument('--zipalign-path', 97 help='Path to the zipalign executable.') 98 parser.add_argument('--key-path', 99 help='Path to keystore for signing.') 100 parser.add_argument('--key-passwd', 101 help='Keystore password') 102 parser.add_argument('--key-name', 103 help='Keystore name') 104 parser.add_argument( 105 '--min-sdk-version', required=True, help='Value of APK\'s minSdkVersion') 106 parser.add_argument( 107 '--best-compression', 108 action='store_true', 109 help='Use zip -9 rather than zip -1') 110 parser.add_argument( 111 '--library-always-compress', 112 action='append', 113 help='The list of library files that we always compress.') 114 parser.add_argument('--warnings-as-errors', 115 action='store_true', 116 help='Treat all warnings as errors.') 117 diff_utils.AddCommandLineFlags(parser) 118 options = parser.parse_args(args) 119 options.assets = action_helpers.parse_gn_list(options.assets) 120 options.uncompressed_assets = action_helpers.parse_gn_list( 121 options.uncompressed_assets) 122 options.native_lib_placeholders = action_helpers.parse_gn_list( 123 options.native_lib_placeholders) 124 options.secondary_native_lib_placeholders = action_helpers.parse_gn_list( 125 options.secondary_native_lib_placeholders) 126 options.java_resources = action_helpers.parse_gn_list(options.java_resources) 127 options.native_libs = action_helpers.parse_gn_list(options.native_libs) 128 options.secondary_native_libs = action_helpers.parse_gn_list( 129 options.secondary_native_libs) 130 options.library_always_compress = action_helpers.parse_gn_list( 131 options.library_always_compress) 132 133 if not options.android_abi and (options.native_libs or 134 options.native_lib_placeholders): 135 raise Exception('Must specify --android-abi with --native-libs') 136 if not options.secondary_android_abi and (options.secondary_native_libs or 137 options.secondary_native_lib_placeholders): 138 raise Exception('Must specify --secondary-android-abi with' 139 ' --secondary-native-libs') 140 if options.is_multi_abi and not (options.android_abi 141 and options.secondary_android_abi): 142 raise Exception('Must specify --is-multi-abi with both --android-abi ' 143 'and --secondary-android-abi.') 144 return options 145 146 147 def _SplitAssetPath(path): 148 """Returns (src, dest) given an asset path in the form src[:dest].""" 149 path_parts = path.split(':') 150 src_path = path_parts[0] 151 if len(path_parts) > 1: 152 dest_path = path_parts[1] 153 else: 154 dest_path = os.path.basename(src_path) 155 return src_path, dest_path 156 157 158 def _ExpandPaths(paths): 159 """Converts src:dst into tuples and enumerates files within directories. 160 161 Args: 162 paths: Paths in the form "src_path:dest_path" 163 164 Returns: 165 A list of (src_path, dest_path) tuples sorted by dest_path (for stable 166 ordering within output .apk). 167 """ 168 ret = [] 169 for path in paths: 170 src_path, dest_path = _SplitAssetPath(path) 171 if os.path.isdir(src_path): 172 for f in build_utils.FindInDirectory(src_path, '*'): 173 ret.append((f, os.path.join(dest_path, f[len(src_path) + 1:]))) 174 else: 175 ret.append((src_path, dest_path)) 176 ret.sort(key=lambda t:t[1]) 177 return ret 178 179 180 def _GetAssetsToAdd(path_tuples, 181 fast_align, 182 disable_compression=False, 183 allow_reads=True, 184 apk_root_dir=''): 185 """Returns the list of file_detail tuples for assets in the apk. 186 187 Args: 188 path_tuples: List of src_path, dest_path tuples to add. 189 fast_align: Whether to perform alignment in python zipfile (alternatively 190 alignment can be done using the zipalign utility out of band). 191 disable_compression: Whether to disable compression. 192 allow_reads: If false, we do not try to read the files from disk (to find 193 their size for example). 194 195 Returns: A list of (src_path, apk_path, compress, alignment) tuple 196 representing what and how assets are added. 197 """ 198 assets_to_add = [] 199 200 # Group all uncompressed assets together in the hope that it will increase 201 # locality of mmap'ed files. 202 for target_compress in (False, True): 203 for src_path, dest_path in path_tuples: 204 compress = not disable_compression and ( 205 os.path.splitext(src_path)[1] not in _NO_COMPRESS_EXTENSIONS) 206 207 if target_compress == compress: 208 # add_to_zip_hermetic() uses this logic to avoid growing small files. 209 # We need it here in order to set alignment correctly. 210 if allow_reads and compress and os.path.getsize(src_path) < 16: 211 compress = False 212 213 if dest_path.startswith('../'): 214 # posixpath.join('', 'foo') == 'foo' 215 apk_path = posixpath.join(apk_root_dir, dest_path[3:]) 216 else: 217 apk_path = 'assets/' + dest_path 218 alignment = 0 if compress and not fast_align else 4 219 assets_to_add.append((apk_path, src_path, compress, alignment)) 220 return assets_to_add 221 222 223 def _AddFiles(apk, details): 224 """Adds files to the apk. 225 226 Args: 227 apk: path to APK to add to. 228 details: A list of file detail tuples (src_path, apk_path, compress, 229 alignment) representing what and how files are added to the APK. 230 """ 231 for apk_path, src_path, compress, alignment in details: 232 # This check is only relevant for assets, but it should not matter if it is 233 # checked for the whole list of files. 234 try: 235 apk.getinfo(apk_path) 236 # Should never happen since write_build_config.py handles merging. 237 raise Exception( 238 'Multiple targets specified the asset path: %s' % apk_path) 239 except KeyError: 240 zip_helpers.add_to_zip_hermetic(apk, 241 apk_path, 242 src_path=src_path, 243 compress=compress, 244 alignment=alignment) 245 246 247 def _GetAbiAlignment(android_abi): 248 if '64' in android_abi: 249 return 0x4000 # 16k alignment 250 return 0x1000 # 4k alignment 251 252 253 def _GetNativeLibrariesToAdd(native_libs, android_abi, fast_align, 254 lib_always_compress): 255 """Returns the list of file_detail tuples for native libraries in the apk. 256 257 Returns: A list of (src_path, apk_path, compress, alignment) tuple 258 representing what and how native libraries are added. 259 """ 260 libraries_to_add = [] 261 262 263 for path in native_libs: 264 basename = os.path.basename(path) 265 compress = any(lib_name in basename for lib_name in lib_always_compress) 266 lib_android_abi = android_abi 267 if path.startswith('android_clang_arm64_hwasan/'): 268 lib_android_abi = 'arm64-v8a-hwasan' 269 270 apk_path = 'lib/%s/%s' % (lib_android_abi, basename) 271 if compress and not fast_align: 272 alignment = 0 273 else: 274 alignment = _GetAbiAlignment(android_abi) 275 libraries_to_add.append((apk_path, path, compress, alignment)) 276 277 return libraries_to_add 278 279 280 def _CreateExpectationsData(native_libs, assets): 281 """Creates list of native libraries and assets.""" 282 native_libs = sorted(native_libs) 283 assets = sorted(assets) 284 285 ret = [] 286 for apk_path, _, compress, alignment in native_libs + assets: 287 ret.append('apk_path=%s, compress=%s, alignment=%s\n' % 288 (apk_path, compress, alignment)) 289 return ''.join(ret) 290 291 292 def main(args): 293 build_utils.InitLogging('APKBUILDER_DEBUG') 294 args = build_utils.ExpandFileArgs(args) 295 options = _ParseArgs(args) 296 297 # Until Python 3.7, there's no better way to set compression level. 298 # The default is 6. 299 if options.best_compression: 300 # Compresses about twice as slow as the default. 301 zlib.Z_DEFAULT_COMPRESSION = 9 302 else: 303 # Compresses about twice as fast as the default. 304 zlib.Z_DEFAULT_COMPRESSION = 1 305 306 # Python's zip implementation duplicates file comments in the central 307 # directory, whereas zipalign does not, so use zipalign for official builds. 308 requires_alignment = options.format == 'apk' 309 # TODO(crbug.com/40286668): Re-enable zipalign once we are using Android V 310 # SDK. 311 run_zipalign = requires_alignment and options.best_compression and False 312 fast_align = bool(requires_alignment and not run_zipalign) 313 314 native_libs = sorted(options.native_libs) 315 316 # Include native libs in the depfile_deps since GN doesn't know about the 317 # dependencies when is_component_build=true. 318 depfile_deps = list(native_libs) 319 320 # For targets that depend on static library APKs, dex paths are created by 321 # the static library's dexsplitter target and GN doesn't know about these 322 # paths. 323 if options.dex_file: 324 depfile_deps.append(options.dex_file) 325 326 secondary_native_libs = [] 327 if options.secondary_native_libs: 328 secondary_native_libs = sorted(options.secondary_native_libs) 329 depfile_deps += secondary_native_libs 330 331 if options.java_resources: 332 # Included via .build_config.json, so need to write it to depfile. 333 depfile_deps.extend(options.java_resources) 334 335 assets = _ExpandPaths(options.assets) 336 uncompressed_assets = _ExpandPaths(options.uncompressed_assets) 337 338 # Included via .build_config.json, so need to write it to depfile. 339 depfile_deps.extend(x[0] for x in assets) 340 depfile_deps.extend(x[0] for x in uncompressed_assets) 341 depfile_deps.append(options.resource_apk) 342 343 # Bundle modules have a structure similar to APKs, except that resources 344 # are compiled in protobuf format (instead of binary xml), and that some 345 # files are located into different top-level directories, e.g.: 346 # AndroidManifest.xml -> manifest/AndroidManifest.xml 347 # classes.dex -> dex/classes.dex 348 # res/ -> res/ (unchanged) 349 # assets/ -> assets/ (unchanged) 350 # <other-file> -> root/<other-file> 351 # 352 # Hence, the following variables are used to control the location of files in 353 # the final archive. 354 if options.format == 'bundle-module': 355 apk_manifest_dir = 'manifest/' 356 apk_root_dir = 'root/' 357 apk_dex_dir = 'dex/' 358 else: 359 apk_manifest_dir = '' 360 apk_root_dir = '' 361 apk_dex_dir = '' 362 363 def _GetAssetDetails(assets, uncompressed_assets, fast_align, allow_reads): 364 ret = _GetAssetsToAdd(assets, 365 fast_align, 366 disable_compression=False, 367 allow_reads=allow_reads, 368 apk_root_dir=apk_root_dir) 369 ret.extend( 370 _GetAssetsToAdd(uncompressed_assets, 371 fast_align, 372 disable_compression=True, 373 allow_reads=allow_reads, 374 apk_root_dir=apk_root_dir)) 375 return ret 376 377 libs_to_add = _GetNativeLibrariesToAdd(native_libs, options.android_abi, 378 fast_align, 379 options.library_always_compress) 380 if options.secondary_android_abi: 381 libs_to_add.extend( 382 _GetNativeLibrariesToAdd(secondary_native_libs, 383 options.secondary_android_abi, 384 fast_align, options.library_always_compress)) 385 386 if options.expected_file: 387 # We compute expectations without reading the files. This allows us to check 388 # expectations for different targets by just generating their build_configs 389 # and not have to first generate all the actual files and all their 390 # dependencies (for example by just passing --only-verify-expectations). 391 asset_details = _GetAssetDetails(assets, 392 uncompressed_assets, 393 fast_align, 394 allow_reads=False) 395 396 actual_data = _CreateExpectationsData(libs_to_add, asset_details) 397 diff_utils.CheckExpectations(actual_data, options) 398 399 if options.only_verify_expectations: 400 if options.depfile: 401 action_helpers.write_depfile(options.depfile, 402 options.actual_file, 403 inputs=depfile_deps) 404 return 405 406 # If we are past this point, we are going to actually create the final apk so 407 # we should recompute asset details again but maybe perform some optimizations 408 # based on the size of the files on disk. 409 assets_to_add = _GetAssetDetails( 410 assets, uncompressed_assets, fast_align, allow_reads=True) 411 412 # Targets generally do not depend on apks, so no need for only_if_changed. 413 with action_helpers.atomic_output(options.output_apk, 414 only_if_changed=False) as f: 415 with zipfile.ZipFile(options.resource_apk) as resource_apk, \ 416 zipfile.ZipFile(f, 'w') as out_apk: 417 418 def add_to_zip(zip_path, data, compress=True, alignment=4): 419 zip_helpers.add_to_zip_hermetic( 420 out_apk, 421 zip_path, 422 data=data, 423 compress=compress, 424 alignment=0 if compress and not fast_align else alignment) 425 426 def copy_resource(zipinfo, out_dir=''): 427 add_to_zip( 428 out_dir + zipinfo.filename, 429 resource_apk.read(zipinfo.filename), 430 compress=zipinfo.compress_type != zipfile.ZIP_STORED) 431 432 # Make assets come before resources in order to maintain the same file 433 # ordering as GYP / aapt. http://crbug.com/561862 434 resource_infos = resource_apk.infolist() 435 436 # 1. AndroidManifest.xml 437 logging.debug('Adding AndroidManifest.xml') 438 copy_resource( 439 resource_apk.getinfo('AndroidManifest.xml'), out_dir=apk_manifest_dir) 440 441 # 2. Assets 442 logging.debug('Adding assets/') 443 _AddFiles(out_apk, assets_to_add) 444 445 # 3. DEX and META-INF/services/ 446 logging.debug('Adding classes.dex') 447 if options.dex_file: 448 with open(options.dex_file, 'rb') as dex_file_obj: 449 if options.dex_file.endswith('.dex'): 450 # This is the case for incremental_install=true. 451 add_to_zip( 452 apk_dex_dir + 'classes.dex', 453 dex_file_obj.read(), 454 compress=not options.uncompress_dex) 455 else: 456 with zipfile.ZipFile(dex_file_obj) as dex_zip: 457 # Add META-INF/services. 458 for name in sorted(dex_zip.namelist()): 459 if name.startswith('META-INF/services/'): 460 # proguard.py does not bundle these files (dex.py does) 461 # because R8 optimizes all ServiceLoader calls. 462 if options.dex_file.endswith('.r8dex.jar'): 463 raise Exception( 464 f'Expected no META-INF/services, but found: {name}' + 465 f'in {options.dex_file}') 466 add_to_zip(apk_root_dir + name, 467 dex_zip.read(name), 468 compress=False) 469 # Add classes.dex. 470 for name in dex_zip.namelist(): 471 if name.endswith('.dex'): 472 add_to_zip(apk_dex_dir + name, 473 dex_zip.read(name), 474 compress=not options.uncompress_dex) 475 476 # 4. Native libraries. 477 logging.debug('Adding lib/') 478 _AddFiles(out_apk, libs_to_add) 479 480 # Add a placeholder lib if the APK should be multi ABI but is missing libs 481 # for one of the ABIs. 482 native_lib_placeholders = options.native_lib_placeholders 483 secondary_native_lib_placeholders = ( 484 options.secondary_native_lib_placeholders) 485 if options.is_multi_abi: 486 if ((secondary_native_libs or secondary_native_lib_placeholders) 487 and not native_libs and not native_lib_placeholders): 488 native_lib_placeholders += ['libplaceholder.so'] 489 if ((native_libs or native_lib_placeholders) 490 and not secondary_native_libs 491 and not secondary_native_lib_placeholders): 492 secondary_native_lib_placeholders += ['libplaceholder.so'] 493 494 # Add placeholder libs. 495 for name in sorted(native_lib_placeholders): 496 # Note: Empty libs files are ignored by md5check (can cause issues 497 # with stale builds when the only change is adding/removing 498 # placeholders). 499 apk_path = 'lib/%s/%s' % (options.android_abi, name) 500 alignment = _GetAbiAlignment(options.android_abi) 501 add_to_zip(apk_path, '', alignment=alignment) 502 503 for name in sorted(secondary_native_lib_placeholders): 504 # Note: Empty libs files are ignored by md5check (can cause issues 505 # with stale builds when the only change is adding/removing 506 # placeholders). 507 apk_path = 'lib/%s/%s' % (options.secondary_android_abi, name) 508 alignment = _GetAbiAlignment(options.secondary_android_abi) 509 add_to_zip(apk_path, '', alignment=alignment) 510 511 # 5. Resources 512 logging.debug('Adding res/') 513 for info in sorted(resource_infos, key=lambda i: i.filename): 514 if info.filename != 'AndroidManifest.xml': 515 copy_resource(info) 516 517 # 6. Java resources that should be accessible via 518 # Class.getResourceAsStream(), in particular parts of Emma jar. 519 # Prebuilt jars may contain class files which we shouldn't include. 520 logging.debug('Adding Java resources') 521 for java_resource in options.java_resources: 522 with zipfile.ZipFile(java_resource, 'r') as java_resource_jar: 523 for apk_path in sorted(java_resource_jar.namelist()): 524 apk_path_lower = apk_path.lower() 525 526 if apk_path_lower.startswith('meta-inf/'): 527 continue 528 if apk_path_lower.endswith('/'): 529 continue 530 if apk_path_lower.endswith('.class'): 531 continue 532 533 add_to_zip(apk_root_dir + apk_path, 534 java_resource_jar.read(apk_path)) 535 536 if options.format == 'apk' and options.key_path: 537 zipalign_path = None if fast_align else options.zipalign_path 538 finalize_apk.FinalizeApk(options.apksigner_jar, 539 zipalign_path, 540 f.name, 541 f.name, 542 options.key_path, 543 options.key_passwd, 544 options.key_name, 545 int(options.min_sdk_version), 546 warnings_as_errors=options.warnings_as_errors) 547 logging.debug('Moving file into place') 548 549 if options.depfile: 550 action_helpers.write_depfile(options.depfile, 551 options.output_apk, 552 inputs=depfile_deps) 553 554 555 if __name__ == '__main__': 556 main(sys.argv[1:])