lint.py (23059B)
1 #!/usr/bin/env python3 2 # 3 # Copyright 2013 The Chromium Authors 4 # Use of this source code is governed by a BSD-style license that can be 5 # found in the LICENSE file. 6 """Runs Android's lint tool.""" 7 8 import argparse 9 import logging 10 import os 11 import shutil 12 import sys 13 import time 14 from xml.dom import minidom 15 from xml.etree import ElementTree 16 17 from util import build_utils 18 from util import manifest_utils 19 from util import server_utils 20 import action_helpers # build_utils adds //build to sys.path. 21 22 _LINT_MD_URL = 'https://chromium.googlesource.com/chromium/src/+/main/build/android/docs/lint.md' # pylint: disable=line-too-long 23 24 # These checks are not useful for chromium. 25 _DISABLED_ALWAYS = [ 26 "AppCompatResource", # Lint does not correctly detect our appcompat lib. 27 "AppLinkUrlError", # As a browser, we have intent filters without a host. 28 "Assert", # R8 --force-enable-assertions is used to enable java asserts. 29 "InflateParams", # Null is ok when inflating views for dialogs. 30 "InlinedApi", # Constants are copied so they are always available. 31 "LintBaseline", # Don't warn about using baseline.xml files. 32 "LintBaselineFixed", # We dont care if baseline.xml has unused entries. 33 "MissingInflatedId", # False positives https://crbug.com/1394222 34 "MissingApplicationIcon", # False positive for non-production targets. 35 "MissingVersion", # We set versions via aapt2, which runs after lint. 36 "NetworkSecurityConfig", # Breaks on library certificates b/269783280. 37 "ObsoleteLintCustomCheck", # We have no control over custom lint checks. 38 "OldTargetApi", # We sometimes need targetSdkVersion to not be latest. 39 "StringFormatCount", # Has false-positives. 40 "SwitchIntDef", # Many C++ enums are not used at all in java. 41 "Typos", # Strings are committed in English first and later translated. 42 "VisibleForTests", # Does not recognize "ForTesting" methods. 43 "UniqueConstants", # Chromium enums allow aliases. 44 "UnusedAttribute", # Chromium apks have various minSdkVersion values. 45 ] 46 47 _RES_ZIP_DIR = 'RESZIPS' 48 _SRCJAR_DIR = 'SRCJARS' 49 _AAR_DIR = 'AARS' 50 51 52 def _SrcRelative(path): 53 """Returns relative path to top-level src dir.""" 54 return os.path.relpath(path, build_utils.DIR_SOURCE_ROOT) 55 56 57 def _GenerateProjectFile(android_manifest, 58 android_sdk_root, 59 cache_dir, 60 partials_dir, 61 sources=None, 62 classpath=None, 63 srcjar_sources=None, 64 resource_sources=None, 65 custom_lint_jars=None, 66 custom_annotation_zips=None, 67 android_sdk_version=None, 68 baseline_path=None): 69 project = ElementTree.Element('project') 70 root = ElementTree.SubElement(project, 'root') 71 # Run lint from output directory: crbug.com/1115594 72 root.set('dir', os.getcwd()) 73 sdk = ElementTree.SubElement(project, 'sdk') 74 # Lint requires that the sdk path be an absolute path. 75 sdk.set('dir', os.path.abspath(android_sdk_root)) 76 if baseline_path is not None: 77 baseline = ElementTree.SubElement(project, 'baseline') 78 baseline.set('file', baseline_path) 79 cache = ElementTree.SubElement(project, 'cache') 80 cache.set('dir', cache_dir) 81 main_module = ElementTree.SubElement(project, 'module') 82 main_module.set('name', 'main') 83 main_module.set('android', 'true') 84 main_module.set('library', 'false') 85 # Required to make lint-resources.xml be written to a per-target path. 86 # https://crbug.com/1515070 and b/324598620 87 main_module.set('partial-results-dir', partials_dir) 88 if android_sdk_version: 89 main_module.set('compile_sdk_version', android_sdk_version) 90 manifest = ElementTree.SubElement(main_module, 'manifest') 91 manifest.set('file', android_manifest) 92 if srcjar_sources: 93 for srcjar_file in srcjar_sources: 94 src = ElementTree.SubElement(main_module, 'src') 95 src.set('file', srcjar_file) 96 # Cannot add generated="true" since then lint does not scan them, and 97 # we get "UnusedResources" lint errors when resources are used only by 98 # generated files. 99 if sources: 100 for source in sources: 101 src = ElementTree.SubElement(main_module, 'src') 102 src.set('file', source) 103 # Cannot set test="true" since we sometimes put Test.java files beside 104 # non-test files, which lint does not allow: 105 # "Test sources cannot be in the same source root as production files" 106 if classpath: 107 for file_path in classpath: 108 classpath_element = ElementTree.SubElement(main_module, 'classpath') 109 classpath_element.set('file', file_path) 110 if resource_sources: 111 for resource_file in resource_sources: 112 resource = ElementTree.SubElement(main_module, 'resource') 113 resource.set('file', resource_file) 114 if custom_lint_jars: 115 for lint_jar in custom_lint_jars: 116 lint = ElementTree.SubElement(main_module, 'lint-checks') 117 lint.set('file', lint_jar) 118 if custom_annotation_zips: 119 for annotation_zip in custom_annotation_zips: 120 annotation = ElementTree.SubElement(main_module, 'annotations') 121 annotation.set('file', annotation_zip) 122 return project 123 124 125 def _RetrieveBackportedMethods(backported_methods_path): 126 with open(backported_methods_path) as f: 127 methods = f.read().splitlines() 128 # Methods look like: 129 # java/util/Set#of(Ljava/lang/Object;)Ljava/util/Set; 130 # But error message looks like: 131 # Call requires API level R (current min is 21): java.util.Set#of [NewApi] 132 methods = (m.replace('/', '\\.') for m in methods) 133 methods = (m[:m.index('(')] for m in methods) 134 return sorted(set(methods)) 135 136 137 def _GenerateConfigXmlTree(orig_config_path, backported_methods): 138 if orig_config_path: 139 root_node = ElementTree.parse(orig_config_path).getroot() 140 else: 141 root_node = ElementTree.fromstring('<lint/>') 142 143 issue_node = ElementTree.SubElement(root_node, 'issue') 144 issue_node.attrib['id'] = 'NewApi' 145 ignore_node = ElementTree.SubElement(issue_node, 'ignore') 146 ignore_node.attrib['regexp'] = '|'.join(backported_methods) 147 return root_node 148 149 150 def _GenerateAndroidManifest(original_manifest_path, extra_manifest_paths, 151 min_sdk_version, android_sdk_version): 152 # Set minSdkVersion in the manifest to the correct value. 153 doc, manifest, app_node = manifest_utils.ParseManifest(original_manifest_path) 154 155 # TODO(crbug.com/40148088): Should this be done using manifest merging? 156 # Add anything in the application node of the extra manifests to the main 157 # manifest to prevent unused resource errors. 158 for path in extra_manifest_paths: 159 _, _, extra_app_node = manifest_utils.ParseManifest(path) 160 for node in extra_app_node: 161 app_node.append(node) 162 163 uses_sdk = manifest.find('./uses-sdk') 164 if uses_sdk is None: 165 uses_sdk = ElementTree.Element('uses-sdk') 166 manifest.insert(0, uses_sdk) 167 uses_sdk.set('{%s}minSdkVersion' % manifest_utils.ANDROID_NAMESPACE, 168 min_sdk_version) 169 uses_sdk.set('{%s}targetSdkVersion' % manifest_utils.ANDROID_NAMESPACE, 170 android_sdk_version) 171 return doc 172 173 174 def _WriteXmlFile(root, path): 175 logging.info('Writing xml file %s', path) 176 build_utils.MakeDirectory(os.path.dirname(path)) 177 with action_helpers.atomic_output(path) as f: 178 # Although we can write it just with ElementTree.tostring, using minidom 179 # makes it a lot easier to read as a human (also on code search). 180 f.write( 181 minidom.parseString(ElementTree.tostring( 182 root, encoding='utf-8')).toprettyxml(indent=' ').encode('utf-8')) 183 184 185 def _RunLint(custom_lint_jar_path, 186 lint_jar_path, 187 backported_methods_path, 188 config_path, 189 manifest_path, 190 extra_manifest_paths, 191 sources, 192 classpath, 193 cache_dir, 194 android_sdk_version, 195 aars, 196 srcjars, 197 min_sdk_version, 198 resource_sources, 199 resource_zips, 200 android_sdk_root, 201 lint_gen_dir, 202 baseline, 203 create_cache, 204 warnings_as_errors=False): 205 logging.info('Lint starting') 206 if not cache_dir: 207 # Use per-target cache directory when --cache-dir is not used. 208 cache_dir = os.path.join(lint_gen_dir, 'cache') 209 # Lint complains if the directory does not exist. 210 # When --create-cache is used, ninja will create this directory because the 211 # stamp file is created within it. 212 os.makedirs(cache_dir, exist_ok=True) 213 214 if baseline and not os.path.exists(baseline): 215 # Generating new baselines is only done locally, and requires more memory to 216 # avoid OOMs. 217 creating_baseline = True 218 lint_xmx = '4G' 219 else: 220 creating_baseline = False 221 lint_xmx = '3G' 222 223 # Lint requires this directory to exist and be cleared. 224 # See b/324598620 225 partials_dir = os.path.join(lint_gen_dir, 'partials') 226 shutil.rmtree(partials_dir, ignore_errors=True) 227 os.makedirs(partials_dir) 228 229 # All paths in lint are based off of relative paths from root with root as the 230 # prefix. Path variable substitution is based off of prefix matching so custom 231 # path variables need to match exactly in order to show up in baseline files. 232 # e.g. lint_path=path/to/output/dir/../../file/in/src 233 root_path = os.getcwd() # This is usually the output directory. 234 pathvar_src = os.path.join( 235 root_path, os.path.relpath(build_utils.DIR_SOURCE_ROOT, start=root_path)) 236 237 cmd = build_utils.JavaCmd(xmx=lint_xmx) + [ 238 '-cp', 239 '{}:{}'.format(lint_jar_path, custom_lint_jar_path), 240 'org.chromium.build.CustomLint', 241 '--sdk-home', 242 android_sdk_root, 243 '--jdk-home', 244 build_utils.JAVA_HOME, 245 '--path-variables', 246 f'SRC={pathvar_src}', 247 '--offline', 248 '--quiet', # Silences lint's "." progress updates. 249 '--stacktrace', # Prints full stacktraces for internal lint errors. 250 ] 251 252 # Only disable for real runs since otherwise you get UnknownIssueId warnings 253 # when disabling custom lint checks since they are not passed during cache 254 # creation. 255 if not create_cache: 256 cmd += [ 257 '--disable', 258 ','.join(_DISABLED_ALWAYS), 259 ] 260 261 if not manifest_path: 262 manifest_path = os.path.join(build_utils.DIR_SOURCE_ROOT, 'build', 263 'android', 'AndroidManifest.xml') 264 265 logging.info('Generating config.xml') 266 backported_methods = _RetrieveBackportedMethods(backported_methods_path) 267 config_xml_node = _GenerateConfigXmlTree(config_path, backported_methods) 268 generated_config_path = os.path.join(lint_gen_dir, 'config.xml') 269 _WriteXmlFile(config_xml_node, generated_config_path) 270 cmd.extend(['--config', generated_config_path]) 271 272 logging.info('Generating Android manifest file') 273 android_manifest_tree = _GenerateAndroidManifest(manifest_path, 274 extra_manifest_paths, 275 min_sdk_version, 276 android_sdk_version) 277 # Just use a hardcoded name, since we may have different target names (and 278 # thus different manifest_paths) using the same lint baseline. Eg. 279 # trichrome_chrome_bundle and trichrome_chrome_32_64_bundle. 280 lint_android_manifest_path = os.path.join(lint_gen_dir, 'AndroidManifest.xml') 281 _WriteXmlFile(android_manifest_tree.getroot(), lint_android_manifest_path) 282 283 resource_root_dir = os.path.join(lint_gen_dir, _RES_ZIP_DIR) 284 shutil.rmtree(resource_root_dir, True) 285 # These are zip files with generated resources (e. g. strings from GRD). 286 logging.info('Extracting resource zips') 287 for resource_zip in resource_zips: 288 # Use a consistent root and name rather than a temporary file so that 289 # suppressions can be local to the lint target and the resource target. 290 resource_dir = os.path.join(resource_root_dir, resource_zip) 291 os.makedirs(resource_dir) 292 resource_sources.extend( 293 build_utils.ExtractAll(resource_zip, path=resource_dir)) 294 295 logging.info('Extracting aars') 296 aar_root_dir = os.path.join(lint_gen_dir, _AAR_DIR) 297 shutil.rmtree(aar_root_dir, True) 298 custom_lint_jars = [] 299 custom_annotation_zips = [] 300 if aars: 301 for aar in aars: 302 # Use relative source for aar files since they are not generated. 303 aar_dir = os.path.join(aar_root_dir, 304 os.path.splitext(_SrcRelative(aar))[0]) 305 os.makedirs(aar_dir) 306 aar_files = build_utils.ExtractAll(aar, path=aar_dir) 307 for f in aar_files: 308 if f.endswith('lint.jar'): 309 custom_lint_jars.append(f) 310 elif f.endswith('annotations.zip'): 311 custom_annotation_zips.append(f) 312 313 logging.info('Extracting srcjars') 314 srcjar_root_dir = os.path.join(lint_gen_dir, _SRCJAR_DIR) 315 shutil.rmtree(srcjar_root_dir, True) 316 srcjar_sources = [] 317 if srcjars: 318 for srcjar in srcjars: 319 # Use path without extensions since otherwise the file name includes 320 # .srcjar and lint treats it as a srcjar. 321 srcjar_dir = os.path.join(srcjar_root_dir, os.path.splitext(srcjar)[0]) 322 os.makedirs(srcjar_dir) 323 # Sadly lint's srcjar support is broken since it only considers the first 324 # srcjar. Until we roll a lint version with that fixed, we need to extract 325 # it ourselves. 326 srcjar_sources.extend(build_utils.ExtractAll(srcjar, path=srcjar_dir)) 327 328 logging.info('Generating project file') 329 project_file_root = _GenerateProjectFile( 330 lint_android_manifest_path, android_sdk_root, cache_dir, partials_dir, 331 sources, classpath, srcjar_sources, resource_sources, custom_lint_jars, 332 custom_annotation_zips, android_sdk_version, baseline) 333 334 project_xml_path = os.path.join(lint_gen_dir, 'project.xml') 335 _WriteXmlFile(project_file_root, project_xml_path) 336 cmd += ['--project', project_xml_path] 337 338 def stdout_filter(output): 339 filter_patterns = [ 340 # This filter is necessary for JDK11. 341 'No issues found', 342 # Custom checks are not always available in every lint run so an 343 # UnknownIssueId warning is sometimes printed for custom checks in the 344 # _DISABLED_ALWAYS list. 345 r'\[UnknownIssueId\]', 346 # If all the warnings are filtered, we should not fail on the final 347 # summary line. 348 r'\d+ errors?, \d+ warnings?', 349 ] 350 return build_utils.FilterLines(output, '|'.join(filter_patterns)) 351 352 def stderr_filter(output): 353 output = build_utils.FilterReflectiveAccessJavaWarnings(output) 354 # Presumably a side-effect of our manual manifest merging, but does not 355 # seem to actually break anything: 356 # "Manifest merger failed with multiple errors, see logs" 357 return build_utils.FilterLines(output, 'Manifest merger failed') 358 359 start = time.time() 360 logging.debug('Lint command %s', ' '.join(cmd)) 361 failed = False 362 363 if creating_baseline and not warnings_as_errors: 364 # Allow error code 6 when creating a baseline: ERRNO_CREATED_BASELINE 365 fail_func = lambda returncode, _: returncode not in (0, 6) 366 else: 367 fail_func = lambda returncode, _: returncode != 0 368 369 try: 370 build_utils.CheckOutput(cmd, 371 print_stdout=True, 372 stdout_filter=stdout_filter, 373 stderr_filter=stderr_filter, 374 fail_on_output=warnings_as_errors, 375 fail_func=fail_func) 376 except build_utils.CalledProcessError as e: 377 failed = True 378 # Do not output the python stacktrace because it is lengthy and is not 379 # relevant to the actual lint error. 380 sys.stderr.write(e.output) 381 finally: 382 # When not treating warnings as errors, display the extra footer. 383 is_debug = os.environ.get('LINT_DEBUG', '0') != '0' 384 385 end = time.time() - start 386 logging.info('Lint command took %ss', end) 387 if not is_debug: 388 shutil.rmtree(aar_root_dir, ignore_errors=True) 389 shutil.rmtree(resource_root_dir, ignore_errors=True) 390 shutil.rmtree(srcjar_root_dir, ignore_errors=True) 391 if os.path.exists(project_xml_path): 392 os.unlink(project_xml_path) 393 shutil.rmtree(partials_dir, ignore_errors=True) 394 395 if failed: 396 print('- For more help with lint in Chrome:', _LINT_MD_URL) 397 if is_debug: 398 print('- DEBUG MODE: Here is the project.xml: {}'.format( 399 _SrcRelative(project_xml_path))) 400 else: 401 print('- Run with LINT_DEBUG=1 to enable lint configuration debugging') 402 sys.exit(1) 403 404 logging.info('Lint completed') 405 406 407 def _ParseArgs(argv): 408 parser = argparse.ArgumentParser() 409 action_helpers.add_depfile_arg(parser) 410 parser.add_argument('--target-name', help='Fully qualified GN target name.') 411 parser.add_argument('--skip-build-server', 412 action='store_true', 413 help='Avoid using the build server.') 414 parser.add_argument('--use-build-server', 415 action='store_true', 416 help='Always use the build server.') 417 parser.add_argument('--lint-jar-path', 418 required=True, 419 help='Path to the lint jar.') 420 parser.add_argument('--custom-lint-jar-path', 421 required=True, 422 help='Path to our custom lint jar.') 423 parser.add_argument('--backported-methods', 424 help='Path to backported methods file created by R8.') 425 parser.add_argument('--cache-dir', 426 help='Path to the directory in which the android cache ' 427 'directory tree should be stored.') 428 parser.add_argument('--config-path', help='Path to lint suppressions file.') 429 parser.add_argument('--lint-gen-dir', 430 required=True, 431 help='Path to store generated xml files.') 432 parser.add_argument('--stamp', help='Path to stamp upon success.') 433 parser.add_argument('--android-sdk-version', 434 help='Version (API level) of the Android SDK used for ' 435 'building.') 436 parser.add_argument('--min-sdk-version', 437 required=True, 438 help='Minimal SDK version to lint against.') 439 parser.add_argument('--android-sdk-root', 440 required=True, 441 help='Lint needs an explicit path to the android sdk.') 442 parser.add_argument('--create-cache', 443 action='store_true', 444 help='Whether this invocation is just warming the cache.') 445 parser.add_argument('--warnings-as-errors', 446 action='store_true', 447 help='Treat all warnings as errors.') 448 parser.add_argument('--sources', 449 help='A list of files containing java and kotlin source ' 450 'files.') 451 parser.add_argument('--aars', help='GN list of included aars.') 452 parser.add_argument('--srcjars', help='GN list of included srcjars.') 453 parser.add_argument('--manifest-path', 454 help='Path to original AndroidManifest.xml') 455 parser.add_argument('--extra-manifest-paths', 456 action='append', 457 help='GYP-list of manifest paths to merge into the ' 458 'original AndroidManifest.xml') 459 parser.add_argument('--resource-sources', 460 default=[], 461 action='append', 462 help='GYP-list of resource sources files, similar to ' 463 'java sources files, but for resource files.') 464 parser.add_argument('--resource-zips', 465 default=[], 466 action='append', 467 help='GYP-list of resource zips, zip files of generated ' 468 'resource files.') 469 parser.add_argument('--classpath', 470 help='List of jars to add to the classpath.') 471 parser.add_argument('--baseline', 472 help='Baseline file to ignore existing errors and fail ' 473 'on new errors.') 474 475 args = parser.parse_args(build_utils.ExpandFileArgs(argv)) 476 args.sources = action_helpers.parse_gn_list(args.sources) 477 args.aars = action_helpers.parse_gn_list(args.aars) 478 args.srcjars = action_helpers.parse_gn_list(args.srcjars) 479 args.resource_sources = action_helpers.parse_gn_list(args.resource_sources) 480 args.extra_manifest_paths = action_helpers.parse_gn_list( 481 args.extra_manifest_paths) 482 args.resource_zips = action_helpers.parse_gn_list(args.resource_zips) 483 args.classpath = action_helpers.parse_gn_list(args.classpath) 484 485 if args.baseline: 486 assert os.path.basename(args.baseline) == 'lint-baseline.xml', ( 487 'The baseline file needs to be named "lint-baseline.xml" in order for ' 488 'the autoroller to find and update it whenever lint is rolled to a new ' 489 'version.') 490 491 return args 492 493 494 def main(): 495 build_utils.InitLogging('LINT_DEBUG') 496 args = _ParseArgs(sys.argv[1:]) 497 498 sources = [] 499 for sources_file in args.sources: 500 sources.extend(build_utils.ReadSourcesList(sources_file)) 501 resource_sources = [] 502 for resource_sources_file in args.resource_sources: 503 resource_sources.extend(build_utils.ReadSourcesList(resource_sources_file)) 504 505 possible_depfile_deps = (args.srcjars + args.resource_zips + sources + 506 resource_sources + [ 507 args.baseline, 508 args.manifest_path, 509 ]) 510 depfile_deps = [p for p in possible_depfile_deps if p] 511 512 if args.depfile: 513 action_helpers.write_depfile(args.depfile, args.stamp, depfile_deps) 514 515 # TODO(wnwen): Consider removing lint cache now that there are only two lint 516 # invocations. 517 # Avoid parallelizing cache creation since lint runs without the cache defeat 518 # the purpose of creating the cache in the first place. Forward the command 519 # after the depfile has been written as siso requires it. 520 if (not args.create_cache and not args.skip_build_server 521 and server_utils.MaybeRunCommand(name=args.target_name, 522 argv=sys.argv, 523 stamp_file=args.stamp, 524 use_build_server=args.use_build_server)): 525 return 526 527 _RunLint(args.custom_lint_jar_path, 528 args.lint_jar_path, 529 args.backported_methods, 530 args.config_path, 531 args.manifest_path, 532 args.extra_manifest_paths, 533 sources, 534 args.classpath, 535 args.cache_dir, 536 args.android_sdk_version, 537 args.aars, 538 args.srcjars, 539 args.min_sdk_version, 540 resource_sources, 541 args.resource_zips, 542 args.android_sdk_root, 543 args.lint_gen_dir, 544 args.baseline, 545 args.create_cache, 546 warnings_as_errors=args.warnings_as_errors) 547 logging.info('Creating stamp file') 548 server_utils.MaybeTouch(args.stamp) 549 550 551 if __name__ == '__main__': 552 sys.exit(main())