update_buildconfig_from_gradle.py (5393B)
1 #!/usr/bin/env python3 2 3 # This Source Code Form is subject to the terms of the Mozilla Public 4 # License, v. 2.0. If a copy of the MPL was not distributed with this 5 # file, You can obtain one at http://mozilla.org/MPL/2.0/. 6 7 8 import argparse 9 import json 10 import logging 11 import os 12 import re 13 import subprocess 14 import sys 15 from collections import defaultdict 16 17 import yaml 18 from mergedeep import merge 19 20 logger = logging.getLogger(__name__) 21 22 _DEFAULT_GRADLE_COMMAND = ("./gradlew", "--console=plain", "--no-parallel") 23 _LOCAL_DEPENDENCY_PATTERN = re.compile( 24 r"(\+|\\)--- project :(?P<local_dependency_name>\S+)\s?.*" 25 ) 26 27 28 def _get_upstream_deps_per_gradle_project(gradle_root, existing_build_config): 29 project_dependencies = defaultdict(set) 30 gradle_projects = _get_gradle_projects(gradle_root, existing_build_config) 31 32 logger.info(f"Looking for dependencies in {gradle_root}") 33 34 # This is eventually going to fail if there's ever enough projects to make the 35 # command line too long. If that happens, we'll need to split this list up and 36 # run gradle more than once. 37 cmd = list(_DEFAULT_GRADLE_COMMAND) 38 cmd.extend([ 39 f"{gradle_project}:dependencies" for gradle_project in sorted(gradle_projects) 40 ]) 41 42 # Parsing output like this is not ideal but bhearsum couldn't find a way 43 # to get the dependencies printed in a better format. If we could convince 44 # gradle to spit out JSON that would be much better. 45 # This is filed as https://bugzilla.mozilla.org/show_bug.cgi?id=1795152 46 current_project_name = None 47 print(f"Running command: {' '.join(cmd)}") 48 try: 49 output = subprocess.check_output(cmd, universal_newlines=True, cwd=gradle_root) 50 except subprocess.CalledProcessError as cpe: 51 print(cpe.output) 52 raise 53 for line in output.splitlines(): 54 # If we find the start of a new component section, update our tracking 55 # variable 56 if line.startswith("Project"): 57 current_project_name = line.split(":", 1)[1].strip("'") 58 59 # If we find a new local dependency, add it. 60 local_dep_match = _LOCAL_DEPENDENCY_PATTERN.search(line) 61 if local_dep_match: 62 local_dependency_name = local_dep_match.group("local_dependency_name") 63 if local_dependency_name != current_project_name: 64 project_dependencies[current_project_name].add(local_dependency_name) 65 66 return { 67 project_name: sorted(project_dependencies[project_name]) 68 for project_name in gradle_projects 69 } 70 71 72 def _get_gradle_projects(gradle_root, existing_build_config): 73 if gradle_root.endswith("android-components"): 74 return list(existing_build_config["projects"].keys()) 75 elif gradle_root.endswith("focus-android"): 76 return ["app"] 77 elif gradle_root.endswith("fenix"): 78 return ["app"] 79 80 raise NotImplementedError(f"Cannot find gradle projects for {gradle_root}") 81 82 83 def is_dir(string): 84 if os.path.isdir(string): 85 return string 86 else: 87 raise argparse.ArgumentTypeError(f'"{string}" is not a directory') 88 89 90 def _parse_args(cmdln_args): 91 parser = argparse.ArgumentParser( 92 description="Calls gradle and generate json file with dependencies" 93 ) 94 parser.add_argument( 95 "gradle_root", 96 metavar="GRADLE_ROOT", 97 type=is_dir, 98 help="The directory where to call gradle from", 99 ) 100 return parser.parse_args(args=cmdln_args) 101 102 103 def _set_logging_config(): 104 logging.basicConfig( 105 level=logging.DEBUG, format="%(asctime)s - %(levelname)s - %(message)s" 106 ) 107 108 109 def _merge_build_config( 110 existing_build_config, upstream_deps_per_project, variants_config 111 ): 112 updated_build_config = { 113 "projects": { 114 project: {"upstream_dependencies": deps} 115 for project, deps in upstream_deps_per_project.items() 116 } 117 } 118 updated_variant_config = {"variants": variants_config} if variants_config else {} 119 return merge(existing_build_config, updated_build_config, updated_variant_config) 120 121 122 def _get_variants(gradle_root): 123 cmd = list(_DEFAULT_GRADLE_COMMAND) + ["printVariants"] 124 output_lines = subprocess.check_output( 125 cmd, universal_newlines=True, cwd=gradle_root 126 ).splitlines() 127 variants_line = [line for line in output_lines if line.startswith("variants: ")][0] 128 variants_json = variants_line.split(" ", 1)[1] 129 return json.loads(variants_json) 130 131 132 def _should_print_variants(gradle_root): 133 return gradle_root.endswith("fenix") or gradle_root.endswith("focus-android") 134 135 136 def main(): 137 args = _parse_args(sys.argv[1:]) 138 gradle_root = args.gradle_root 139 build_config_file = os.path.join(gradle_root, ".buildconfig.yml") 140 _set_logging_config() 141 142 with open(build_config_file) as f: 143 existing_build_config = yaml.safe_load(f) 144 145 upstream_deps_per_project = _get_upstream_deps_per_gradle_project( 146 gradle_root, existing_build_config 147 ) 148 149 variants_config = ( 150 _get_variants(gradle_root) if _should_print_variants(gradle_root) else {} 151 ) 152 merged_build_config = _merge_build_config( 153 existing_build_config, upstream_deps_per_project, variants_config 154 ) 155 156 with open(build_config_file, "w") as f: 157 yaml.safe_dump(merged_build_config, f) 158 logger.info(f"Updated {build_config_file} with latest gradle config!") 159 160 161 __name__ == "__main__" and main()