vendor_and_commit.py (12300B)
1 # This Source Code Form is subject to the terms of the Mozilla Public 2 # License, v. 2.0. If a copy of the MPL was not distributed with this 3 # file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 import argparse 5 import atexit 6 import os 7 import re 8 import sys 9 10 from filter_git_changes import filter_git_changes 11 from run_operations import ( 12 ErrorHelp, 13 RepoType, 14 detect_repo_type, 15 get_last_line, 16 run_git, 17 run_hg, 18 run_shell, 19 update_resume_state, 20 ) 21 22 # This script vendors moz-libwebrtc, handles add/deletes/renames and 23 # commits the newly vendored code with the provided commit message. 24 25 script_name = os.path.basename(__file__) 26 error_help = ErrorHelp() 27 error_help.set_prefix(f"*** ERROR *** {script_name} did not complete successfully") 28 error_help.set_postfix( 29 f"Please resolve the error and then continue running {script_name}" 30 ) 31 repo_type = detect_repo_type() 32 33 34 def early_exit_handler(): 35 error_help.print_help() 36 37 38 def log_output_lines(lines, log_dir, filename): 39 if len(lines) == 0: 40 return 41 42 with open(os.path.join(log_dir, filename), "w") as ofile: 43 for line in lines: 44 ofile.write(line) 45 ofile.write("\n") 46 47 48 def vendor_current_stack(github_branch, github_path, script_dir): 49 print("-------") 50 print(f"------- Vendor {github_branch} from {github_path}") 51 print("-------") 52 cmd = [ 53 "./mach", 54 "python", 55 f"{script_dir}/vendor-libwebrtc.py", 56 "--from-local", 57 github_path, 58 "--commit", 59 github_branch, 60 "libwebrtc", 61 ] 62 cmd = " ".join(cmd) 63 run_shell(cmd) 64 65 66 def restore_mozbuild_files(target_dir, log_dir): 67 print("-------") 68 print("------- Restore moz.build files from repo") 69 print("-------") 70 if repo_type == RepoType.GIT: 71 cmd = f"git restore '{target_dir}/**moz.build'" 72 stdout_lines = run_shell(cmd) 73 else: 74 cmd = f'hg revert --include "{target_dir}/**moz.build" {target_dir}' 75 stdout_lines = run_shell(cmd) # run_shell to allow file wildcard 76 log_output_lines(stdout_lines, log_dir, "log-regen-mozbuild-files.txt") 77 78 79 def remove_deleted_upstream_files( 80 github_path, github_sha, target_dir, log_dir, handle_noop_commit 81 ): 82 if handle_noop_commit: 83 return 84 deleted_paths = filter_git_changes(github_path, github_sha, "D") 85 deleted_paths = [re.sub("^.\t", "", x) for x in deleted_paths] 86 deleted_paths = [os.path.join(target_dir, x) for x in deleted_paths] 87 if len(deleted_paths) != 0: 88 print("-------") 89 print("------- Remove deleted upstream files") 90 print("-------") 91 if repo_type == RepoType.GIT: 92 cmd = f"git rm {' '.join(deleted_paths)}" 93 stdout_lines = run_git(cmd, ".") 94 else: 95 cmd = f"hg rm {' '.join(deleted_paths)}" 96 stdout_lines = run_hg(cmd) 97 log_output_lines(stdout_lines, log_dir, "log-deleted-upstream-files.txt") 98 99 100 def add_new_upstream_files( 101 github_path, github_sha, target_dir, log_dir, handle_noop_commit 102 ): 103 if handle_noop_commit: 104 return 105 added_paths = filter_git_changes(github_path, github_sha, "A") 106 added_paths = [re.sub("^.\t", "", x) for x in added_paths] 107 added_paths = [os.path.join(target_dir, x) for x in added_paths] 108 if len(added_paths) != 0: 109 print("-------") 110 print("------- Add new upstream files") 111 print("-------") 112 if repo_type == RepoType.GIT: 113 cmd = f"git add {' '.join(added_paths)}" 114 stdout_lines = run_git(cmd, ".") 115 else: 116 cmd = f"hg add {' '.join(added_paths)}" 117 stdout_lines = run_hg(cmd) 118 log_output_lines(stdout_lines, log_dir, "log-new-upstream-files.txt") 119 120 121 def handle_renamed_upstream_files( 122 github_path, github_sha, target_dir, log_dir, handle_noop_commit 123 ): 124 if handle_noop_commit: 125 return 126 renamed_paths = filter_git_changes(github_path, github_sha, "R") 127 renamed_paths = [re.sub("^.[0-9]+\t", "", x) for x in renamed_paths] 128 renamed_paths = [x.split("\t") for x in renamed_paths] 129 renamed_paths = [ 130 " ".join([os.path.join(target_dir, file) for file in rename_pair]) 131 for rename_pair in renamed_paths 132 ] 133 if len(renamed_paths) != 0: 134 print("-------") 135 print("------- Handle renamed upstream files") 136 print("-------") 137 for x in renamed_paths: 138 if repo_type == RepoType.GIT: 139 # git doesn't have an equivalent command to mercurial 140 # where a rename/mv can be indicated retroactively, so 141 # remove the old file and add the new file. 142 source, destination = x.split(" ") 143 cmd = f"git rm {source} ; git add {destination}" 144 stdout_lines = run_shell(cmd) 145 else: 146 cmd = f"hg rename --after {x}" 147 stdout_lines = run_hg(cmd) 148 log_output_lines(stdout_lines, log_dir, "log-renamed-upstream-files.txt") 149 150 151 def commit_all_changes(github_sha, commit_msg_filename, target_dir): 152 print("-------") 153 print(f"------- Commit vendored changes from {github_sha}") 154 print("-------") 155 if repo_type == RepoType.GIT: 156 cmd = f"git commit --file={commit_msg_filename} {target_dir}" 157 run_git(cmd, ".") 158 else: 159 cmd = f"hg commit -l {commit_msg_filename} {target_dir}" 160 run_hg(cmd) 161 162 163 def vendor_and_commit( 164 script_dir, 165 github_path, 166 github_branch, 167 github_sha, 168 target_dir, 169 state_dir, 170 log_dir, 171 commit_msg_filename, 172 ): 173 # register the exit handler after the arg parser completes so '--help' doesn't exit with 174 # an error. 175 atexit.register(early_exit_handler) 176 177 print(f"script_dir: {script_dir}") 178 print(f"github_path: {github_path}") 179 print(f"github_branch: {github_branch}") 180 print(f"github_sha: {github_sha}") 181 print(f"target_dir: {target_dir}") 182 print(f"state_dir: {state_dir}") 183 print(f"log_dir: {log_dir}") 184 print(f"commit_msg_filename: {commit_msg_filename}") 185 186 resume_state_filename = os.path.join(state_dir, "vendor_and_commit.resume") 187 188 noop_commit_path = os.path.join(state_dir, f"{github_sha}.no-op-cherry-pick-msg") 189 handle_noop_commit = os.path.exists(noop_commit_path) 190 print(f"noop_commit_path: {noop_commit_path}") 191 print(f"handle_noop_commit: {handle_noop_commit}") 192 if handle_noop_commit: 193 print("***") 194 print("*** Detected special commit msg, setting handle_noop_commit.") 195 print("*** This commit is flagged as having been handled by a") 196 print("*** previous commit, meaning the changed file count between") 197 print("*** upstream and our vendored commit will not match. The") 198 print("*** commit message is annotated with info on the previous") 199 print("*** commit.") 200 print("***") 201 202 resume_state = "" 203 if os.path.exists(resume_state_filename): 204 resume_state = get_last_line(resume_state_filename).strip() 205 print(f"resume_state: '{resume_state}'") 206 207 if len(resume_state) == 0: 208 update_resume_state("resume2", resume_state_filename) 209 error_help.set_help( 210 f"Running script '{script_dir}/vendor-libwebrtc.py' failed.\n" 211 f"Please manually confirm that all changes from git ({github_path})\n" 212 "are reflected in the output of 'hg diff'" 213 ) 214 vendor_current_stack(github_branch, github_path, script_dir) 215 error_help.set_help(None) 216 217 if len(resume_state) == 0 or resume_state == "resume2": 218 resume_state = "" 219 update_resume_state("resume3", resume_state_filename) 220 error_help.set_help( 221 "An error occurred while restoring moz.build files after vendoring.\n" 222 "Verify no moz.build files are modified, missing, or changed." 223 ) 224 restore_mozbuild_files(target_dir, log_dir) 225 error_help.set_help(None) 226 227 if len(resume_state) == 0 or resume_state == "resume3": 228 resume_state = "" 229 update_resume_state("resume4", resume_state_filename) 230 error_help.set_help( 231 "An error occurred while removing deleted upstream files.\n" 232 "Verify files deleted in the newest upstream commit are also\n" 233 "shown as deleted in the output of 'hg status'" 234 ) 235 remove_deleted_upstream_files( 236 github_path, github_sha, target_dir, log_dir, handle_noop_commit 237 ) 238 error_help.set_help(None) 239 240 if len(resume_state) == 0 or resume_state == "resume4": 241 resume_state = "" 242 update_resume_state("resume5", resume_state_filename) 243 error_help.set_help( 244 "An error occurred while adding new upstream files.\n" 245 "Verify files added in the newest upstream commit are also\n" 246 "shown as added in the output of 'hg status'" 247 ) 248 add_new_upstream_files( 249 github_path, github_sha, target_dir, log_dir, handle_noop_commit 250 ) 251 error_help.set_help(None) 252 253 if len(resume_state) == 0 or resume_state == "resume5": 254 resume_state = "" 255 update_resume_state("resume6", resume_state_filename) 256 error_help.set_help( 257 "An error occurred while adding handling renamed upstream files.\n" 258 "Verify files renamed in the newest upstream commit are also\n" 259 "shown as renamed/moved in the output of 'hg status'" 260 ) 261 handle_renamed_upstream_files( 262 github_path, github_sha, target_dir, log_dir, handle_noop_commit 263 ) 264 error_help.set_help(None) 265 266 if len(resume_state) == 0 or resume_state == "resume6": 267 resume_state = "" 268 update_resume_state("", resume_state_filename) 269 error_help.set_help( 270 "An error occurred while committing the vendored changes to Mercurial.\n" 271 ) 272 # write the base of latest commit sha for use next time 273 with open(os.path.join(target_dir, "README.mozilla.last-vendor"), "a") as f: 274 # write the the command line used 275 f.write("# base of lastest vendoring\n") 276 f.write(f"{github_sha}\n") 277 commit_all_changes(github_sha, commit_msg_filename, target_dir) 278 error_help.set_help(None) 279 280 # unregister the exit handler so the normal exit doesn't falsely 281 # report as an error. 282 atexit.unregister(early_exit_handler) 283 284 285 if __name__ == "__main__": 286 # first, check which repo we're in, git or hg 287 if repo_type is None or not isinstance(repo_type, RepoType): 288 error_help.set_help("Unable to detect repo (git or hg)") 289 sys.exit(1) 290 291 default_target_dir = "third_party/libwebrtc" 292 default_state_dir = ".moz-fast-forward" 293 default_log_dir = ".moz-fast-forward/logs" 294 295 parser = argparse.ArgumentParser( 296 description="Vendor from local copy of moz-libwebrtc and commit" 297 ) 298 parser.add_argument( 299 "--repo-path", 300 required=True, 301 help="path to libwebrtc repo", 302 ) 303 parser.add_argument( 304 "--script-path", 305 required=True, 306 help="path to script directory", 307 ) 308 parser.add_argument( 309 "--commit-sha", 310 required=True, 311 help="sha of commit to examine", 312 ) 313 parser.add_argument( 314 "--branch", 315 default="mozpatches", 316 help="moz-libwebrtc branch (defaults to mozpatches)", 317 ) 318 parser.add_argument( 319 "--target-path", 320 default=default_target_dir, 321 help=f"target path for vendoring (defaults to {default_target_dir})", 322 ) 323 parser.add_argument( 324 "--state-path", 325 default=default_state_dir, 326 help=f"path to state directory (defaults to {default_state_dir})", 327 ) 328 parser.add_argument( 329 "--log-path", 330 default=default_log_dir, 331 help=f"path to log directory (defaults to {default_log_dir})", 332 ) 333 parser.add_argument( 334 "--commit-msg-path", 335 required=True, 336 help="path to file containing commit message", 337 ) 338 args = parser.parse_args() 339 340 vendor_and_commit( 341 args.script_path, 342 args.repo_path, 343 args.branch, 344 args.commit_sha, 345 args.target_path, # os.path.abspath(args.target_path), 346 args.state_path, 347 args.log_path, 348 args.commit_msg_path, 349 )