cherry_pick_commit.py (18761B)
1 # This Source Code Form is subject to the terms of the Mozilla Public 2 # License, v. 2.0. If a copy of the MPL was not distributed with this 3 # file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 5 import argparse 6 import atexit 7 import os 8 import re 9 import shutil 10 import sys 11 12 from filter_git_changes import filter_git_changes 13 from restore_patch_stack import restore_patch_stack 14 from run_operations import ( 15 ErrorHelp, 16 RepoType, 17 detect_repo_type, 18 get_last_line, 19 git_status, 20 run_git, 21 run_hg, 22 run_shell, 23 update_resume_state, 24 ) 25 from vendor_and_commit import vendor_and_commit 26 27 # This script cherry-picks an upstream commit with the appropriate 28 # commit message, and adds the no-op commit tracking file for the when 29 # we vendor the upstream commit later. 30 31 script_name = os.path.basename(__file__) 32 error_help = ErrorHelp() 33 error_help.set_prefix(f"*** ERROR *** {script_name} did not complete successfully") 34 35 repo_type = detect_repo_type() 36 37 38 def early_exit_handler(): 39 error_help.print_help() 40 41 42 def write_commit_message_file( 43 commit_message_filename, 44 github_path, 45 github_sha, 46 bug_number, 47 reviewers, 48 ): 49 print(f"commit_message_filename: {commit_message_filename}") 50 print(f"github_path: {github_path}") 51 print(f"github_sha: {github_sha}") 52 print(f"bug_number: {bug_number}") 53 54 cmd = f"git show --format=%H --no-patch {github_sha}" 55 stdout_lines = run_git(cmd, github_path) 56 github_long_sha = stdout_lines[0] 57 print(f"github_long_sha: {github_long_sha}") 58 59 cmd = f"git show --format=%s%n%n%b --no-patch {github_sha}" 60 github_commit_msg_lines = run_git(cmd, github_path) 61 62 with open(commit_message_filename, "w") as ofile: 63 ofile.write( 64 f"Bug {bug_number} - Cherry-pick upstream libwebrtc commit {github_sha} r?{reviewers}" 65 ) 66 ofile.write("\n") 67 ofile.write("\n") 68 ofile.write( 69 f"Upstream commit: https://webrtc.googlesource.com/src/+/{github_long_sha}" 70 ) 71 ofile.write("\n") 72 for line in github_commit_msg_lines: 73 ofile.write(f" {line}") 74 ofile.write("\n") 75 76 77 def cherry_pick_commit( 78 commit_message_filename, 79 github_path, 80 github_sha, 81 ): 82 print(f"commit_message_filename: {commit_message_filename}") 83 print(f"github_path: {github_path}") 84 print(f"github_sha: {github_sha}") 85 86 cmd = f"git cherry-pick --no-commit {github_sha}" 87 run_git(cmd, github_path) 88 89 cmd = f"git commit --file {os.path.abspath(commit_message_filename)}" 90 run_git(cmd, github_path) 91 92 93 def write_noop_tracking_file( 94 github_sha, 95 bug_number, 96 ): 97 noop_basename = f"{github_sha}.no-op-cherry-pick-msg" 98 noop_filename = os.path.join(args.state_path, noop_basename) 99 print(f"noop_filename: {noop_filename}") 100 with open(noop_filename, "w") as ofile: 101 ofile.write(f"We cherry-picked this in bug {bug_number}") 102 ofile.write("\n") 103 shutil.copy(noop_filename, args.patch_path) 104 if repo_type == RepoType.GIT: 105 cmd = f"git add {os.path.join(args.patch_path, noop_basename)}" 106 run_git(cmd, ".") 107 cmd = "git commit --amend --no-edit" 108 run_git(cmd, ".") 109 else: 110 cmd = f"hg add {os.path.join(args.patch_path, noop_basename)}" 111 run_hg(cmd) 112 cmd = f"hg amend {os.path.join(args.patch_path, noop_basename)}" 113 run_hg(cmd) 114 115 116 if __name__ == "__main__": 117 # first, check which repo we're in, git or hg 118 if repo_type is None or not isinstance(repo_type, RepoType): 119 print("Unable to detect repo (git or hg)") 120 sys.exit(1) 121 122 default_target_dir = "third_party/libwebrtc" 123 default_state_dir = ".moz-fast-forward" 124 default_log_dir = ".moz-fast-forward/logs" 125 default_tmp_dir = ".moz-fast-forward/tmp" 126 default_script_dir = "dom/media/webrtc/third_party_build" 127 default_patch_dir = "third_party/libwebrtc/moz-patch-stack" 128 default_repo_dir = ".moz-fast-forward/moz-libwebrtc" 129 default_tar_name = "moz-libwebrtc.tar.gz" 130 131 parser = argparse.ArgumentParser( 132 description="Cherry-pick upstream libwebrtc commit" 133 ) 134 parser.add_argument( 135 "--target-path", 136 default=default_target_dir, 137 help=f"target path for vendoring (defaults to {default_target_dir})", 138 ) 139 parser.add_argument( 140 "--state-path", 141 default=default_state_dir, 142 help=f"path to state directory (defaults to {default_state_dir})", 143 ) 144 parser.add_argument( 145 "--log-path", 146 default=default_log_dir, 147 help=f"path to log directory (defaults to {default_log_dir})", 148 ) 149 parser.add_argument( 150 "--tmp-path", 151 default=default_tmp_dir, 152 help=f"path to tmp directory (defaults to {default_tmp_dir})", 153 ) 154 parser.add_argument( 155 "--script-path", 156 default=default_script_dir, 157 help=f"path to script directory (defaults to {default_script_dir})", 158 ) 159 parser.add_argument( 160 "--repo-path", 161 default=default_repo_dir, 162 help=f"path to moz-libwebrtc repo (defaults to {default_repo_dir})", 163 ) 164 parser.add_argument( 165 "--tar-name", 166 default=default_tar_name, 167 help=f"name of tar file (defaults to {default_tar_name})", 168 ) 169 parser.add_argument( 170 "--commit-sha", 171 required=True, 172 help="sha of commit to examine", 173 ) 174 parser.add_argument( 175 "--branch", 176 default="mozpatches", 177 help="moz-libwebrtc branch (defaults to mozpatches)", 178 ) 179 parser.add_argument( 180 "--commit-bug-number", 181 type=int, 182 required=True, 183 help="integer Bugzilla number (example: 1800920)", 184 ) 185 parser.add_argument( 186 "--patch-path", 187 default=default_patch_dir, 188 help=f"path to save patches (defaults to {default_patch_dir})", 189 ) 190 parser.add_argument( 191 "--reviewers", 192 required=True, 193 help='reviewers for cherry-picked patch (like "ng,mjf")', 194 ) 195 parser.add_argument( 196 "--abort", 197 action="store_true", 198 default=False, 199 help="abort an interrupted cherry-pick", 200 ) 201 parser.add_argument( 202 "--continue", 203 dest="cont", # because args.continue causes syntax errors 204 action="store_true", 205 default=False, 206 help="continue an interrupted cherry-pick", 207 ) 208 parser.add_argument( 209 "--skip-restore", 210 action="store_true", 211 default=False, 212 help="to skip restoring the patch-stack if it is already restored and verified", 213 ) 214 args = parser.parse_args() 215 216 # register the exit handler after the arg parser completes so '--help' doesn't exit with 217 # an error. 218 atexit.register(early_exit_handler) 219 220 commit_message_filename = os.path.join(args.tmp_path, "cherry-pick-commit_msg.txt") 221 222 resume_state_filename = os.path.join(args.state_path, "cherry_pick_commit.resume") 223 resume_state = "" 224 if os.path.exists(resume_state_filename): 225 resume_state = get_last_line(resume_state_filename).strip() 226 print(f"resume_state: '{resume_state}'") 227 228 # don't allow abort/continue flags if not in resume state 229 error_help.set_help( 230 "--abort or --continue flags are not allowed when not in resume state" 231 ) 232 if len(resume_state) == 0 and (args.abort or args.cont): 233 sys.exit(1) 234 error_help.set_help(None) 235 236 # detect missing abort/continue flags if in resume state 237 error_help.set_help("cherry-pick in progress, use --abort or --continue") 238 if len(resume_state) != 0 and not args.abort and not args.cont: 239 sys.exit(1) 240 error_help.set_help(None) 241 242 # handle aborting cherry-pick 243 if args.abort: 244 if repo_type == RepoType.GIT: 245 run_git(f"git restore --staged {args.target_path}", ".") 246 run_git(f"git restore {args.target_path}", ".") 247 run_git(f"git clean -f {args.target_path}", ".") 248 else: 249 run_hg("hg revert --all") 250 run_hg(f"hg purge {args.target_path}") 251 # If the resume_state is not resume2 or resume3 that means we 252 # may have committed something to the Mozilla repo. First we 253 # need to check for our cherry-pick commit message, and if 254 # found, remove that commit. 255 if resume_state not in ("resume2", "resume3"): 256 # check for committed patch and backout 257 if repo_type == RepoType.GIT: 258 stdout_lines = run_git("git show --oneline --no-patch", ".") 259 else: 260 stdout_lines = run_hg("hg log --template {desc|firstline}\n -r .") 261 # check for "Cherry-pick upstream libwebrtc commit" 262 print(f"stdout_lines before filter: {stdout_lines}") 263 stdout_lines = [ 264 line 265 for line in stdout_lines 266 if re.findall("Cherry-pick upstream libwebrtc commit", line) 267 ] 268 print(f"looking for commit: {stdout_lines}") 269 if len(stdout_lines) > 0: 270 if repo_type == RepoType.GIT: 271 cmd = "git reset --hard HEAD^" 272 print(f"calling '{cmd}'") 273 run_git(cmd, ".") 274 else: 275 cmd = "hg prune ." 276 print(f"calling '{cmd}'") 277 run_hg(cmd) 278 print("restoring patch stack") 279 restore_patch_stack( 280 args.repo_path, 281 args.branch, 282 os.path.abspath(args.patch_path), 283 args.state_path, 284 args.tar_name, 285 ) 286 # reset the resume file 287 print("reset resume file") 288 update_resume_state("", resume_state_filename) 289 print("after resetting resume file") 290 atexit.unregister(early_exit_handler) 291 sys.exit(0) 292 293 # make sure the relevant bits of the Mozilla repo are clean before 294 # beginning 295 error_help.set_help( 296 f"There are modified or untracked files under {args.target_path}.\n" 297 f"Please cleanup the repo under {args.target_path} before running {script_name}" 298 ) 299 if repo_type == RepoType.GIT: 300 stdout_lines = git_status(".", args.target_path) 301 else: 302 stdout_lines = run_hg(f"hg status {args.target_path}") 303 if len(stdout_lines) != 0: 304 sys.exit(1) 305 error_help.set_help(None) 306 307 if len(resume_state) == 0: 308 if args.skip_restore is False: 309 # Restoring is done with each new cherry-pick to ensure that 310 # guidance from verify_vendoring (the next step) is 311 # accurate. If living dangerously is your thing, you can 312 # skip this step. 313 print("restoring patch stack") 314 restore_patch_stack( 315 args.repo_path, 316 args.branch, 317 os.path.abspath(args.patch_path), 318 args.state_path, 319 args.tar_name, 320 ) 321 update_resume_state("resume2", resume_state_filename) 322 323 # make sure the github repo exists 324 error_help.set_help( 325 f"No moz-libwebrtc github repo found at {args.repo_path}\n" 326 f"Please run restore_patch_stack.py before running {script_name}" 327 ) 328 if not os.path.exists(args.repo_path): 329 sys.exit(1) 330 error_help.set_help(None) 331 332 # Other scripts assume the short-sha is used for various comparisons, so 333 # make sure the provided sha is in the short form. 334 cmd = f"git rev-parse --short {args.commit_sha}" 335 args.commit_sha = run_git(cmd, args.repo_path)[0] 336 337 if len(resume_state) == 0 or resume_state == "resume2": 338 resume_state = "" 339 update_resume_state("resume3", resume_state_filename) 340 # Run verify_vendoring to make sure we have a sane patch-stack. 341 print("verifying patch stack") 342 run_shell(f"bash {args.script_path}/verify_vendoring.sh", False) 343 344 if len(resume_state) == 0 or resume_state == "resume3": 345 resume_state = "" 346 update_resume_state("resume4", resume_state_filename) 347 print("-------") 348 print(f"------- write commit message file {commit_message_filename}") 349 print("-------") 350 write_commit_message_file( 351 commit_message_filename, 352 args.repo_path, 353 args.commit_sha, 354 args.commit_bug_number, 355 args.reviewers, 356 ) 357 358 if len(resume_state) == 0 or resume_state == "resume4": 359 resume_state = "" 360 update_resume_state("resume5", resume_state_filename) 361 print("-------") 362 print(f"------- cherry-pick {args.commit_sha} into {args.repo_path}") 363 print("-------") 364 full_commit_message_filename = os.path.abspath(commit_message_filename) 365 error_help.set_help( 366 f"The cherry-pick operation of {args.commit_sha} has failed.\n" 367 "To fix this issue, you will need to jump to the github\n" 368 f"repo at {args.repo_path} .\n" 369 "Please resolve all the cherry-pick conflicts, and commit the changes\n" 370 "using:\n" 371 f" git commit --file {full_commit_message_filename}\n" 372 "\n" 373 "When the github cherry-pick is complete, resume running this\n" 374 f"script ({script_name})" 375 ) 376 cherry_pick_commit( 377 commit_message_filename, 378 args.repo_path, 379 args.commit_sha, 380 ) 381 error_help.set_help(None) 382 383 if len(resume_state) == 0 or resume_state == "resume5": 384 resume_state = "" 385 update_resume_state("resume6", resume_state_filename) 386 print("-------") 387 print(f"------- vendor from {args.repo_path}") 388 print("-------") 389 error_help.set_help( 390 f"Vendoring the newly cherry-picked git commit ({args.commit_sha}) has failed.\n" 391 "The Mozilla repo is in an unknown state. This failure is\n" 392 "rare and thus makes it difficult to provide definitive guidance.\n" 393 "In essence, the current failing command is:\n" 394 f"./mach python {args.script_path}/vendor_and_commit.py \\\n" 395 f" --script-path {args.script_path} \\\n" 396 f" --repo-path {args.repo_path} \\\n" 397 f" --branch {args.branch} \\\n" 398 f" --commit-sha {args.commit_sha} \\\n" 399 f" --target-path {args.target_path} \\\n" 400 f" --state-path {args.state_path} \\\n" 401 f" --log-path {args.log_path} \\\n" 402 f" --commit-msg-path {commit_message_filename}\n" 403 "\n" 404 "Additional guidance may be in the terminal output above. Resolve\n" 405 "issues encountered by vendor_and_commit.py followed by re-running\n" 406 "vendor_and_commit.py to resume/complete its processing. After\n" 407 "vendor_and_commit.py completes successfully, resume running\n" 408 f"this script ({script_name})" 409 ) 410 vendor_and_commit( 411 args.script_path, 412 args.repo_path, 413 args.branch, 414 args.commit_sha, 415 args.target_path, # os.path.abspath(args.target_path), 416 args.state_path, 417 args.log_path, 418 commit_message_filename, 419 ) 420 error_help.set_help(None) 421 422 if len(resume_state) == 0 or resume_state == "resume6": 423 resume_state = "" 424 update_resume_state("resume7", resume_state_filename) 425 error_help.set_help( 426 "Reverting change to 'third_party/libwebrtc/README.mozilla.last-vendor'\n" 427 "has failed. The cherry-pick commit should not modify\n" 428 "'third_party/libwebrtc/README.mozilla'. If necessary\n" 429 "manually revert changes to 'third_party/libwebrtc/README.mozilla'\n" 430 f"in the cherry-pick commit and re-run {script_name}\n" 431 "to complete the cherry-pick processing." 432 ) 433 # The normal vendoring process updates README.mozilla with info 434 # on what commit was vendored and the command line used to do 435 # the vendoring. Since we're only reusing the vendoring script 436 # here, we don't want to update the README.mozilla file. 437 if repo_type == RepoType.GIT: 438 cmd = ( 439 "git checkout HEAD^ -- third_party/libwebrtc/README.mozilla.last-vendor" 440 ) 441 run_git(cmd, ".") 442 cmd = "git commit --amend --no-edit" 443 run_git(cmd, ".") 444 else: 445 cmd = "hg revert -r tip^ third_party/libwebrtc/README.mozilla.last-vendor" 446 run_hg(cmd) 447 cmd = "hg amend" 448 run_hg(cmd) 449 error_help.set_help(None) 450 451 if len(resume_state) == 0 or resume_state == "resume7": 452 resume_state = "" 453 update_resume_state("resume8", resume_state_filename) 454 # get the files changed from the newly vendored cherry-pick 455 # commit in the Mozilla repo 456 if repo_type == RepoType.GIT: 457 cmd = "git show --format='' --name-status | grep -v 'README.'" 458 else: 459 cmd = "hg status --change tip --exclude '**/README.*'" 460 stdout_lines = run_shell(cmd) # run_shell to allow file wildcard 461 print(f"Mozilla repo changes:\n{stdout_lines}") 462 mozilla_file_change_cnt = len(stdout_lines) 463 464 # get the files changed from the original cherry-picked patch in 465 # the libwebrtc github repo 466 libwebrtc_paths_changed = filter_git_changes( 467 args.repo_path, args.commit_sha, None 468 ) 469 print(f"Libwebrtc repo changes:\n{libwebrtc_paths_changed}") 470 libwebrtc_file_change_cnt = len(libwebrtc_paths_changed) 471 472 error_help.set_help( 473 f"Vendoring the cherry-pick of commit {args.commit_sha} has failed due to mismatched\n" 474 f"changed file counts between the Mozilla repo ({mozilla_file_change_cnt}) " 475 f"and the libwebrtc repo ({libwebrtc_file_change_cnt}).\n" 476 "This may be because the mozilla patch-stack was not verified after\n" 477 "running restore_patch_stack.py. After reconciling the changes in\n" 478 f"the newly committed patch, please re-run {script_name} to complete\n" 479 "the cherry-pick processing." 480 ) 481 if mozilla_file_change_cnt != libwebrtc_file_change_cnt: 482 sys.exit(1) 483 error_help.set_help(None) 484 485 if len(resume_state) == 0 or resume_state == "resume8": 486 resume_state = "" 487 update_resume_state("", resume_state_filename) 488 print("-------") 489 print("------- write the noop tracking file") 490 print("-------") 491 write_noop_tracking_file(args.commit_sha, args.commit_bug_number) 492 493 # unregister the exit handler so the normal exit doesn't falsely 494 # report as an error. 495 atexit.unregister(early_exit_handler)