save_patch_stack.py (14098B)
1 # This Source Code Form is subject to the terms of the Mozilla Public 2 # License, v. 2.0. If a copy of the MPL was not distributed with this 3 # file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 import argparse 5 import atexit 6 import os 7 import re 8 import shutil 9 import sys 10 11 from run_operations import ( 12 ErrorHelp, 13 RepoType, 14 check_repo_status, 15 detect_repo_type, 16 git_is_config_set, 17 git_status, 18 is_mac_os, 19 run_git, 20 run_hg, 21 run_shell, 22 ) 23 24 # This script saves the mozilla patch stack and no-op commit tracking 25 # files. This makes our fast-forward process much more resilient by 26 # saving the intermediate state after each upstream commit is processed. 27 28 script_name = os.path.basename(__file__) 29 error_help = ErrorHelp() 30 error_help.set_prefix(f"*** ERROR *** {script_name} did not complete successfully") 31 32 repo_type = detect_repo_type() 33 34 35 @atexit.register 36 def early_exit_handler(): 37 error_help.print_help() 38 39 40 def build_repo_name_from_path(input_dir): 41 # strip off the (probably moz-patch-stack) patch-stack location 42 output_dir = os.path.dirname(os.path.relpath(input_dir)) 43 44 # if directory is under third_party (likely), give us the 45 # part after third_party 46 if os.path.commonpath([output_dir, "third_party"]) != "": 47 output_dir = os.path.relpath(output_dir, "third_party") 48 49 return output_dir 50 51 52 def write_patch_files_with_prefix( 53 github_path, 54 patch_directory, 55 start_commit_sha, 56 end_commit_sha, 57 prefix, 58 ): 59 cmd = f"git format-patch --keep-subject --no-signature --output-directory {patch_directory} {start_commit_sha}..{end_commit_sha}" 60 run_git(cmd, github_path) 61 62 # remove the commit summary from the file name and add provided prefix 63 patches_to_rename = os.listdir(patch_directory) 64 for file in patches_to_rename: 65 shortened_name = re.sub(r"^(\d\d\d\d)-.*\.patch", f"{prefix}\\1.patch", file) 66 os.rename( 67 os.path.join(patch_directory, file), 68 os.path.join(patch_directory, shortened_name), 69 ) 70 71 72 def write_prestack_and_standard_patches( 73 github_path, 74 patch_directory, 75 start_commit_sha, 76 end_commit_sha, 77 ): 78 # grab the log of our patches that live on top of libwebrtc, and find 79 # the commit of our base patch. 80 cmd = f"git log --oneline {start_commit_sha}..{end_commit_sha}" 81 stdout_lines = run_git(cmd, github_path) 82 base_commit_summary = "Bug 1376873 - Rollup of local modifications" 83 found_lines = [s for s in stdout_lines if base_commit_summary in s] 84 if len(found_lines) == 0: 85 error_help.set_help( 86 "The base commit for Mozilla's patch-stack was not found in the\n" 87 "git log. The commit summary we're looking for is:\n" 88 f"{base_commit_summary}" 89 ) 90 sys.exit(1) 91 base_commit_sha = found_lines[0].split(" ")[0] 92 print(f"Found base_commit_sha: {base_commit_sha}") 93 94 # First, a word about pre-stack and standard patches. During the 95 # libwebrtc update process, there are 2 cases where we insert 96 # patches between the upstream trunk commit we're current based on 97 # and the Mozilla patch-stack: 98 # 1) Release branch commits are frequently used. These are patches 99 # that are typically cherry-picks of further upstream commits 100 # that are needed to fix bugs in a upstream release. When 101 # beginning the Mozilla libwebrtc update process, it is necessary 102 # to copy those release branch commits and insert them between 103 # the trunk commit we're based on, and the Mozilla patch-stack we 104 # carry in order for vendoring to produce the same result with 105 # which we ended the previous update. 106 # 2) During update process, to avoid fixing/unfixing/refixing rebase 107 # conflicts due to upstream landing/reverting/relanding patches, 108 # our update scripts look ahead in the upstream commits for 109 # reverted and relanded patches. If found, we insert the 110 # reverted commit before our Mozilla patch-stack. This has the 111 # effect of creating a virtually empty commit for the patch that 112 # originally lands and then the revert commit. Using this 113 # technique allows us to fix any potential rebase conflicts when 114 # the commit is eventually relanded the final time. 115 # 116 # Pre-stack commits (written with a 'p' prefix) are everything that 117 # we insert between upstream and the Mozilla patch-stack from the 118 # two categories above. 119 # 120 # Standard commits (written with a 's' prefix) are the Mozilla 121 # patch-stack commits. 122 # 123 # Note: the prefixes are also conveniently alphabetical so that 124 # restoring them can be done with a simple 'git am *.patch' command. 125 126 # write only the pre-stack patches out 127 write_patch_files_with_prefix( 128 github_path, patch_directory, f"{start_commit_sha}", f"{base_commit_sha}^", "p" 129 ) 130 131 # write only the "standard" stack patches out 132 write_patch_files_with_prefix( 133 github_path, patch_directory, f"{base_commit_sha}^", f"{end_commit_sha}", "s" 134 ) 135 136 137 def handle_missing_files(patch_directory): 138 # get missing files (that should be marked removed) 139 if repo_type == RepoType.GIT: 140 stdout_lines = git_status(".", patch_directory) 141 stdout_lines = [ 142 m[0] for m in (re.findall(r"^ D (.*)", line) for line in stdout_lines) if m 143 ] 144 if len(stdout_lines) != 0: 145 cmd = f"git rm {' '.join(stdout_lines)}" 146 run_git(cmd, ".") 147 else: 148 cmd = f"hg status --no-status --deleted {patch_directory}" 149 stdout_lines = run_hg(cmd) 150 if len(stdout_lines) != 0: 151 cmd = f"hg rm {' '.join(stdout_lines)}" 152 run_hg(cmd) 153 154 155 def handle_unknown_files(patch_directory): 156 # get unknown files (that should be marked added) 157 if repo_type == RepoType.GIT: 158 stdout_lines = git_status(".", patch_directory) 159 stdout_lines = [ 160 m[0] 161 for m in (re.findall(r"^\?\? (.*)", line) for line in stdout_lines) 162 if m 163 ] 164 if len(stdout_lines) != 0: 165 cmd = f"git add {' '.join(stdout_lines)}" 166 run_git(cmd, ".") 167 else: 168 cmd = f"hg status --no-status --unknown {patch_directory}" 169 stdout_lines = run_hg(cmd) 170 if len(stdout_lines) != 0: 171 cmd = f"hg add {' '.join(stdout_lines)}" 172 run_hg(cmd) 173 174 175 def handle_modified_files(patch_directory): 176 if repo_type == RepoType.HG: 177 # for the mercurial case, there is no work to be done here 178 return 179 180 stdout_lines = git_status(".", patch_directory) 181 stdout_lines = [ 182 m[0] for m in (re.findall(r"^ M (.*)", line) for line in stdout_lines) if m 183 ] 184 if len(stdout_lines) != 0: 185 cmd = f"git add {' '.join(stdout_lines)}" 186 run_git(cmd, ".") 187 188 189 def save_patch_stack( 190 github_path, 191 github_branch, 192 patch_directory, 193 state_directory, 194 target_branch_head, 195 bug_number, 196 no_pre_stack, 197 ): 198 # remove the current patch files 199 files_to_remove = os.listdir(patch_directory) 200 for file in files_to_remove: 201 os.remove(os.path.join(patch_directory, file)) 202 203 # find the base of the patch stack 204 cmd = f"git merge-base {github_branch} {target_branch_head}" 205 stdout_lines = run_git(cmd, github_path) 206 merge_base = stdout_lines[0] 207 208 if no_pre_stack: 209 write_patch_files_with_prefix( 210 github_path, patch_directory, f"{merge_base}", f"{github_branch}", "" 211 ) 212 else: 213 write_prestack_and_standard_patches( 214 github_path, 215 patch_directory, 216 f"{merge_base}", 217 f"{github_branch}", 218 ) 219 220 # remove the unhelpful first line of the patch files that only 221 # causes diff churn. For reasons why we can't skip creating backup 222 # files during the in-place editing, see: 223 # https://stackoverflow.com/questions/5694228/sed-in-place-flag-that-works-both-on-mac-bsd-and-linux 224 run_shell(f"sed -i'.bak' -e '1d' {patch_directory}/*.patch") 225 run_shell(f"rm {patch_directory}/*.patch.bak") 226 227 # it is also helpful to save the no-op-cherry-pick-msg files from 228 # the state directory so that if we're restoring a patch-stack we 229 # also restore the possibly consumed no-op tracking files. 230 if state_directory != "": 231 no_op_files = [ 232 path 233 for path in os.listdir(state_directory) 234 if re.findall(".*no-op-cherry-pick-msg$", path) 235 ] 236 for file in no_op_files: 237 shutil.copy(os.path.join(state_directory, file), patch_directory) 238 239 handle_missing_files(patch_directory) 240 handle_unknown_files(patch_directory) 241 handle_modified_files(patch_directory) 242 243 # if any files are marked for add/remove/modify, commit them 244 if repo_type == RepoType.GIT: 245 stdout_lines = git_status(".", patch_directory) 246 stdout_lines = [ 247 line for line in stdout_lines if re.findall(r"^(M|A|D) .*", line) 248 ] 249 else: 250 cmd = f"hg status --added --removed --modified {patch_directory}" 251 stdout_lines = run_hg(cmd) 252 if (len(stdout_lines)) != 0: 253 print(f"Updating {len(stdout_lines)} files in {patch_directory}") 254 if bug_number is None: 255 if repo_type == RepoType.GIT: 256 run_git("git commit --amend --no-edit", ".") 257 else: 258 run_hg("hg amend") 259 else: 260 cmd = ( 261 f"{'git commit -m' if repo_type == RepoType.GIT else 'hg commit --message'} " 262 f"'Bug {bug_number} - " 263 f"updated {build_repo_name_from_path(patch_directory)} " 264 f"patch stack' {patch_directory}" 265 ) 266 stdout_lines = run_shell(cmd) 267 268 269 def verify_git_repo_configuration(): 270 config_help = [ 271 "This script fails frequently on macOS (Darwin) without running the", 272 "following configuration steps:", 273 " git config set feature.manyFiles true", 274 " git update-index --index-version 4", 275 " git config set core.fsmonitor true", 276 "", 277 "Note: this configuration should be safe to remain set in the firefox", 278 "repository, and may increase everyday performance. If you would like", 279 "to revert the effects after using this script, please use:", 280 " git config unset core.fsmonitor", 281 " git config unset feature.manyFiles", 282 ] 283 if is_mac_os() and not ( 284 git_is_config_set("feature.manyfiles", ".") 285 and git_is_config_set("core.fsmonitor", ".") 286 ): 287 error_help.set_help("\n".join(config_help)) 288 sys.exit(1) 289 290 291 if __name__ == "__main__": 292 # first, check which repo we're in, git or hg 293 if repo_type is None or not isinstance(repo_type, RepoType): 294 error_help.set_help("Unable to detect repo (git or hg)") 295 sys.exit(1) 296 297 default_patch_dir = "third_party/libwebrtc/moz-patch-stack" 298 default_script_dir = "dom/media/webrtc/third_party_build" 299 default_state_dir = ".moz-fast-forward" 300 301 parser = argparse.ArgumentParser( 302 description="Save moz-libwebrtc github patch stack" 303 ) 304 parser.add_argument( 305 "--repo-path", 306 required=True, 307 help="path to libwebrtc repo", 308 ) 309 parser.add_argument( 310 "--branch", 311 default="mozpatches", 312 help="moz-libwebrtc branch (defaults to mozpatches)", 313 ) 314 parser.add_argument( 315 "--patch-path", 316 default=default_patch_dir, 317 help=f"path to save patches (defaults to {default_patch_dir})", 318 ) 319 parser.add_argument( 320 "--state-path", 321 default=default_state_dir, 322 help=f"path to state directory (defaults to {default_state_dir})", 323 ) 324 parser.add_argument( 325 "--target-branch-head", 326 required=True, 327 help="target branch head for fast-forward, should match MOZ_TARGET_UPSTREAM_BRANCH_HEAD in config_env", 328 ) 329 parser.add_argument( 330 "--script-path", 331 default=default_script_dir, 332 help=f"path to script directory (defaults to {default_script_dir})", 333 ) 334 parser.add_argument( 335 "--separate-commit-bug-number", 336 type=int, 337 help="integer Bugzilla number (example: 1800920), if provided will write patch stack as separate commit", 338 ) 339 parser.add_argument( 340 "--no-pre-stack", 341 action="store_true", 342 default=False, 343 help="don't look for pre-stack/standard patches, simply write the patches all sequentially", 344 ) 345 parser.add_argument( 346 "--skip-startup-sanity", 347 action="store_true", 348 default=False, 349 help="skip checking for clean repo and doing the initial verify vendoring", 350 ) 351 args = parser.parse_args() 352 353 if repo_type == RepoType.GIT: 354 verify_git_repo_configuration() 355 356 if not args.skip_startup_sanity: 357 # make sure the mercurial repo is clean before beginning 358 error_help.set_help( 359 "There are modified or untracked files in the repo.\n" 360 f"Please start with a clean repo before running {script_name}" 361 ) 362 stdout_lines = check_repo_status(repo_type) 363 if len(stdout_lines) != 0: 364 sys.exit(1) 365 366 # make sure the github repo exists 367 error_help.set_help( 368 f"No moz-libwebrtc github repo found at {args.repo_path}\n" 369 f"Please run restore_patch_stack.py before running {script_name}" 370 ) 371 if not os.path.exists(args.repo_path): 372 sys.exit(1) 373 error_help.set_help(None) 374 375 print("Verifying vendoring before saving patch-stack...") 376 run_shell(f"bash {args.script_path}/verify_vendoring.sh", False) 377 378 save_patch_stack( 379 args.repo_path, 380 args.branch, 381 os.path.abspath(args.patch_path), 382 args.state_path, 383 args.target_branch_head, 384 args.separate_commit_bug_number, 385 args.no_pre_stack, 386 ) 387 388 # unregister the exit handler so the normal exit doesn't falsely 389 # report as an error. 390 atexit.unregister(early_exit_handler)