mercurial.py (17185B)
1 #!/usr/bin/env python 2 # This Source Code Form is subject to the terms of the Mozilla Public 3 # License, v. 2.0. If a copy of the MPL was not distributed with this file, 4 # You can obtain one at http://mozilla.org/MPL/2.0/. 5 """Mercurial VCS support.""" 6 7 import hashlib 8 import os 9 import re 10 import subprocess 11 import sys 12 from collections import namedtuple 13 14 try: 15 from urlparse import urlsplit 16 except ImportError: 17 from urllib.parse import urlsplit 18 19 import mozharness 20 from mozharness.base.errors import HgErrorList, VCSException 21 from mozharness.base.log import LogMixin, OutputParser 22 from mozharness.base.script import ScriptMixin 23 from mozharness.base.transfer import TransferMixin 24 25 sys.path.insert(1, os.path.dirname(os.path.dirname(os.path.dirname(sys.path[0])))) 26 27 28 external_tools_path = os.path.join( 29 os.path.abspath(os.path.dirname(os.path.dirname(mozharness.__file__))), 30 "external_tools", 31 ) 32 33 34 HG_OPTIONS = ["--config", "ui.merge=internal:merge"] 35 36 # MercurialVCS {{{1 37 # TODO Make the remaining functions more mozharness-friendly. 38 # TODO Add the various tag functionality that are currently in 39 # build/tools/scripts to MercurialVCS -- generic tagging logic belongs here. 40 REVISION, BRANCH = 0, 1 41 42 43 class RepositoryUpdateRevisionParser(OutputParser): 44 """Parse `hg pull` output for "repository unrelated" errors.""" 45 46 revision = None 47 RE_UPDATED = re.compile("^updated to ([a-f0-9]{40})$") 48 49 def parse_single_line(self, line): 50 m = self.RE_UPDATED.match(line) 51 if m: 52 self.revision = m.group(1) 53 54 return super().parse_single_line(line) 55 56 57 def make_hg_url(hg_host, repo_path, protocol="http", revision=None, filename=None): 58 """Helper function. 59 60 Construct a valid hg url from a base hg url (hg.mozilla.org), 61 repo_path, revision and possible filename 62 """ 63 base = "%s://%s" % (protocol, hg_host) 64 repo = "/".join(p.strip("/") for p in [base, repo_path]) 65 if not filename: 66 if not revision: 67 return repo 68 else: 69 return "/".join([p.strip("/") for p in [repo, "rev", revision]]) 70 else: 71 assert revision 72 return "/".join([p.strip("/") for p in [repo, "raw-file", revision, filename]]) 73 74 75 class MercurialVCS(ScriptMixin, LogMixin, TransferMixin): 76 # For the most part, scripts import mercurial, update 77 # tag-release.py imports 78 # apply_and_push, update, get_revision, out, BRANCH, REVISION, 79 # get_branches, cleanOutgoingRevs 80 81 def __init__(self, log_obj=None, config=None, vcs_config=None, script_obj=None): 82 super().__init__() 83 self.can_share = None 84 self.log_obj = log_obj 85 self.script_obj = script_obj 86 if config: 87 self.config = config 88 else: 89 self.config = {} 90 # vcs_config = { 91 # hg_host: hg_host, 92 # repo: repository, 93 # branch: branch, 94 # revision: revision, 95 # ssh_username: ssh_username, 96 # ssh_key: ssh_key, 97 # } 98 self.vcs_config = vcs_config or {} 99 self.hg = self.query_exe("hg", return_type="list") + HG_OPTIONS 100 101 def _make_absolute(self, repo): 102 if repo.startswith("file://"): 103 path = repo[len("file://") :] 104 repo = "file://%s" % os.path.abspath(path) 105 elif "://" not in repo: 106 repo = os.path.abspath(repo) 107 return repo 108 109 def get_repo_name(self, repo): 110 return repo.rstrip("/").split("/")[-1] 111 112 def get_repo_path(self, repo): 113 repo = self._make_absolute(repo) 114 if repo.startswith("/"): 115 return repo.lstrip("/") 116 else: 117 return urlsplit(repo).path.lstrip("/") 118 119 def get_revision_from_path(self, path): 120 """Returns which revision directory `path` currently has checked out.""" 121 return self.get_output_from_command( 122 self.hg + ["parent", "--template", "{node}"], cwd=path 123 ) 124 125 def get_branch_from_path(self, path): 126 branch = self.get_output_from_command(self.hg + ["branch"], cwd=path) 127 return str(branch).strip() 128 129 def get_branches_from_path(self, path): 130 branches = [] 131 for line in self.get_output_from_command( 132 self.hg + ["branches", "-c"], cwd=path 133 ).splitlines(): 134 branches.append(line.split()[0]) 135 return branches 136 137 def hg_ver(self): 138 """Returns the current version of hg, as a tuple of 139 (major, minor, build)""" 140 ver_string = self.get_output_from_command(self.hg + ["-q", "version"]) 141 match = re.search(r"\(version ([0-9.]+)\)", ver_string) 142 if match: 143 bits = match.group(1).split(".") 144 if len(bits) < 3: 145 bits += (0,) 146 ver = tuple(int(b) for b in bits) 147 else: 148 ver = (0, 0, 0) 149 self.debug("Running hg version %s" % str(ver)) 150 return ver 151 152 def update(self, dest, branch=None, revision=None): 153 """Updates working copy `dest` to `branch` or `revision`. 154 If revision is set, branch will be ignored. 155 If neither is set then the working copy will be updated to the 156 latest revision on the current branch. Local changes will be 157 discarded. 158 """ 159 # If we have a revision, switch to that 160 msg = "Updating %s" % dest 161 if branch: 162 msg += " to branch %s" % branch 163 if revision: 164 msg += " revision %s" % revision 165 self.info("%s." % msg) 166 if revision is not None: 167 cmd = self.hg + ["update", "-C", "-r", revision] 168 if self.run_command(cmd, cwd=dest, error_list=HgErrorList): 169 raise VCSException("Unable to update %s to %s!" % (dest, revision)) 170 else: 171 # Check & switch branch 172 local_branch = self.get_branch_from_path(dest) 173 174 cmd = self.hg + ["update", "-C"] 175 176 # If this is different, checkout the other branch 177 if branch and branch != local_branch: 178 cmd.append(branch) 179 180 if self.run_command(cmd, cwd=dest, error_list=HgErrorList): 181 raise VCSException("Unable to update %s!" % dest) 182 return self.get_revision_from_path(dest) 183 184 def clone(self, repo, dest, branch=None, revision=None, update_dest=True): 185 """Clones hg repo and places it at `dest`, replacing whatever else 186 is there. The working copy will be empty. 187 188 If `revision` is set, only the specified revision and its ancestors 189 will be cloned. If revision is set, branch is ignored. 190 191 If `update_dest` is set, then `dest` will be updated to `revision` 192 if set, otherwise to `branch`, otherwise to the head of default. 193 """ 194 msg = "Cloning %s to %s" % (repo, dest) 195 if branch: 196 msg += " on branch %s" % branch 197 if revision: 198 msg += " to revision %s" % revision 199 self.info("%s." % msg) 200 parent_dest = os.path.dirname(dest) 201 if parent_dest and not os.path.exists(parent_dest): 202 self.mkdir_p(parent_dest) 203 if os.path.exists(dest): 204 self.info("Removing %s before clone." % dest) 205 self.rmtree(dest) 206 207 cmd = self.hg + ["clone"] 208 if not update_dest: 209 cmd.append("-U") 210 211 if revision: 212 cmd.extend(["-r", revision]) 213 elif branch: 214 # hg >= 1.6 supports -b branch for cloning 215 ver = self.hg_ver() 216 if ver >= (1, 6, 0): 217 cmd.extend(["-b", branch]) 218 219 cmd.extend([repo, dest]) 220 output_timeout = self.config.get( 221 "vcs_output_timeout", self.vcs_config.get("output_timeout") 222 ) 223 if ( 224 self.run_command(cmd, error_list=HgErrorList, output_timeout=output_timeout) 225 != 0 226 ): 227 raise VCSException("Unable to clone %s to %s!" % (repo, dest)) 228 229 if update_dest: 230 return self.update(dest, branch, revision) 231 232 def common_args(self, revision=None, branch=None, ssh_username=None, ssh_key=None): 233 """Fill in common hg arguments, encapsulating logic checks that 234 depend on mercurial versions and provided arguments 235 """ 236 args = [] 237 if ssh_username or ssh_key: 238 opt = ["-e", "ssh"] 239 if ssh_username: 240 opt[1] += " -l %s" % ssh_username 241 if ssh_key: 242 opt[1] += " -i %s" % ssh_key 243 args.extend(opt) 244 if revision: 245 args.extend(["-r", revision]) 246 elif branch: 247 if self.hg_ver() >= (1, 6, 0): 248 args.extend(["-b", branch]) 249 return args 250 251 def pull(self, repo, dest, update_dest=True, **kwargs): 252 """Pulls changes from hg repo and places it in `dest`. 253 254 If `revision` is set, only the specified revision and its ancestors 255 will be pulled. 256 257 If `update_dest` is set, then `dest` will be updated to `revision` 258 if set, otherwise to `branch`, otherwise to the head of default. 259 """ 260 msg = "Pulling %s to %s" % (repo, dest) 261 if update_dest: 262 msg += " and updating" 263 self.info("%s." % msg) 264 if not os.path.exists(dest): 265 # Error or clone? 266 # If error, should we have a halt_on_error=False above? 267 self.error("Can't hg pull in nonexistent directory %s." % dest) 268 return -1 269 # Convert repo to an absolute path if it's a local repository 270 repo = self._make_absolute(repo) 271 cmd = self.hg + ["pull"] 272 cmd.extend(self.common_args(**kwargs)) 273 cmd.append(repo) 274 output_timeout = self.config.get( 275 "vcs_output_timeout", self.vcs_config.get("output_timeout") 276 ) 277 if ( 278 self.run_command( 279 cmd, cwd=dest, error_list=HgErrorList, output_timeout=output_timeout 280 ) 281 != 0 282 ): 283 raise VCSException("Can't pull in %s!" % dest) 284 285 if update_dest: 286 branch = self.vcs_config.get("branch") 287 revision = self.vcs_config.get("revision") 288 return self.update(dest, branch=branch, revision=revision) 289 290 # Defines the places of attributes in the tuples returned by `out' 291 292 def out(self, src, remote, **kwargs): 293 """Check for outgoing changesets present in a repo""" 294 self.info("Checking for outgoing changesets from %s to %s." % (src, remote)) 295 cmd = self.hg + ["-q", "out", "--template", "{node} {branches}\n"] 296 cmd.extend(self.common_args(**kwargs)) 297 cmd.append(remote) 298 if os.path.exists(src): 299 try: 300 revs = [] 301 for line in ( 302 self.get_output_from_command(cmd, cwd=src, throw_exception=True) 303 .rstrip() 304 .split("\n") 305 ): 306 try: 307 rev, branch = line.split() 308 # Mercurial displays no branch at all if the revision 309 # is on "default" 310 except ValueError: 311 rev = line.rstrip() 312 branch = "default" 313 revs.append((rev, branch)) 314 return revs 315 except subprocess.CalledProcessError as inst: 316 # In some situations, some versions of Mercurial return "1" 317 # if no changes are found, so we need to ignore this return 318 # code 319 if inst.returncode == 1: 320 return [] 321 raise 322 323 def push(self, src, remote, push_new_branches=True, **kwargs): 324 # This doesn't appear to work with hg_ver < (1, 6, 0). 325 # Error out, or let you try? 326 self.info("Pushing new changes from %s to %s." % (src, remote)) 327 cmd = self.hg + ["push"] 328 cmd.extend(self.common_args(**kwargs)) 329 if push_new_branches and self.hg_ver() >= (1, 6, 0): 330 cmd.append("--new-branch") 331 cmd.append(remote) 332 status = self.run_command( 333 cmd, 334 cwd=src, 335 error_list=HgErrorList, 336 success_codes=(0, 1), 337 return_type="num_errors", 338 ) 339 if status: 340 raise VCSException("Can't push %s to %s!" % (src, remote)) 341 return status 342 343 @property 344 def robustcheckout_path(self): 345 """Path to the robustcheckout extension.""" 346 ext = os.path.join(external_tools_path, "robustcheckout.py") 347 if os.path.exists(ext): 348 return ext 349 350 def ensure_repo_and_revision(self): 351 """Makes sure that `dest` is has `revision` or `branch` checked out 352 from `repo`. 353 354 Do what it takes to make that happen, including possibly clobbering 355 dest. 356 """ 357 c = self.vcs_config 358 dest = c["dest"] 359 repo_url = c["repo"] 360 rev = c.get("revision") 361 branch = c.get("branch") 362 purge = c.get("clone_with_purge", False) 363 upstream = c.get("clone_upstream_url") 364 365 # The API here is kind of bad because we're relying on state in 366 # self.vcs_config instead of passing arguments. This confuses 367 # scripts that have multiple repos. This includes the clone_tools() 368 # step :( 369 370 if not rev and not branch: 371 self.warning('did not specify revision or branch; assuming "default"') 372 branch = "default" 373 374 share_base = c.get("vcs_share_base") or os.environ.get("HG_SHARE_BASE_DIR") 375 if share_base and c.get("use_vcs_unique_share"): 376 # Bug 1277041 - update migration scripts to support robustcheckout 377 # fake a share but don't really share 378 share_base = os.path.join(share_base, hashlib.md5(dest).hexdigest()) 379 380 # We require shared storage is configured because it guarantees we 381 # only have 1 local copy of logical repo stores. 382 if not share_base: 383 raise VCSException( 384 "vcs share base not defined; refusing to operate sub-optimally" 385 ) 386 387 if not self.robustcheckout_path: 388 raise VCSException("could not find the robustcheckout Mercurial extension") 389 390 # Log HG version and install info to aid debugging. 391 self.run_command(self.hg + ["--version"]) 392 self.run_command(self.hg + ["debuginstall", "--config=ui.username=worker"]) 393 394 args = self.hg + [ 395 "--config", 396 "extensions.robustcheckout=%s" % self.robustcheckout_path, 397 "robustcheckout", 398 repo_url, 399 dest, 400 "--sharebase", 401 share_base, 402 ] 403 if purge: 404 args.append("--purge") 405 if upstream: 406 args.extend(["--upstream", upstream]) 407 408 if rev: 409 args.extend(["--revision", rev]) 410 if branch: 411 args.extend(["--branch", branch]) 412 413 parser = RepositoryUpdateRevisionParser( 414 config=self.config, log_obj=self.log_obj 415 ) 416 if self.run_command(args, output_parser=parser): 417 raise VCSException("repo checkout failed!") 418 419 if not parser.revision: 420 raise VCSException("could not identify revision updated to") 421 422 return parser.revision 423 424 def cleanOutgoingRevs(self, reponame, remote, username, sshKey): 425 # TODO retry 426 self.info("Wiping outgoing local changes from %s to %s." % (reponame, remote)) 427 outgoingRevs = self.out( 428 src=reponame, remote=remote, ssh_username=username, ssh_key=sshKey 429 ) 430 for r in reversed(outgoingRevs): 431 self.run_command( 432 self.hg + ["strip", "-n", r[REVISION]], 433 cwd=reponame, 434 error_list=HgErrorList, 435 ) 436 437 def query_pushinfo(self, repository, revision): 438 """Query the pushdate and pushid of a repository/revision. 439 This is intended to be used on hg.mozilla.org/mozilla-central and 440 similar. It may or may not work for other hg repositories. 441 """ 442 PushInfo = namedtuple("PushInfo", ["pushid", "pushdate"]) 443 444 try: 445 url = "%s/json-pushes?changeset=%s" % (repository, revision) 446 self.info("Pushdate URL is: %s" % url) 447 contents = self.retry(self.load_json_from_url, args=(url,)) 448 449 # The contents should be something like: 450 # { 451 # "28537": { 452 # "changesets": [ 453 # "1d0a914ae676cc5ed203cdc05c16d8e0c22af7e5", 454 # ], 455 # "date": 1428072488, 456 # "user": "user@mozilla.com" 457 # } 458 # } 459 # 460 # So we grab the first element ("28537" in this case) and then pull 461 # out the 'date' field. 462 pushid = next(contents.keys()) 463 self.info("Pushid is: %s" % pushid) 464 pushdate = contents[pushid]["date"] 465 self.info("Pushdate is: %s" % pushdate) 466 return PushInfo(pushid, pushdate) 467 468 except Exception: 469 self.exception("Failed to get push info from hg.mozilla.org") 470 raise 471 472 473 # __main__ {{{1 474 if __name__ == "__main__": 475 pass