run_tc.py (16917B)
1 #!/usr/bin/env python3 2 # mypy: allow-untyped-defs 3 4 """Wrapper script for running jobs in Taskcluster 5 6 This is intended for running test jobs in Taskcluster. The script 7 takes a two positional arguments which are the name of the test job 8 and the script to actually run. 9 10 The name of the test job is used to determine whether the script should be run 11 for this push (this is in lieu of having a proper decision task). There are 12 several ways that the script can be scheduled to run 13 14 1. The output of wpt test-jobs includes the job name 15 2. The job name is included in a job declaration (see below) 16 3. The string "all" is included in the job declaration 17 4. The job name is set to "all" 18 19 A job declaration is a line appearing in the pull request body (for 20 pull requests) or first commit message (for pushes) of the form: 21 22 tc-jobs: job1,job2,[...] 23 24 In addition, there are a number of keyword arguments used to set options for the 25 environment in which the jobs run. Documentation for these is in the command help. 26 27 As well as running the script, the script sets two environment variables; 28 GITHUB_BRANCH which is the branch that the commits will merge into (if it's a PR) 29 or the branch that the commits are on (if it's a push), and GITHUB_PULL_REQUEST 30 which is the string "false" if the event triggering this job wasn't a pull request 31 or the pull request number if it was. The semantics of these variables are chosen 32 to match the corresponding TRAVIS_* variables. 33 34 Note: for local testing in the Docker image the script ought to still work, but 35 full functionality requires that the TASK_EVENT environment variable is set to 36 the serialization of a GitHub event payload. 37 """ 38 39 import argparse 40 import fnmatch 41 import json 42 import os 43 import subprocess 44 import sys 45 import tarfile 46 import tempfile 47 import zipfile 48 49 sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '..', '..'))) 50 from tools.wpt.utils import get_download_to_descriptor 51 52 root = os.path.abspath( 53 os.path.join(os.path.dirname(__file__), 54 os.pardir, 55 os.pardir)) 56 57 58 def run(cmd, return_stdout=False, **kwargs): 59 print(" ".join(cmd)) 60 if return_stdout: 61 f = subprocess.check_output 62 if "encoding" not in kwargs: 63 kwargs["encoding"] = "utf-8" 64 else: 65 f = subprocess.check_call 66 return f(cmd, **kwargs) 67 68 69 def start(cmd): 70 print(" ".join(cmd)) 71 subprocess.Popen(cmd) 72 73 74 def get_parser(): 75 p = argparse.ArgumentParser() 76 p.add_argument("--oom-killer", 77 action="store_true", 78 help="Run userspace OOM killer") 79 p.add_argument("--hosts", 80 dest="hosts_file", 81 action="store_true", 82 default=True, 83 help="Setup wpt entries in hosts file") 84 p.add_argument("--no-hosts", 85 dest="hosts_file", 86 action="store_false", 87 help="Don't setup wpt entries in hosts file") 88 p.add_argument("--browser", 89 action="append", 90 default=[], 91 help="Browsers that will be used in the job") 92 p.add_argument("--channel", 93 choices=["experimental", "canary", "dev", "nightly", "beta", "stable"], 94 help="Chrome browser channel") 95 p.add_argument("--xvfb", 96 action="store_true", 97 help="Start xvfb") 98 p.add_argument("--install-certificates", action="store_true", default=None, 99 help="Install web-platform.test certificates to UA store") 100 p.add_argument("--no-install-certificates", action="store_false", default=None, 101 help="Don't install web-platform.test certificates to UA store") 102 p.add_argument("--no-setup-repository", action="store_false", dest="setup_repository", 103 help="Don't run any repository setup steps, instead use the existing worktree. " 104 "This is useful for local testing.") 105 p.add_argument("--checkout", 106 help="Revision to checkout before starting job") 107 p.add_argument("--ref", 108 help="Git ref for the commit that should be run") 109 p.add_argument("--head-rev", 110 help="Commit at the head of the branch when the decision task ran") 111 p.add_argument("--merge-rev", 112 help="Provisional merge commit for PR when the decision task ran") 113 p.add_argument("script", 114 help="Script to run for the job") 115 p.add_argument("script_args", 116 nargs=argparse.REMAINDER, 117 help="Additional arguments to pass to the script") 118 return p 119 120 121 def start_userspace_oom_killer(): 122 # Start userspace OOM killer: https://github.com/rfjakob/earlyoom 123 # It will report memory usage every minute and prefer to kill browsers. 124 start(["sudo", "earlyoom", "-p", "-r", "60", "--prefer=(chrome|firefox)", "--avoid=python"]) 125 126 127 def make_hosts_file(): 128 run(["sudo", "sh", "-c", "./wpt make-hosts-file >> /etc/hosts"]) 129 130 131 def checkout_revision(rev): 132 run(["git", "checkout", "--quiet", rev]) 133 134 135 def install_certificates(): 136 run(["sudo", "cp", "tools/certs/cacert.pem", 137 "/usr/local/share/ca-certificates/cacert.crt"]) 138 run(["sudo", "update-ca-certificates"]) 139 140 141 def start_dbus(): 142 # Start system bus 143 run(["sudo", "service", "dbus", "start"]) 144 # Start user bus and set env 145 dbus_env = run(["dbus-launch"], return_stdout=True) 146 for dbus_env_line in dbus_env.splitlines(): 147 dbus_env_name, dbus_env_value = dbus_env_line.split("=", 1) 148 assert (dbus_env_name.startswith("DBUS_SESSION")) 149 os.environ[dbus_env_name] = dbus_env_value 150 assert ("DBUS_SESSION_BUS_ADDRESS" in os.environ) 151 152 153 def install_chrome(channel): 154 if channel == "canary": 155 # Chrome for Testing Canary is installed via --install-browser 156 return 157 if channel in ("experimental", "dev"): 158 deb_archive = "google-chrome-unstable_current_amd64.deb" 159 elif channel == "beta": 160 deb_archive = "google-chrome-beta_current_amd64.deb" 161 elif channel == "stable": 162 deb_archive = "google-chrome-stable_current_amd64.deb" 163 else: 164 raise ValueError("Unrecognized release channel: %s" % channel) 165 166 dest = os.path.join("/tmp", deb_archive) 167 deb_url = "https://dl.google.com/linux/direct/%s" % deb_archive 168 with open(dest, "wb") as f: 169 get_download_to_descriptor(f, deb_url) 170 171 run(["sudo", "apt-get", "-qqy", "update"]) 172 run(["sudo", "gdebi", "-qn", "/tmp/%s" % deb_archive]) 173 174 175 def start_xvfb(): 176 start(["sudo", "Xvfb", os.environ["DISPLAY"], "-screen", "0", 177 "%sx%sx%s" % (os.environ["SCREEN_WIDTH"], 178 os.environ["SCREEN_HEIGHT"], 179 os.environ["SCREEN_DEPTH"])]) 180 start(["sudo", "fluxbox", "-display", os.environ["DISPLAY"]]) 181 182 183 def set_variables(event): 184 # Set some variables that we use to get the commits on the current branch 185 ref_prefix = "refs/heads/" 186 pull_request = "false" 187 branch = None 188 if "pull_request" in event: 189 pull_request = str(event["pull_request"]["number"]) 190 # Note that this is the branch that a PR will merge to, 191 # not the branch name for the PR 192 branch = event["pull_request"]["base"]["ref"] 193 elif "ref" in event: 194 branch = event["ref"] 195 if branch.startswith(ref_prefix): 196 branch = branch[len(ref_prefix):] 197 198 os.environ["GITHUB_PULL_REQUEST"] = pull_request 199 if branch: 200 os.environ["GITHUB_BRANCH"] = branch 201 202 203 def task_url(task_id): 204 root_url = os.environ['TASKCLUSTER_ROOT_URL'] 205 if root_url == 'https://taskcluster.net': 206 queue_base = "https://queue.taskcluster.net/v1/task" 207 else: 208 queue_base = root_url + "/api/queue/v1/task" 209 210 return "%s/%s" % (queue_base, task_id) 211 212 213 def download_artifacts(artifacts): 214 artifact_list_by_task = {} 215 for artifact in artifacts: 216 base_url = task_url(artifact["task"]) 217 if artifact["task"] not in artifact_list_by_task: 218 with tempfile.TemporaryFile() as f: 219 get_download_to_descriptor(f, base_url + "/artifacts") 220 f.seek(0) 221 artifacts_data = json.load(f) 222 artifact_list_by_task[artifact["task"]] = artifacts_data 223 224 artifacts_data = artifact_list_by_task[artifact["task"]] 225 print("DEBUG: Got artifacts %s" % artifacts_data) 226 found = False 227 for candidate in artifacts_data["artifacts"]: 228 print("DEBUG: candidate: %s glob: %s" % (candidate["name"], artifact["glob"])) 229 if fnmatch.fnmatch(candidate["name"], artifact["glob"]): 230 found = True 231 print("INFO: Fetching aritfact %s from task %s" % (candidate["name"], artifact["task"])) 232 file_name = candidate["name"].rsplit("/", 1)[1] 233 url = base_url + "/artifacts/" + candidate["name"] 234 dest_path = os.path.expanduser(os.path.join("~", artifact["dest"], file_name)) 235 dest_dir = os.path.dirname(dest_path) 236 if not os.path.exists(dest_dir): 237 os.makedirs(dest_dir) 238 with open(dest_path, "wb") as f: 239 get_download_to_descriptor(f, url) 240 241 if artifact.get("extract"): 242 unpack(dest_path) 243 if not found: 244 print("WARNING: No artifact found matching %s in task %s" % (artifact["glob"], artifact["task"])) 245 246 247 def unpack(path): 248 dest = os.path.dirname(path) 249 if tarfile.is_tarfile(path): 250 run(["tar", "-xf", path], cwd=os.path.dirname(path)) 251 elif zipfile.is_zipfile(path): 252 with zipfile.ZipFile(path) as archive: 253 archive.extractall(dest) 254 else: 255 print("ERROR: Don't know how to extract %s" % path) 256 raise Exception 257 258 259 def setup_environment(args): 260 if "TASK_ARTIFACTS" in os.environ: 261 artifacts = json.loads(os.environ["TASK_ARTIFACTS"]) 262 download_artifacts(artifacts) 263 264 if args.hosts_file: 265 make_hosts_file() 266 267 if args.install_certificates: 268 install_certificates() 269 270 if "chrome" in args.browser: 271 assert args.channel is not None 272 install_chrome(args.channel) 273 274 # These browsers use dbus for various features. 275 if any(b in args.browser for b in ["chrome", "webkitgtk_minibrowser", "wpewebkit_minibrowser"]): 276 start_dbus() 277 278 if args.xvfb: 279 start_xvfb() 280 281 if args.oom_killer: 282 start_userspace_oom_killer() 283 284 285 def setup_repository(args): 286 is_pr = os.environ.get("GITHUB_PULL_REQUEST", "false") != "false" 287 288 # Initially task_head points at the same commit as the ref we want to test. 289 # However that may not be the same commit as we actually want to test if 290 # the branch changed since the decision task ran. The branch may have 291 # changed because someone has pushed more commits (either to the PR 292 # or later commits to the branch), or because someone has pushed to the 293 # base branch for the PR. 294 # 295 # In that case we take a different approach depending on whether this is a 296 # PR or a push to a branch. 297 # If this is a push to a branch, and the original commit is still fetchable, 298 # we try to fetch that (it may not be in the case of e.g. a force push). 299 # If it's not fetchable then we fail the run. 300 # For a PR we are testing the provisional merge commit. If that's changed it 301 # could be that the PR branch was updated or the base branch was updated. In the 302 # former case we fail the run because testing an old commit is a waste of 303 # resources. In the latter case we assume it's OK to use the current merge 304 # instead of the one at the time the decision task ran. 305 306 if args.ref: 307 if is_pr: 308 assert args.ref.endswith("/merge") 309 expected_head = args.merge_rev 310 else: 311 expected_head = args.head_rev 312 313 task_head = run(["git", "rev-parse", "task_head"], return_stdout=True).strip() 314 315 if task_head != expected_head: 316 if not is_pr: 317 try: 318 run(["git", "fetch", "origin", expected_head]) 319 run(["git", "reset", "--hard", expected_head]) 320 except subprocess.CalledProcessError: 321 print("CRITICAL: task_head points at %s, expected %s and " 322 "unable to fetch expected commit.\n" 323 "This may be because the branch was updated" % (task_head, expected_head)) 324 sys.exit(1) 325 else: 326 # Convert the refs/pulls/<id>/merge to refs/pulls/<id>/head 327 head_ref = args.ref.rsplit("/", 1)[0] + "/head" 328 try: 329 remote_head = run(["git", "ls-remote", "origin", head_ref], 330 return_stdout=True).split("\t")[0] 331 except subprocess.CalledProcessError: 332 print("CRITICAL: Failed to read remote ref %s" % head_ref) 333 sys.exit(1) 334 if remote_head != args.head_rev: 335 print("CRITICAL: task_head points at %s, expected %s. " 336 "This may be because the branch was updated" % (task_head, expected_head)) 337 sys.exit(1) 338 print("INFO: Merge commit changed from %s to %s due to base branch changes. " 339 "Running task anyway." % (expected_head, task_head)) 340 341 if os.environ.get("GITHUB_PULL_REQUEST", "false") != "false": 342 parents = run(["git", "rev-parse", "task_head^@"], 343 return_stdout=True).strip().split() 344 if len(parents) == 2: 345 base_head = parents[0] 346 pr_head = parents[1] 347 348 run(["git", "branch", "base_head", base_head]) 349 run(["git", "branch", "pr_head", pr_head]) 350 else: 351 print("ERROR: Pull request HEAD wasn't a 2-parent merge commit; " 352 "expected to test the merge of PR into the base") 353 commit = run(["git", "rev-parse", "task_head"], 354 return_stdout=True).strip() 355 print("HEAD: %s" % commit) 356 print("Parents: %s" % ", ".join(parents)) 357 sys.exit(1) 358 359 branch = os.environ.get("GITHUB_BRANCH") 360 if branch: 361 # Ensure that the remote base branch exists 362 # TODO: move this somewhere earlier in the task 363 run(["git", "fetch", "--quiet", "origin", "%s:%s" % (branch, branch)]) 364 365 checkout_rev = args.checkout if args.checkout is not None else "task_head" 366 checkout_revision(checkout_rev) 367 368 refs = run(["git", "for-each-ref", "refs/heads"], return_stdout=True) 369 print("INFO: git refs:\n%s" % refs) 370 print("INFO: checked out commit:\n%s" % run(["git", "rev-parse", "HEAD"], 371 return_stdout=True)) 372 373 374 def fetch_event_data(): 375 try: 376 task_id = os.environ["TASK_ID"] 377 except KeyError: 378 print("WARNING: Missing TASK_ID environment variable") 379 # For example under local testing 380 return None 381 382 with tempfile.TemporaryFile() as f: 383 get_download_to_descriptor(f, task_url(task_id)) 384 f.seek(0) 385 task_data = json.load(f) 386 event_data = task_data.get("extra", {}).get("github_event") 387 if event_data is not None: 388 return json.loads(event_data) 389 390 391 def include_job(job): 392 # Only for supporting pre decision-task PRs 393 # Special case things that unconditionally run on pushes, 394 # assuming a higher layer is filtering the required list of branches 395 if "GITHUB_PULL_REQUEST" not in os.environ: 396 return True 397 398 if (os.environ["GITHUB_PULL_REQUEST"] == "false" and 399 job == "run-all"): 400 return True 401 402 jobs_str = run([os.path.join(root, "wpt"), 403 "test-jobs"], return_stdout=True) 404 print(jobs_str) 405 return job in set(jobs_str.splitlines()) 406 407 408 def main(): 409 args = get_parser().parse_args() 410 411 if "TASK_EVENT" in os.environ: 412 event = json.loads(os.environ["TASK_EVENT"]) 413 else: 414 event = fetch_event_data() 415 416 if event: 417 set_variables(event) 418 419 if args.setup_repository: 420 setup_repository(args) 421 422 # Hack for backwards compatibility 423 if args.script in ["run-all", "lint", "update_built", "tools_unittest", 424 "wpt_integration", "resources_unittest", 425 "wptrunner_infrastructure", "stability", "affected_tests"]: 426 job = args.script 427 if not include_job(job): 428 return 429 args.script = args.script_args[0] 430 args.script_args = args.script_args[1:] 431 432 # Run the job 433 setup_environment(args) 434 os.chdir(root) 435 cmd = [args.script] + args.script_args 436 print(" ".join(cmd)) 437 sys.exit(subprocess.call(cmd)) 438 439 440 if __name__ == "__main__": 441 main() # type: ignore