manifest_build.py (6677B)
1 # mypy: allow-untyped-defs 2 3 import json 4 import logging 5 import os 6 import subprocess 7 import sys 8 import tempfile 9 10 import requests 11 12 from pathlib import Path 13 14 15 here = os.path.abspath(os.path.dirname(__file__)) 16 wpt_root = os.path.abspath(os.path.join(here, os.pardir, os.pardir)) 17 18 if wpt_root not in sys.path: 19 sys.path.append(wpt_root) 20 21 from tools.wpt.testfiles import get_git_cmd 22 23 logging.basicConfig(level=logging.INFO) 24 logger = logging.getLogger(__name__) 25 26 27 class Status: 28 SUCCESS = 0 29 FAIL = 1 30 31 32 def run(cmd, return_stdout=False, **kwargs): 33 logger.info(" ".join(cmd)) 34 if return_stdout: 35 f = subprocess.check_output 36 else: 37 f = subprocess.check_call 38 return f(cmd, **kwargs) 39 40 41 def create_manifest(path): 42 run(["./wpt", "manifest", "-p", path]) 43 44 45 def create_web_feature_manifest(path): 46 run(["./wpt", "web-features-manifest", "-p", path]) 47 48 49 def compress_manifest(path): 50 for args in [["gzip", "-k", "-f", "--best"], 51 ["bzip2", "-k", "-f", "--best"], 52 ["zstd", "-k", "-f", "--ultra", "-22", "-q"]]: 53 run(args + [path]) 54 55 56 def request(url, desc, method=None, data=None, json_data=None, params=None, headers=None): 57 github_token = os.environ.get("GITHUB_TOKEN") 58 default_headers = { 59 "Authorization": "token %s" % github_token, 60 "Accept": "application/vnd.github.machine-man-preview+json" 61 } 62 63 _headers = default_headers 64 if headers is not None: 65 _headers.update(headers) 66 67 kwargs = {"params": params, 68 "headers": _headers} 69 try: 70 logger.info("Requesting URL %s" % url) 71 if json_data is not None or data is not None: 72 if method is None: 73 method = requests.post 74 kwargs["json"] = json_data 75 kwargs["data"] = data 76 elif method is None: 77 method = requests.get 78 79 resp = method(url, **kwargs) 80 81 except Exception as e: 82 logger.error(f"{desc} failed:\n{e}") 83 return None 84 85 try: 86 resp.raise_for_status() 87 except requests.HTTPError: 88 logger.error("%s failed: Got HTTP status %s. Response:" % 89 (desc, resp.status_code)) 90 logger.error(resp.text) 91 return None 92 93 try: 94 return resp.json() 95 except ValueError: 96 logger.error("%s failed: Returned data was not JSON Response:" % desc) 97 logger.error(resp.text) 98 99 100 def get_pr(owner, repo, sha): 101 data = request("https://api.github.com/search/issues?q=type:pr+is:merged+repo:%s/%s+sha:%s" % 102 (owner, repo, sha), "Getting PR") 103 if data is None: 104 return None 105 106 items = data["items"] 107 if len(items) == 0: 108 logger.error("No PR found for %s" % sha) 109 return None 110 if len(items) > 1: 111 logger.warning("Found multiple PRs for %s" % sha) 112 113 pr = items[0] 114 115 return pr["number"] 116 117 118 def get_file_upload_details(manifest_path, sha): 119 """ 120 For a given file, generate details used to upload to GitHub. 121 """ 122 path = Path(manifest_path) 123 stem = path.stem 124 extension = path.suffix 125 upload_filename_prefix = f"{stem}-{sha}{extension}" 126 upload_label_prefix = path.name 127 upload_desc = f"{stem.title()} upload" 128 return upload_filename_prefix, upload_label_prefix, upload_desc 129 130 131 def create_release(manifest_file_paths, owner, repo, sha, tag, body): 132 logger.info(f"Creating a release for tag='{tag}', target_commitish='{sha}'") 133 create_url = f"https://api.github.com/repos/{owner}/{repo}/releases" 134 create_data = {"tag_name": tag, 135 "target_commitish": sha, 136 "name": tag, 137 "body": body, 138 "draft": True} 139 create_resp = request(create_url, "Release creation", json_data=create_data) 140 if not create_resp: 141 return False 142 143 # Upload URL contains '{?name,label}' at the end which we want to remove 144 upload_url = create_resp["upload_url"].split("{", 1)[0] 145 146 upload_exts = [".gz", ".bz2", ".zst"] 147 for manifest_path in manifest_file_paths: 148 upload_filename_prefix, upload_label_prefix, upload_desc = get_file_upload_details(manifest_path, sha) 149 for upload_ext in upload_exts: 150 upload_filename = f"{upload_filename_prefix}{upload_ext}" 151 params = {"name": upload_filename, 152 "label": f"{upload_label_prefix}{upload_ext}"} 153 154 with open(f"{manifest_path}{upload_ext}", "rb") as f: 155 upload_data = f.read() 156 157 logger.info("Uploading %s bytes" % len(upload_data)) 158 159 upload_resp = request(upload_url, upload_desc, data=upload_data, params=params, 160 headers={'Content-Type': 'application/octet-stream'}) 161 if not upload_resp: 162 return False 163 164 release_id = create_resp["id"] 165 edit_url = f"https://api.github.com/repos/{owner}/{repo}/releases/{release_id}" 166 edit_data = create_data.copy() 167 edit_data["draft"] = False 168 edit_resp = request(edit_url, "Release publishing", method=requests.patch, json_data=edit_data) 169 if not edit_resp: 170 return False 171 172 logger.info("Released %s" % edit_resp["html_url"]) 173 return True 174 175 176 def should_dry_run(): 177 with open(os.environ["GITHUB_EVENT_PATH"]) as f: 178 event = json.load(f) 179 logger.info(json.dumps(event, indent=2)) 180 181 if "pull_request" in event: 182 logger.info("Dry run for PR") 183 return True 184 if event.get("ref") != "refs/heads/master": 185 logger.info("Dry run for ref %s" % event.get("ref")) 186 return True 187 return False 188 189 190 def main(): 191 dry_run = should_dry_run() 192 193 manifest_path = os.path.join(tempfile.mkdtemp(), "MANIFEST.json") 194 web_features_manifest_path = os.path.join(tempfile.mkdtemp(), "WEB_FEATURES_MANIFEST.json") 195 196 create_manifest(manifest_path) 197 create_web_feature_manifest(web_features_manifest_path) 198 199 compress_manifest(manifest_path) 200 compress_manifest(web_features_manifest_path) 201 202 owner, repo = os.environ["GITHUB_REPOSITORY"].split("/", 1) 203 204 git = get_git_cmd(wpt_root) 205 head_rev = git("rev-parse", "HEAD").strip() 206 body = git("show", "--no-patch", "--format=%B", "HEAD") 207 208 if dry_run: 209 return Status.SUCCESS 210 211 pr = get_pr(owner, repo, head_rev) 212 if pr is None: 213 return Status.FAIL 214 tag_name = "merge_pr_%s" % pr 215 216 manifest_paths = [manifest_path, web_features_manifest_path] 217 if not create_release(manifest_paths, owner, repo, head_rev, tag_name, body): 218 return Status.FAIL 219 220 return Status.SUCCESS 221 222 223 if __name__ == "__main__": 224 code = main() # type: ignore 225 assert isinstance(code, int) 226 sys.exit(code)