fetch-cft-chromedriver.py (9471B)
1 #!/usr/bin/python3 -u 2 3 # This Source Code Form is subject to the terms of the Mozilla Public 4 # License, v. 2.0. If a copy of the MPL was not distributed with this 5 # file, You can obtain one at http://mozilla.org/MPL/2.0/. 6 7 """ 8 This script downloads chromedriver for a given platform and then 9 packages the driver along with the revision and uploads the archive. 10 This is currently accomplished by using "last known good version" of 11 the chromedrivers associated with Chrome for Testing. The `Canary` 12 channel is specified as it is required for the Chromium-as-Release 13 performance tests. 14 """ 15 16 import argparse 17 import errno 18 import os 19 import shutil 20 import subprocess 21 import tempfile 22 23 import requests 24 from redo import retriable 25 26 CHROME_FOR_TESTING_INFO = { 27 "linux": { 28 "platform": "linux64", 29 "dir": "cft-chromedriver-linux", 30 "result": "cft-cd-linux.tar.bz2", 31 "result_backup": "cft-cd-linux-backup.tar.bz2", 32 "chromedriver": "chromedriver_linux64.zip", 33 }, 34 "win64": { 35 "platform": "win64", 36 "dir": "cft-chromedriver-win64", 37 "result": "cft-cd-win64.tar.bz2", 38 "result_backup": "cft-cd-win64-backup.tar.bz2", 39 "chromedriver": "chromedriver_win32.zip", 40 }, 41 "mac": { 42 "platform": "mac-x64", 43 "dir": "cft-chromedriver-mac", 44 "result": "cft-cd-mac.tar.bz2", 45 "result_backup": "cft-cd-mac-backup.tar.bz2", 46 "chromedriver": "chromedriver_mac64.zip", 47 }, 48 "mac-arm": { 49 "platform": "mac-arm64", 50 "dir": "cft-chromedriver-mac", 51 "result": "cft-cd-mac-arm.tar.bz2", 52 "result_backup": "cft-cd-mac-arm-backup.tar.bz2", 53 "chromedriver": "chromedriver_mac64.zip", 54 }, 55 } 56 57 LAST_GOOD_CFT_JSON = ( 58 "https://googlechromelabs.github.io/chrome-for-testing/" 59 "last-known-good-versions-with-downloads.json" 60 ) 61 62 MILESTONE_CFT_JSON = ( 63 "https://googlechromelabs.github.io/chrome-for-testing/" 64 "latest-versions-per-milestone-with-downloads.json" 65 ) 66 67 68 def log(msg): 69 print("build-cft-chromedriver: %s" % msg) 70 71 72 @retriable(attempts=7, sleeptime=5, sleepscale=2) 73 def fetch_file(url, filepath): 74 """Download a file from the given url to a given file.""" 75 size = 4096 76 r = requests.get(url, stream=True) 77 r.raise_for_status() 78 79 with open(filepath, "wb") as fd: 80 for chunk in r.iter_content(size): 81 fd.write(chunk) 82 83 84 def unzip(zippath, target): 85 """Unzips an archive to the target location.""" 86 log("Unpacking archive at: %s to: %s" % (zippath, target)) 87 unzip_command = ["unzip", "-q", "-o", zippath, "-d", target] 88 subprocess.check_call(unzip_command) 89 90 91 def get_cft_metadata(endpoint=LAST_GOOD_CFT_JSON): 92 """Send a request to the Chrome for Testing's last 93 good json URL (default) and get the json payload which will have 94 the download URLs that we need. 95 """ 96 res = requests.get(endpoint) 97 data = res.json() 98 99 return data 100 101 102 def get_cd_url(data, cft_platform, channel): 103 """Given the json data, get the download URL's for 104 the correct platform 105 """ 106 for p in data["channels"][channel]["downloads"]["chromedriver"]: 107 if p["platform"] == cft_platform: 108 return p["url"] 109 raise Exception("Platform not found") 110 111 112 def get_chromedriver_revision(data, channel): 113 """Grab revision metadata from payload""" 114 return data["channels"][channel]["revision"] 115 116 117 def fetch_chromedriver(download_url, cft_dir): 118 """Get the chromedriver for the given cft url repackage it.""" 119 120 tmpzip = os.path.join(tempfile.mkdtemp(), "cd-tmp.zip") 121 log("Downloading chromedriver from %s" % download_url) 122 fetch_file(download_url, tmpzip) 123 124 tmppath = tempfile.mkdtemp() 125 unzip(tmpzip, tmppath) 126 127 # Find the chromedriver then copy it to the chromium directory 128 cd_path = None 129 for dirpath, _, filenames in os.walk(tmppath): 130 for filename in filenames: 131 if filename in {"chromedriver", "chromedriver.exe"}: 132 cd_path = os.path.join(dirpath, filename) 133 break 134 if cd_path is not None: 135 break 136 if cd_path is None: 137 raise Exception("Could not find chromedriver binary in %s" % tmppath) 138 log("Copying chromedriver from: %s to: %s" % (cd_path, cft_dir)) 139 shutil.copy(cd_path, cft_dir) 140 141 142 def get_backup_chromedriver(version, cft_data, cft_platform): 143 """Download a backup chromedriver for the transitionary period of machine auto updates. 144 145 If no version is specified, by default grab the N-1 version of the latest Stable channel 146 chromedriver. 147 148 """ 149 log("Grabbing a backup chromedriver...") 150 if not version: 151 log("No version specified") 152 # Get latest stable version and subtract 1 153 current_stable_version = cft_data["channels"]["Stable"]["version"].split(".")[0] 154 version = str(int(current_stable_version) - 1) 155 log("Fetching major version %s" % version) 156 157 milestone_metadata = get_cft_metadata(MILESTONE_CFT_JSON) 158 backup_revision = milestone_metadata["milestones"][version]["revision"] 159 backup_version = milestone_metadata["milestones"][version]["version"].split(".")[0] 160 161 backup_url = None 162 for p in milestone_metadata["milestones"][version]["downloads"]["chromedriver"]: 163 if p["platform"] == cft_platform: 164 backup_url = p["url"] 165 166 log("Found backup chromedriver") 167 168 if not backup_url: 169 raise Exception("Platform not found") 170 171 return backup_url, backup_revision, backup_version 172 173 174 def get_version_from_json(data, channel): 175 return data["channels"][channel]["version"].split(".")[0] 176 177 178 def insert_channel_in_archive_name(filename, channel): 179 parts = filename.rsplit(".", maxsplit=2) 180 if len(parts) != 3: 181 raise ValueError( 182 f"Unexpected filename format: '{filename}'. Expected a file with two extensions (e.g., '.tar.bz2')." 183 ) 184 return f"{parts[0]}-{channel.lower()}.{parts[1]}.{parts[2]}" 185 186 187 def build_cft_archive(platform, channel, backup, version): 188 """Download and store a chromedriver for a given platform.""" 189 upload_dir = os.environ.get("UPLOAD_DIR") 190 if upload_dir: 191 # Create the upload directory if it doesn't exist. 192 try: 193 log("Creating upload directory in %s..." % os.path.abspath(upload_dir)) 194 os.makedirs(upload_dir) 195 except OSError as e: 196 if e.errno != errno.EEXIST: 197 raise 198 199 cft_platform = CHROME_FOR_TESTING_INFO[platform]["platform"] 200 201 data = get_cft_metadata() 202 if backup: 203 cft_chromedriver_url, revision, payload_version = get_backup_chromedriver( 204 version, data, cft_platform 205 ) 206 tar_file = CHROME_FOR_TESTING_INFO[platform]["result_backup"] 207 else: 208 cft_chromedriver_url = get_cd_url(data, cft_platform, channel) 209 revision = get_chromedriver_revision(data, channel) 210 payload_version = get_version_from_json(data, channel) 211 # For clarity, include channel in artifact name. 212 tar_file = insert_channel_in_archive_name( 213 CHROME_FOR_TESTING_INFO[platform]["result"], channel 214 ) 215 # Make a temporary location for the file 216 tmppath = tempfile.mkdtemp() 217 218 # Create the directory format expected for browsertime setup in taskgraph transform 219 artifact_dir = CHROME_FOR_TESTING_INFO[platform]["dir"] 220 if backup or channel in ("Stable", "Beta"): 221 # need to prepend the major version to the artifact dir due to how raptor browsertime 222 # ensures the correct version is used with chrome stable. 223 artifact_dir = payload_version + artifact_dir 224 cft_dir = os.path.join(tmppath, artifact_dir) 225 os.mkdir(cft_dir) 226 227 # Store the revision number and chromedriver 228 fetch_chromedriver(cft_chromedriver_url, cft_dir) 229 revision_file = os.path.join(cft_dir, ".REVISION") 230 with open(revision_file, "w+") as f: 231 f.write(str(revision)) 232 233 tar_command = ["tar", "cjf", tar_file, "-C", tmppath, artifact_dir] 234 log("Revision is %s" % revision) 235 log("Added revision to %s file." % revision_file) 236 237 log("Tarring with the command: %s" % str(tar_command)) 238 subprocess.check_call(tar_command) 239 240 upload_dir = os.environ.get("UPLOAD_DIR") 241 if upload_dir: 242 # Move the tarball to the output directory for upload. 243 log("Moving %s to the upload directory..." % tar_file) 244 shutil.copy(tar_file, os.path.join(upload_dir, tar_file)) 245 246 shutil.rmtree(tmppath) 247 248 249 def parse_args(): 250 """Read command line arguments and return options.""" 251 parser = argparse.ArgumentParser() 252 parser.add_argument( 253 "--platform", 254 help="Corresponding platform of CfT chromedriver to fetch.", 255 required=True, 256 ) 257 # Bug 1869592 - Add optional flag to provide CfT channel e.g. Canary, Stable, etc. 258 parser.add_argument( 259 "--channel", 260 help="Corresponding channel of CfT chromedriver to fetch.", 261 required=False, 262 default="Canary", 263 ) 264 parser.add_argument( 265 "--backup", 266 help="Determine if we are grabbing a backup chromedriver version.", 267 required=False, 268 default=False, 269 action="store_true", 270 ) 271 parser.add_argument( 272 "--version", 273 help="Pin the revision if necessary for current platform", 274 required=False, 275 default="", 276 ) 277 return parser.parse_args() 278 279 280 if __name__ == "__main__": 281 args = vars(parse_args()) 282 build_cft_archive(**args)