fetch-chromium.py (6727B)
1 #!/usr/bin/python3 -u 2 3 # This Source Code Form is subject to the terms of the Mozilla Public 4 # License, v. 2.0. If a copy of the MPL was not distributed with this 5 # file, You can obtain one at http://mozilla.org/MPL/2.0/. 6 7 """ 8 This script downloads the latest chromium build (or a manually 9 defined version) for a given platform. It then uploads the build, 10 with the revision of the build stored in a REVISION file. 11 """ 12 13 import argparse 14 import errno 15 import os 16 import shutil 17 import subprocess 18 import tempfile 19 20 import requests 21 from redo import retriable 22 23 LAST_CHANGE_URL = ( 24 # formatted with platform 25 "https://www.googleapis.com/download/storage/v1/b/" 26 "chromium-browser-snapshots/o/{}%2FLAST_CHANGE?alt=media" 27 ) 28 29 CHROMIUM_BASE_URL = ( 30 # formatted with (platform/revision/archive) 31 "https://www.googleapis.com/download/storage/v1/b/" 32 "chromium-browser-snapshots/o/{}%2F{}%2F{}?alt=media" 33 ) 34 35 36 CHROMIUM_INFO = { 37 "linux": { 38 "platform": "Linux_x64", 39 "chromium": "chrome-linux.zip", 40 "dir": "chrome-linux", 41 "result": "chromium-linux.tar.bz2", 42 "chromedriver": "chromedriver_linux64.zip", 43 }, 44 "win32": { 45 "platform": "Win", 46 "chromium": "chrome-win.zip", 47 "dir": "chrome-win", 48 "result": "chromium-win32.tar.bz2", 49 "chromedriver": "chromedriver_win32.zip", 50 }, 51 "win64": { 52 "platform": "Win", 53 "chromium": "chrome-win.zip", 54 "dir": "chrome-win", 55 "result": "chromium-win64.tar.bz2", 56 "chromedriver": "chromedriver_win32.zip", 57 }, 58 "mac": { 59 "platform": "Mac", 60 "chromium": "chrome-mac.zip", 61 "dir": "chrome-mac", 62 "result": "chromium-mac.tar.bz2", 63 "chromedriver": "chromedriver_mac64.zip", 64 }, 65 "mac-arm": { 66 "platform": "Mac_Arm", 67 "chromium": "chrome-mac.zip", 68 "dir": "chrome-mac", 69 "result": "chromium-mac-arm.tar.bz2", 70 "chromedriver": "chromedriver_mac64.zip", 71 }, 72 } 73 74 75 def log(msg): 76 print("build-chromium: %s" % msg) 77 78 79 @retriable(attempts=7, sleeptime=5, sleepscale=2) 80 def fetch_file(url, filepath): 81 """Download a file from the given url to a given file.""" 82 size = 4096 83 r = requests.get(url, stream=True) 84 r.raise_for_status() 85 86 with open(filepath, "wb") as fd: 87 for chunk in r.iter_content(size): 88 fd.write(chunk) 89 90 91 def unzip(zippath, target): 92 """Unzips an archive to the target location.""" 93 log("Unpacking archive at: %s to: %s" % (zippath, target)) 94 unzip_command = ["unzip", "-q", "-o", zippath, "-d", target] 95 subprocess.check_call(unzip_command) 96 97 98 @retriable(attempts=7, sleeptime=5, sleepscale=2) 99 def fetch_chromium_revision(platform): 100 """Get the revision of the latest chromium build.""" 101 chromium_platform = CHROMIUM_INFO[platform]["platform"] 102 revision_url = LAST_CHANGE_URL.format(chromium_platform) 103 104 log("Getting revision number for latest %s chromium build..." % chromium_platform) 105 106 # Expecting a file with a single number indicating the latest 107 # chromium build with a chromedriver that we can download 108 r = requests.get(revision_url, timeout=30) 109 r.raise_for_status() 110 111 chromium_revision = r.content.decode("utf-8") 112 return chromium_revision.strip() 113 114 115 def fetch_chromedriver(platform, revision, chromium_dir): 116 """Get the chromedriver for the given revision and repackage it.""" 117 if not revision: 118 revision = fetch_chromium_revision(platform) 119 120 download_url = CHROMIUM_BASE_URL.format( 121 CHROMIUM_INFO[platform]["platform"], 122 revision, 123 CHROMIUM_INFO[platform]["chromedriver"], 124 ) 125 126 tmpzip = os.path.join(tempfile.mkdtemp(), "cd-tmp.zip") 127 log("Downloading chromedriver from %s" % download_url) 128 fetch_file(download_url, tmpzip) 129 130 tmppath = tempfile.mkdtemp() 131 unzip(tmpzip, tmppath) 132 133 # Find the chromedriver then copy it to the chromium directory 134 cd_path = None 135 for dirpath, _, filenames in os.walk(tmppath): 136 for filename in filenames: 137 if filename in {"chromedriver", "chromedriver.exe"}: 138 cd_path = os.path.join(dirpath, filename) 139 break 140 if cd_path is not None: 141 break 142 if cd_path is None: 143 raise Exception("Could not find chromedriver binary in %s" % tmppath) 144 log("Copying chromedriver from: %s to: %s" % (cd_path, chromium_dir)) 145 shutil.copy(cd_path, chromium_dir) 146 return revision 147 148 149 def build_chromium_archive(platform, revision=None): 150 """ 151 Download and store a chromium build for a given platform. 152 153 Retrieves either the latest version, or uses a pre-defined version if 154 the `--revision` option is given a revision. 155 """ 156 upload_dir = os.environ.get("UPLOAD_DIR") 157 if upload_dir: 158 # Create the upload directory if it doesn't exist. 159 try: 160 log("Creating upload directory in %s..." % os.path.abspath(upload_dir)) 161 os.makedirs(upload_dir) 162 except OSError as e: 163 if e.errno != errno.EEXIST: 164 raise 165 166 # Make a temporary location for the file 167 tmppath = tempfile.mkdtemp() 168 169 # Create the directory format expected for browsertime setup in taskgraph transform 170 artifact_dir = CHROMIUM_INFO[platform]["dir"] 171 chromium_dir = os.path.join(tmppath, artifact_dir) 172 os.mkdir(chromium_dir) 173 174 # Store the revision number and chromedriver 175 revision = fetch_chromedriver(platform, revision, chromium_dir) 176 revision_file = os.path.join(chromium_dir, ".REVISION") 177 with open(revision_file, "w+") as f: 178 f.write(str(revision)) 179 180 tar_file = CHROMIUM_INFO[platform]["result"] 181 tar_command = ["tar", "cjf", tar_file, "-C", tmppath, artifact_dir] 182 log("Added revision to %s file." % revision_file) 183 184 log("Tarring with the command: %s" % str(tar_command)) 185 subprocess.check_call(tar_command) 186 187 upload_dir = os.environ.get("UPLOAD_DIR") 188 if upload_dir: 189 # Move the tarball to the output directory for upload. 190 log("Moving %s to the upload directory..." % tar_file) 191 shutil.copy(tar_file, os.path.join(upload_dir, tar_file)) 192 193 shutil.rmtree(tmppath) 194 195 196 def parse_args(): 197 """Read command line arguments and return options.""" 198 parser = argparse.ArgumentParser() 199 parser.add_argument( 200 "--platform", help="Platform version of chromium to build.", required=True 201 ) 202 parser.add_argument( 203 "--revision", 204 help="Revision of chromium to build to get. " 205 "(Defaults to the newest chromium build).", 206 default=None, 207 ) 208 209 return parser.parse_args() 210 211 212 if __name__ == "__main__": 213 args = vars(parse_args()) 214 build_chromium_archive(**args)