run.py (6674B)
1 #!/usr/bin/env python3 2 # This Source Code Form is subject to the terms of the Mozilla Public 3 # License, v. 2.0. If a copy of the MPL was not distributed with this 4 # file, # You can obtain one at http://mozilla.org/MPL/2.0/. 5 6 import sys 7 8 sys.path.append("/builds/worker/checkouts/gecko/third_party/python") 9 sys.path.append(".") 10 11 import base64 12 import os 13 import platform 14 import signal 15 import stat 16 import subprocess 17 18 import requests 19 20 import taskcluster 21 22 # Bump this number when you need to cause a commit for the job to re-run: 22 23 24 if len(sys.argv) < 3: 25 print("Usage:", sys.argv[0], "gecko-dev-path updatebot-path [moz-fetches-dir]") 26 sys.exit(1) 27 28 GECKO_DEV_PATH = sys.argv[1].replace("/", os.path.sep) 29 UPDATEBOT_PATH = sys.argv[2].replace("/", os.path.sep) 30 31 # Only needed on Windows 32 if len(sys.argv) > 3: 33 FETCHES_PATH = sys.argv[3].replace("/", os.path.sep) 34 else: 35 FETCHES_PATH = None 36 37 HOME_PATH = os.path.expanduser("~") 38 39 OPERATING_MODE = ( 40 "prod" 41 if os.environ.get("GECKO_HEAD_REPOSITORY", "") 42 == "https://hg.mozilla.org/mozilla-central" 43 else "dev" 44 ) 45 46 DEV_PHAB_URL = "https://phabricator-dev.allizom.org/" 47 PROD_PHAB_URL = "https://phabricator.services.mozilla.com/" 48 49 phabricator_url = DEV_PHAB_URL if OPERATING_MODE == "dev" else PROD_PHAB_URL 50 51 52 def log(*args): 53 print(*args) 54 55 56 def get_secret(name): 57 secret = None 58 if "TASK_ID" in os.environ: 59 secrets_url = ( 60 os.environ.get("TASKCLUSTER_PROXY_URL", "http://taskcluster") 61 + "/secrets/v1/secret/project/updatebot/" 62 + ("3" if OPERATING_MODE == "prod" else "2") 63 + "/" 64 + name 65 ) 66 res = requests.get(secrets_url) 67 res.raise_for_status() 68 secret = res.json() 69 else: 70 secrets = taskcluster.Secrets(taskcluster.optionsFromEnvironment()) 71 secret = secrets.get("project/updatebot/" + OPERATING_MODE + "/" + name) 72 secret = secret["secret"] if "secret" in secret else None 73 secret = secret["value"] if "value" in secret else None 74 return secret 75 76 77 # Get TC Secrets ======================================= 78 log("Operating mode is ", OPERATING_MODE) 79 log("Getting secrets...") 80 bugzilla_api_key = get_secret("bugzilla-api-key") 81 phabricator_token = get_secret("phabricator-token") 82 try_sshkey = get_secret("try-sshkey") 83 database_config = get_secret("database-password") 84 sentry_url = get_secret("sentry-url") 85 sql_proxy_config = get_secret("sql-proxy-config") 86 87 # Update Updatebot ======================================= 88 if OPERATING_MODE == "dev": 89 """ 90 If we are in development mode, we will update from github. 91 (This command will probably only work if we checkout a branch FWIW.) 92 93 This allows us to iterate faster by committing to github and 94 re-running the cron job on Taskcluster, without rebuilding the 95 Docker image. 96 97 However, this mechanism is bypassing the security feature we 98 have in-tree, where upstream out-of-tree code is fixed at a known 99 revision and cannot be changed without a commit to m-c. 100 101 Therefore, we only do this in dev mode when running on try. 102 """ 103 104 os.chdir(UPDATEBOT_PATH) 105 log("Performing git repo update...") 106 command = ["git", "symbolic-ref", "-q", "HEAD"] 107 108 r = subprocess.run(command, check=False) 109 if r.returncode == 0: 110 # This indicates we are on a branch, and not a specific revision 111 subprocess.check_call(["git", "pull", "origin"]) 112 113 # Set Up SSH & Phabricator ============================== 114 os.chdir(HOME_PATH) 115 log("Setting up ssh and phab keys...") 116 with open("id_rsa", "w") as sshkey: 117 sshkey.write(try_sshkey) 118 os.chmod("id_rsa", stat.S_IRUSR | stat.S_IWUSR) 119 120 arc_filename = ".arcrc" 121 if platform.system() == "Windows": 122 arc_path = os.path.join(FETCHES_PATH, "..", "AppData", "Roaming") 123 os.makedirs(arc_path, exist_ok=True) 124 os.chdir(arc_path) 125 log("Writing %s to %s" % (arc_filename, arc_path)) 126 else: 127 os.chdir(HOME_PATH) 128 129 arcrc = open(arc_filename, "w") 130 towrite = """ 131 { 132 "hosts": { 133 "PHAB_URL_HERE": { 134 "token": "TOKENHERE" 135 } 136 } 137 } 138 """.replace("TOKENHERE", phabricator_token).replace( 139 "PHAB_URL_HERE", phabricator_url + "api/" 140 ) 141 arcrc.write(towrite) 142 arcrc.close() 143 os.chmod(arc_filename, stat.S_IRUSR | stat.S_IWUSR) 144 145 # Set up the Cloud SQL Proxy ============================= 146 os.chdir(HOME_PATH) 147 log("Setting up cloud_sql_proxy...") 148 with open("sql-proxy-key", "w") as proxy_key_file: 149 proxy_key_file.write( 150 base64.b64decode(sql_proxy_config["key-value"]).decode("utf-8") 151 ) 152 153 instance_name = sql_proxy_config["instance-name"] 154 if platform.system() == "Linux": 155 sql_proxy_command = "cloud_sql_proxy" 156 else: 157 sql_proxy_command = os.path.join(UPDATEBOT_PATH, "..", "cloud_sql_proxy.exe") 158 159 sql_proxy_command += ( 160 " -instances=" + instance_name + "=tcp:3306 -credential_file=sql-proxy-key" 161 ) 162 sql_proxy_args = { 163 "stdout": subprocess.PIPE, 164 "stderr": subprocess.PIPE, 165 "shell": True, 166 "start_new_session": True, 167 } 168 169 if platform.system() == "Windows": 170 si = subprocess.STARTUPINFO() 171 si.dwFlags = subprocess.CREATE_NEW_PROCESS_GROUP 172 173 sql_proxy_args["startupinfo"] = si 174 175 sql_proxy = subprocess.Popen((sql_proxy_command), **sql_proxy_args) 176 177 try: 178 (stdout, stderr) = sql_proxy.communicate(input=None, timeout=2) 179 log("sql proxy stdout:", stdout.decode("utf-8")) 180 log("sql proxy stderr:", stderr.decode("utf-8")) 181 except subprocess.TimeoutExpired: 182 log("no sqlproxy output in 2 seconds, this means it probably didn't error.") 183 log("sqlproxy pid:", sql_proxy.pid) 184 185 database_config["host"] = "127.0.0.1" 186 187 # Vendor ================================================= 188 log("Getting Updatebot ready...") 189 os.chdir(UPDATEBOT_PATH) 190 localconfig = { 191 "General": { 192 "env": OPERATING_MODE, 193 "gecko-path": GECKO_DEV_PATH, 194 "soft_timeout": 3600, 195 }, 196 "Logging": { 197 "local": True, 198 "sentry": True, 199 "sentry_config": {"url": sentry_url, "debug": False}, 200 }, 201 "Database": database_config, 202 "Bugzilla": { 203 "apikey": bugzilla_api_key, 204 }, 205 "Taskcluster": { 206 "url_treeherder": "https://treeherder.mozilla.org/", 207 "url_taskcluster": "http://taskcluster/", 208 }, 209 } 210 211 log("Writing local config file") 212 config = open("localconfig.py", "w") 213 config.write("localconfig = " + str(localconfig)) 214 config.close() 215 216 log("Running updatebot") 217 # On Windows, Updatebot is run by windows-setup.sh 218 if platform.system() == "Linux": 219 subprocess.check_call(["python3", "-m", "poetry", "run", "./automation.py"]) 220 221 # Clean up =============================================== 222 log("Killing cloud_sql_proxy") 223 os.kill(sql_proxy.pid, signal.SIGTERM)