comm-task-env (6400B)
1 #!/usr/bin/python3 -u 2 # This Source Code Form is subject to the terms of the Mozilla Public 3 # License, v. 2.0. If a copy of the MPL was not distributed with this 4 # file, You can obtain one at http://mozilla.org/MPL/2.0/. 5 """ 6 Thunderbird build environment prep for run-task, 7 for use with comm-central derived repositories. 8 9 This script is meant to run prior to run-task on repositories like 10 comm-central that need to check out a copy of a mozilla repository 11 in order to build. 12 See bug 1491371 for background on why this is necessary. 13 14 A project will have a file named ".gecko_rev.yml" in it's root. See the 15 constant "GECKO_REV_CONF" if you want to change that. To download it, the 16 script uses the project repository URL and the revision number. 17 Those are defined in the environment variables: 18 COMM_HEAD_REPOSITORY 19 COMM_HEAD_REV 20 21 .gecko_rev.yml has a structure like (for comm-central): 22 ``` 23 GECKO_BASE_REPOSITORY: https://hg.mozilla.org/mozilla-unified 24 GECKO_HEAD_REPOSITORY: https://hg.mozilla.org/mozilla-central 25 GECKO_HEAD_REF: default 26 ``` 27 or for branches: 28 ``` 29 GECKO_BASE_REPOSITORY: https://hg.mozilla.org/mozilla-unified 30 GECKO_HEAD_REPOSITORY: https://hg.mozilla.org/releases/mozilla-beta 31 GECKO_HEAD_REF: THUNDERBIRD_60_VERBRANCH 32 GECKO_HEAD_REV: 6a830d12f15493a70b1192022c9985eba2139910 33 34 Note about GECKO_HEAD_REV and GECKO_HEAD_REF: 35 GECKO_HEAD_REF is a branch name or "default". 36 GECKO_HEAD_REV is a revision hash. 37 ``` 38 """ 39 40 import sys 41 42 import os 43 import socket 44 import time 45 from datetime import datetime 46 from pprint import pformat 47 48 import urllib.error 49 import urllib.request 50 51 import yaml 52 53 if sys.version_info[0:2] < (3, 5): 54 print('run-task-wrapper requires Python 3.5+') 55 sys.exit(1) 56 57 GECKO_REV_CONF = ".gecko_rev.yml" 58 DEBUG = bool(os.environ.get("RTW_DEBUG", False)) 59 60 61 def print_message(msg, prefix=__file__, level=""): 62 """ 63 Print messages. 64 :param object msg: message to print, usually a string, but not always 65 :param str prefix: message prefix 66 :param str level: message level (DEBUG, ERROR, INFO) 67 """ 68 if not isinstance(msg, str): 69 msg = pformat(msg) 70 now = datetime.utcnow().isoformat() 71 # slice microseconds to 3 decimals. 72 now = now[:-3] if now[-7:-6] == '.' else now 73 if level: 74 sys.stdout.write('[{prefix} {now}Z] {level}: {msg}\n'.format( 75 prefix=prefix, now=now, level=level, msg=msg)) 76 else: 77 sys.stdout.write('[{prefix} {now}Z] {msg}\n'.format( 78 prefix=prefix, now=now, msg=msg)) 79 sys.stdout.flush() 80 81 82 def error_exit(msg): 83 """Print the error message and exit with error.""" 84 print_message(msg, level="ERROR") 85 if DEBUG: 86 raise Exception(msg) 87 88 sys.exit(1) 89 90 91 def print_debug(msg): 92 """Prints a message with DEBUG prefix if DEBUG is enabled 93 with the environment variable "RTW_DEBUG". 94 """ 95 if DEBUG: 96 print_message(msg, level="DEBUG") 97 98 99 def check_environ(): 100 """Check that the necessary environment variables to find the 101 comm- repository are defined. (Set in .taskcluster.yml) 102 :return: tuple(str, str) 103 """ 104 print_debug("Checking environment variables...") 105 project_head_repo = os.environ.get("COMM_HEAD_REPOSITORY", None) 106 project_head_rev = os.environ.get("COMM_HEAD_REV", None) 107 108 if project_head_repo is None or project_head_rev is None: 109 error_exit("Environment NOT Ok:\n\tHead: {}\n\tRev: {}\n").format( 110 project_head_repo, project_head_rev) 111 112 print_debug("Environment Ok:\n\tHead: {}\n\tRev: {}\n".format( 113 project_head_repo, project_head_rev)) 114 return project_head_repo, project_head_rev 115 116 117 def download_url(url, retry=1): 118 """Downloads the given URL. Naively retries (when asked) upon failure 119 :param url: str 120 :param retry: int 121 :return: str 122 """ 123 # Use 1-based counting for display and calculation purposes. 124 for i in range(1, retry+1): 125 try: 126 print_message('Fetching {}. Attempt {} of {}.'.format( 127 url, i, retry)) 128 with urllib.request.urlopen(url, timeout=10) as response: 129 data = response.read().decode("utf-8") 130 return data 131 except (urllib.error.URLError, socket.timeout) as exc: 132 print_message('Unable to retrieve {}'.format(url)) 133 if isinstance(exc, urllib.error.URLError): 134 print_message(exc.reason) 135 else: # socket.timeout 136 print_message('Connection timed out.') 137 138 if i < retry: # No more retries 139 wait_time = i * 5 # fail #1: sleep 5s. #2, sleep 10s 140 print_message('Retrying in {} seconds.'.format(wait_time)) 141 time.sleep(wait_time) 142 143 error_exit('No more retry attempts! Aborting.') 144 145 146 def fetch_gecko_conf(project_head_repo, project_revision): 147 """Downloads .gecko_rev.yml from the project repository 148 :param project_head_repo: str 149 :param project_revision: str 150 :return: dict 151 """ 152 gecko_conf_url = '/'.join( 153 [project_head_repo, 'raw-file', project_revision, GECKO_REV_CONF]) 154 155 gecko_conf_yml = download_url(gecko_conf_url, retry=5) 156 157 try: 158 gecko_conf = yaml.safe_load(gecko_conf_yml) 159 return gecko_conf 160 except yaml.YAMLError as exc: 161 err_txt = ["Error processing Gecko YAML configuration."] 162 if hasattr(exc, "problem_mark"): 163 mark = exc.problem_mark # pylint: disable=no-member 164 err_txt.append("Error position: line {}, column {}".format( 165 mark.line + 1, mark.column + 1)) 166 error_exit('\n'.join(err_txt)) 167 168 169 def update_environment(gecko_conf): 170 """Adds the new variables defined in gecko_conf to the 171 running environment. 172 :param gecko_conf: dict 173 """ 174 print_message("Updating environment with:") 175 print_message(gecko_conf) 176 os.environ.update(gecko_conf) 177 178 print_debug("New environment:") 179 print_debug(os.environ) 180 181 182 def exec_run_task(args): 183 """Executes run-task with a modified environment.""" 184 print_message("Executing: {}".format(pformat(args))) 185 os.execv(args[0], args[0:]) 186 187 188 def main(): 189 """Main function.""" 190 args = sys.argv[1:] # Remaining args starting with run-task 191 192 project_head_repo, project_revision = check_environ() 193 gecko_conf = fetch_gecko_conf(project_head_repo, project_revision) 194 update_environment(gecko_conf) 195 exec_run_task(args) 196 197 198 if __name__ == "__main__": 199 main()