bouncer_check.py (7049B)
1 #!/usr/bin/env python 2 # lint_ignore=E501 3 # This Source Code Form is subject to the terms of the Mozilla Public 4 # License, v. 2.0. If a copy of the MPL was not distributed with this file, 5 # You can obtain one at http://mozilla.org/MPL/2.0/. 6 """bouncer_check.py 7 8 A script to check HTTP statuses of Bouncer products to be shipped. 9 """ 10 11 import os 12 import sys 13 14 sys.path.insert(1, os.path.dirname(os.path.dirname(sys.path[0]))) 15 16 from mozharness.base.script import BaseScript 17 from mozharness.mozilla.automation import EXIT_STATUS_DICT, TBPL_FAILURE 18 19 BOUNCER_URL_PATTERN = "{bouncer_prefix}?product={product}&os={os}&lang={lang}" 20 21 22 class BouncerCheck(BaseScript): 23 config_options = [ 24 [ 25 ["--version"], 26 { 27 "dest": "version", 28 "help": "Version of release, eg: 39.0b5", 29 }, 30 ], 31 [ 32 ["--product-field"], 33 { 34 "dest": "product_field", 35 "help": "Version field of release from product details, eg: LATEST_FIREFOX_VERSION", # NOQA: E501 36 }, 37 ], 38 [ 39 ["--products-url"], 40 { 41 "dest": "products_url", 42 "help": "The URL of the current Firefox product versions", 43 "type": str, 44 "default": "https://product-details.mozilla.org/1.0/firefox_versions.json", 45 }, 46 ], 47 [ 48 ["--previous-version"], 49 { 50 "dest": "prev_versions", 51 "action": "extend", 52 "help": "Previous version(s)", 53 }, 54 ], 55 [ 56 ["--locale"], 57 { 58 "dest": "locales", 59 # Intentionally limited for several reasons: 60 # 1) faster to check 61 # 2) do not need to deal with situation when a new locale 62 # introduced and we do not have partials for it yet 63 # 3) it mimics the old Sentry behaviour that worked for ages 64 # 4) no need to handle ja-JP-mac 65 "default": ["en-US", "de", "it", "zh-TW"], 66 "action": "append", 67 "help": "List of locales to check.", 68 }, 69 ], 70 [ 71 ["-j", "--parallelization"], 72 { 73 "dest": "parallelization", 74 "default": 20, 75 "type": int, 76 "help": "Number of HTTP sessions running in parallel", 77 }, 78 ], 79 ] 80 81 def __init__(self, require_config_file=True): 82 super().__init__( 83 config_options=self.config_options, 84 require_config_file=require_config_file, 85 config={ 86 "cdn_urls": [ 87 "download-installer.cdn.mozilla.net", 88 "download.cdn.mozilla.net", 89 "download.mozilla.org", 90 "archive.mozilla.org", 91 ], 92 }, 93 all_actions=[ 94 "check-bouncer", 95 ], 96 default_actions=[ 97 "check-bouncer", 98 ], 99 ) 100 101 def _pre_config_lock(self, rw_config): 102 super()._pre_config_lock(rw_config) 103 104 if "product_field" not in self.config: 105 return 106 107 firefox_versions = self.load_json_url(self.config["products_url"]) 108 109 if self.config["product_field"] not in firefox_versions: 110 self.fatal("Unknown Firefox label: {}".format(self.config["product_field"])) 111 self.config["version"] = firefox_versions[self.config["product_field"]] 112 self.log("Set Firefox version {}".format(self.config["version"])) 113 114 def check_url(self, session, url): 115 from redo import retry 116 from requests.exceptions import HTTPError 117 118 try: 119 from urllib.parse import urlparse 120 except ImportError: 121 # Python 2 122 from urlparse import urlparse 123 124 def do_check_url(): 125 self.log(f"Checking {url}") 126 r = session.head(url, verify=True, timeout=10, allow_redirects=True) 127 try: 128 r.raise_for_status() 129 except HTTPError: 130 self.error(f"FAIL: {url}, status: {r.status_code}") 131 raise 132 133 final_url = urlparse(r.url) 134 if final_url.scheme != "https": 135 self.error(f"FAIL: URL scheme is not https: {r.url}") 136 self.return_code = EXIT_STATUS_DICT[TBPL_FAILURE] 137 138 if final_url.netloc not in self.config["cdn_urls"]: 139 self.error(f"FAIL: host not in allowed locations: {r.url}") 140 self.return_code = EXIT_STATUS_DICT[TBPL_FAILURE] 141 142 try: 143 retry(do_check_url, sleeptime=3, max_sleeptime=10, attempts=3) 144 except HTTPError: 145 # The error was already logged above. 146 self.return_code = EXIT_STATUS_DICT[TBPL_FAILURE] 147 return 148 149 def get_urls(self): 150 for product in self.config["products"].values(): 151 product_name = product["product-name"] % {"version": self.config["version"]} 152 for bouncer_platform in product["platforms"]: 153 for locale in self.config["locales"]: 154 url = BOUNCER_URL_PATTERN.format( 155 bouncer_prefix=self.config["bouncer_prefix"], 156 product=product_name, 157 os=bouncer_platform, 158 lang=locale, 159 ) 160 yield url 161 162 for product in self.config.get("partials", {}).values(): 163 for prev_version in self.config.get("prev_versions", []): 164 product_name = product["product-name"] % { 165 "version": self.config["version"], 166 "prev_version": prev_version, 167 } 168 for bouncer_platform in product["platforms"]: 169 for locale in self.config["locales"]: 170 url = BOUNCER_URL_PATTERN.format( 171 bouncer_prefix=self.config["bouncer_prefix"], 172 product=product_name, 173 os=bouncer_platform, 174 lang=locale, 175 ) 176 yield url 177 178 def check_bouncer(self): 179 from concurrent import futures 180 181 import requests 182 183 session = requests.Session() 184 http_adapter = requests.adapters.HTTPAdapter( 185 pool_connections=self.config["parallelization"], 186 pool_maxsize=self.config["parallelization"], 187 ) 188 session.mount("https://", http_adapter) 189 session.mount("http://", http_adapter) 190 191 with futures.ThreadPoolExecutor(self.config["parallelization"]) as e: 192 fs = [] 193 for url in self.get_urls(): 194 fs.append(e.submit(self.check_url, session, url)) 195 for f in futures.as_completed(fs): 196 f.result() 197 198 199 if __name__ == "__main__": 200 BouncerCheck().run_and_exit()