creator.py (9275B)
1 # This Source Code Form is subject to the terms of the Mozilla Public 2 # License, v. 2.0. If a copy of the MPL was not distributed with this 3 # file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 """ Creates or updates profiles. 5 6 The profile creation works as following: 7 8 For each scenario: 9 10 - The latest indexed profile is picked on TC, if none we create a fresh profile 11 - The scenario is done against it 12 - The profile is uploaded on TC, replacing the previous one as the freshest 13 14 For each platform we keep a changelog file that keep track of each update 15 with the Task ID. That offers us the ability to get a profile from a specific 16 date in the past. 17 18 Artifacts are staying in TaskCluster for 3 months, and then they are removed, 19 so the oldest profile we can get is 3 months old. Profiles are being updated 20 continuously, so even after 3 months they are still getting "older". 21 22 When Firefox changes its version, profiles from the previous version 23 should work as expected. Each profile tarball comes with a metadata file 24 that keep track of the Firefox version that was used and the profile age. 25 """ 26 import os 27 import tempfile 28 import shutil 29 30 from arsenic import get_session 31 from arsenic.browsers import Firefox 32 33 from condprof.util import fresh_profile, logger, obfuscate_file, obfuscate, get_version 34 from condprof.helpers import close_extra_windows 35 from condprof.scenarii import scenarii 36 from condprof.client import get_profile, ProfileNotFoundError 37 from condprof.archiver import Archiver 38 from condprof.customization import get_customization 39 from condprof.metadata import Metadata 40 41 42 START, INIT_GECKODRIVER, START_SESSION, START_SCENARIO = range(4) 43 44 45 class ProfileCreator: 46 def __init__( 47 self, 48 scenario, 49 customization, 50 archive, 51 changelog, 52 force_new, 53 env, 54 skip_logs=False, 55 remote_test_root="/sdcard/test_root/", 56 ): 57 self.env = env 58 self.scenario = scenario 59 self.customization = customization 60 self.archive = archive 61 self.changelog = changelog 62 self.force_new = force_new 63 self.skip_logs = skip_logs 64 self.remote_test_root = remote_test_root 65 self.customization_data = get_customization(customization) 66 self.tmp_dir = None 67 68 # Make a temporary directory for the logs if an 69 # archive dir is not provided 70 if not self.archive: 71 if os.environ.get("MOZ_UPLOAD_DIR"): 72 self.tmp_dir = os.path.join( 73 os.environ.get("MOZ_UPLOAD_DIR"), "condprof" 74 ) 75 os.makedirs(self.tmp_dir, exist_ok=True) 76 else: 77 self.tmp_dir = tempfile.mkdtemp() 78 79 def _log_filename(self, name): 80 filename = "%s-%s-%s.log" % ( 81 name, 82 self.scenario, 83 self.customization_data["name"], 84 ) 85 return os.path.join(self.archive or self.tmp_dir, filename) 86 87 async def run(self, headless=True): 88 logger.info( 89 "Building %s x %s" % (self.scenario, self.customization_data["name"]) 90 ) 91 92 if self.scenario in self.customization_data.get("ignore_scenario", []): 93 logger.info("Skipping (ignored scenario in that customization)") 94 return 95 96 filter_by_platform = self.customization_data.get("platforms") 97 if filter_by_platform and self.env.target_platform not in filter_by_platform: 98 logger.info("Skipping (ignored platform in that customization)") 99 return 100 101 with self.env.get_device( 102 2828, verbose=True, remote_test_root=self.remote_test_root 103 ) as device: 104 try: 105 with self.env.get_browser(): 106 metadata = await self.build_profile(device, headless) 107 except Exception: 108 raise 109 finally: 110 if not self.skip_logs: 111 self.env.dump_logs() 112 113 if not self.archive: 114 return 115 116 logger.info("Creating generic archive") 117 names = ["profile-%(platform)s-%(name)s-%(customization)s.tgz"] 118 if metadata["name"] == "full" and metadata["customization"] == "default": 119 names = [ 120 "profile-%(platform)s-%(name)s-%(customization)s.tgz", 121 "profile-v%(version)s-%(platform)s-%(name)s-%(customization)s.tgz", 122 ] 123 124 for name in names: 125 # The following removes files from the profile before archival. An 126 # alternative is to exclude the file in the _filter of create_archive. 127 shutil.rmtree(os.path.join(self.env.profile, "cache"), ignore_errors=True) 128 shutil.rmtree(os.path.join(self.env.profile, "cache2"), ignore_errors=True) 129 130 archiver = Archiver(self.scenario, self.env.profile, self.archive) 131 # the archive name is of the form 132 # profile[-vXYZ.x]-<platform>-<scenario>-<customization>.tgz 133 name = name % metadata 134 archive_name = os.path.join(self.archive, name) 135 dir = os.path.dirname(archive_name) 136 if not os.path.exists(dir): 137 os.makedirs(dir) 138 archiver.create_archive(archive_name) 139 logger.info("Archive created at %s" % archive_name) 140 statinfo = os.stat(archive_name) 141 logger.info("Current size is %d" % statinfo.st_size) 142 143 logger.info("Extracting logs") 144 if "logs" in metadata: 145 logs = metadata.pop("logs") 146 for prefix, prefixed_logs in logs.items(): 147 for log in prefixed_logs: 148 content = obfuscate(log["content"])[1] 149 with open(os.path.join(dir, prefix + "-" + log["name"]), "wb") as f: 150 f.write(content.encode("utf-8")) 151 152 if metadata.get("result", 0) != 0: 153 logger.info("The scenario returned a bad exit code") 154 raise Exception(metadata.get("result_message", "scenario error")) 155 self.changelog.append("update", **metadata) 156 157 async def build_profile(self, device, headless): 158 scenario = self.scenario 159 profile = self.env.profile 160 customization_data = self.customization_data 161 162 scenario_func = scenarii[scenario] 163 if scenario in customization_data.get("scenario", {}): 164 options = customization_data["scenario"][scenario] 165 logger.info("Loaded options for that scenario %s" % str(options)) 166 else: 167 options = {} 168 169 # Adding general options 170 options["platform"] = self.env.target_platform 171 172 if not self.force_new: 173 try: 174 custom_name = customization_data["name"] 175 get_profile(profile, self.env.target_platform, scenario, custom_name) 176 except ProfileNotFoundError: 177 # XXX we'll use a fresh profile for now 178 fresh_profile(profile, customization_data) 179 else: 180 fresh_profile(profile, customization_data) 181 182 logger.info("Updating profile located at %r" % profile) 183 metadata = Metadata(profile) 184 185 logger.info("Starting the Gecko app...") 186 adb_logs = self._log_filename("adb") 187 self.env.prepare(logfile=adb_logs) 188 geckodriver_logs = self._log_filename("geckodriver") 189 logger.info("Writing geckodriver logs in %s" % geckodriver_logs) 190 step = START 191 try: 192 firefox_instance = Firefox(**self.env.get_browser_args(headless)) 193 step = INIT_GECKODRIVER 194 with open(geckodriver_logs, "w") as glog: 195 geckodriver = self.env.get_geckodriver(log_file=glog) 196 step = START_SESSION 197 async with get_session(geckodriver, firefox_instance) as session: 198 step = START_SCENARIO 199 self.env.check_session(session) 200 logger.info("Running the %s scenario" % scenario) 201 metadata.update(await scenario_func(session, options)) 202 logger.info("%s scenario done." % scenario) 203 await close_extra_windows(session) 204 except Exception: 205 logger.error("%s scenario broke!" % scenario) 206 if step == START: 207 logger.info("Could not initialize the browser") 208 elif step == INIT_GECKODRIVER: 209 logger.info("Could not initialize Geckodriver") 210 elif step == START_SESSION: 211 logger.info( 212 "Could not start the session, check %s first" % geckodriver_logs 213 ) 214 else: 215 logger.info("Could not run the scenario, probably a faulty scenario") 216 raise 217 finally: 218 self.env.stop_browser() 219 for logfile in (adb_logs, geckodriver_logs): 220 if os.path.exists(logfile): 221 obfuscate_file(logfile) 222 self.env.collect_profile() 223 224 # writing metadata 225 metadata.write( 226 name=self.scenario, 227 customization=self.customization_data["name"], 228 version=self.env.get_browser_version(), 229 platform=self.env.target_platform, 230 ) 231 232 logger.info("Profile at %s.\nDone." % profile) 233 return metadata