skia_gold_session.py (24280B)
1 # Copyright 2020 The Chromium Authors 2 # Use of this source code is governed by a BSD-style license that can be 3 # found in the LICENSE file. 4 """Class for interacting with the Skia Gold image diffing service.""" 5 6 import enum 7 import logging 8 import os 9 import platform 10 import shutil 11 import sys 12 import tempfile 13 import time 14 from typing import Any, Dict, List, Optional, Tuple 15 16 import dataclasses # Built-in, but pylint gives an ordering false positive. 17 18 from skia_gold_common import skia_gold_properties 19 20 CHROMIUM_SRC = os.path.realpath( 21 os.path.join(os.path.dirname(__file__), '..', '..')) 22 23 GOLDCTL_BINARY = os.path.join(CHROMIUM_SRC, 'tools', 'skia_goldctl') 24 if sys.platform == 'win32': 25 GOLDCTL_BINARY = os.path.join(GOLDCTL_BINARY, 'win', 'goldctl') + '.exe' 26 elif sys.platform == 'darwin': 27 machine = platform.machine().lower() 28 if any(machine.startswith(m) for m in ('arm64', 'aarch64')): 29 GOLDCTL_BINARY = os.path.join(GOLDCTL_BINARY, 'mac_arm64', 'goldctl') 30 else: 31 GOLDCTL_BINARY = os.path.join(GOLDCTL_BINARY, 'mac_amd64', 'goldctl') 32 else: 33 GOLDCTL_BINARY = os.path.join(GOLDCTL_BINARY, 'linux', 'goldctl') 34 35 36 StepRetVal = Tuple[int, Optional[str]] 37 38 39 class SkiaGoldSession(): 40 @enum.unique 41 class StatusCodes(enum.IntEnum): 42 """Status codes for RunComparison.""" 43 SUCCESS = 0 44 AUTH_FAILURE = 1 45 INIT_FAILURE = 2 46 COMPARISON_FAILURE_REMOTE = 3 47 COMPARISON_FAILURE_LOCAL = 4 48 LOCAL_DIFF_FAILURE = 5 49 NO_OUTPUT_MANAGER = 6 50 51 @dataclasses.dataclass 52 class ComparisonResults(): 53 """Struct-like object for storing results of an image comparison.""" 54 public_triage_link: Optional[str] = None 55 internal_triage_link: Optional[str] = None 56 triage_link_omission_reason: Optional[str] = None 57 local_diff_given_image: Optional[str] = None 58 local_diff_closest_image: Optional[str] = None 59 local_diff_diff_image: Optional[str] = None 60 61 def __init__(self, 62 working_dir: str, 63 gold_properties: skia_gold_properties.SkiaGoldProperties, 64 keys_file: str, 65 corpus: str, 66 instance: str, 67 bucket: Optional[str] = None): 68 """Abstract class to handle all aspects of image comparison via Skia Gold. 69 70 A single SkiaGoldSession is valid for a single instance/corpus/keys_file 71 combination. 72 73 Args: 74 working_dir: The directory to store config files, etc. 75 gold_properties: A skia_gold_properties.SkiaGoldProperties instance for 76 the current test run. 77 keys_file: A path to a JSON file containing various comparison config data 78 such as corpus and debug information like the hardware/software 79 configuration the images will be produced on. 80 corpus: The corpus that images that will be compared belong to. 81 instance: The name of the Skia Gold instance to interact with. 82 bucket: Overrides the formulaic Google Storage bucket name generated by 83 goldctl 84 """ 85 self._working_dir = working_dir 86 self._gold_properties = gold_properties 87 self._corpus = corpus 88 self._instance = instance 89 self._bucket = bucket 90 self._local_png_directory = (self._gold_properties.local_png_directory 91 or tempfile.mkdtemp()) 92 with tempfile.NamedTemporaryFile(suffix='.txt', 93 dir=working_dir, 94 delete=False) as triage_link_file: 95 self._triage_link_file = triage_link_file.name 96 # A map of image name to ComparisonResults for that image. 97 self._comparison_results: Dict[str, SkiaGoldSession.ComparisonResults] = {} 98 self._authenticated = False 99 self._initialized = False 100 101 # Copy the given keys file to the working directory in case it ends up 102 # getting deleted before we try to use it. 103 self._keys_file = os.path.join(working_dir, 'gold_keys.json') 104 shutil.copy(keys_file, self._keys_file) 105 106 def RunComparison(self, 107 name: str, 108 png_file: str, 109 output_manager: Optional[Any] = None, 110 inexact_matching_args: Optional[List[str]] = None, 111 use_luci: bool = True, 112 service_account: Optional[str] = None, 113 optional_keys: Optional[Dict[str, str]] = None, 114 force_dryrun: bool = False) -> StepRetVal: 115 """Helper method to run all steps to compare a produced image. 116 117 Handles authentication, itnitialization, comparison, and, if necessary, 118 local diffing. 119 120 Args: 121 name: The name of the image being compared. 122 png_file: A path to a PNG file containing the image to be compared. 123 output_manager: An output manager to use to store diff links. The 124 argument's type depends on what type a subclasses' _StoreDiffLinks 125 implementation expects. Can be None even if _StoreDiffLinks expects 126 a valid input, but will fail if it ever actually needs to be used. 127 inexact_matching_args: A list of strings containing extra command line 128 arguments to pass to Gold for inexact matching. Can be omitted to use 129 exact matching. 130 use_luci: If true, authentication will use the service account provided by 131 the LUCI context. If false, will attempt to use whatever is set up in 132 gsutil, which is only supported for local runs. 133 service_account: If set, uses the provided service account instead of 134 LUCI_CONTEXT or whatever is set in gsutil. 135 optional_keys: A dict containing optional key/value pairs to pass to Gold 136 for this comparison. Optional keys are keys unrelated to the 137 configuration the image was produced on, e.g. a comment or whether 138 Gold should treat the image as ignored. 139 force_dryrun: A boolean denoting whether dryrun should be forced on 140 regardless of whether this is a local comparison or not. 141 142 Returns: 143 A tuple (status, error). |status| is a value from 144 SkiaGoldSession.StatusCodes signifying the result of the comparison. 145 |error| is an error message describing the status if not successful. 146 """ 147 # TODO(b/295350872): Remove this and other timestamp logging in this code 148 # once the source of flaky slowness is tracked down. 149 logging.info('Starting Gold auth') 150 start_time = time.time() 151 auth_rc, auth_stdout = self.Authenticate(use_luci=use_luci, 152 service_account=service_account) 153 logging.info('Gold auth took %fs', time.time() - start_time) 154 if auth_rc: 155 return self.StatusCodes.AUTH_FAILURE, auth_stdout 156 157 logging.info('Starting Gold initialization') 158 start_time = time.time() 159 init_rc, init_stdout = self.Initialize() 160 logging.info('Gold initialization took %fs', time.time() - start_time) 161 if init_rc: 162 return self.StatusCodes.INIT_FAILURE, init_stdout 163 164 logging.info('Starting Gold comparison in shared code') 165 start_time = time.time() 166 compare_rc, compare_stdout = self.Compare( 167 name=name, 168 png_file=png_file, 169 inexact_matching_args=inexact_matching_args, 170 optional_keys=optional_keys, 171 force_dryrun=force_dryrun) 172 logging.info('Gold comparison in shared code took %fs', 173 time.time() - start_time) 174 if not compare_rc: 175 return self.StatusCodes.SUCCESS, None 176 177 logging.error('Gold comparison failed: %s', compare_stdout) 178 if not self._gold_properties.local_pixel_tests: 179 return self.StatusCodes.COMPARISON_FAILURE_REMOTE, compare_stdout 180 181 if self._RequiresOutputManager() and not output_manager: 182 return (self.StatusCodes.NO_OUTPUT_MANAGER, 183 'No output manager for local diff images') 184 185 diff_rc, diff_stdout = self.Diff(name=name, 186 png_file=png_file, 187 output_manager=output_manager) 188 if diff_rc: 189 return self.StatusCodes.LOCAL_DIFF_FAILURE, diff_stdout 190 return self.StatusCodes.COMPARISON_FAILURE_LOCAL, compare_stdout 191 192 def Authenticate(self, 193 use_luci: bool = True, 194 service_account: Optional[str] = None) -> StepRetVal: 195 """Authenticates with Skia Gold for this session. 196 197 Args: 198 use_luci: If true, authentication will use the service account provided 199 by the LUCI context. If false, will attempt to use whatever is set up 200 in gsutil, which is only supported for local runs. 201 service_account: If set, uses the provided service account instead of 202 LUCI_CONTEXT or whatever is set in gsutil. 203 204 Returns: 205 A tuple (return_code, output). |return_code| is the return code of the 206 authentication process. |output| is the stdout + stderr of the 207 authentication process. 208 """ 209 if self._authenticated: 210 return 0, None 211 if self._gold_properties.bypass_skia_gold_functionality: 212 logging.warning('Not actually authenticating with Gold due to ' 213 '--bypass-skia-gold-functionality being present.') 214 return 0, None 215 assert not (use_luci and service_account) 216 217 auth_cmd = [GOLDCTL_BINARY, 'auth', '--work-dir', self._working_dir] 218 if use_luci: 219 auth_cmd.append('--luci') 220 elif service_account: 221 auth_cmd.extend(['--service-account', service_account]) 222 elif not self._gold_properties.local_pixel_tests: 223 raise RuntimeError( 224 'Cannot authenticate to Skia Gold with use_luci=False without a ' 225 'service account unless running local pixel tests') 226 227 rc, stdout = self._RunCmdForRcAndOutput(auth_cmd) 228 if rc == 0: 229 self._authenticated = True 230 return rc, stdout 231 232 def Initialize(self) -> StepRetVal: 233 """Initializes the working directory if necessary. 234 235 This can technically be skipped if the same information is passed to the 236 command used for image comparison, but that is less efficient under the 237 hood. Doing it that way effectively requires an initialization for every 238 comparison (~250 ms) instead of once at the beginning. 239 240 Returns: 241 A tuple (return_code, output). |return_code| is the return code of the 242 initialization process. |output| is the stdout + stderr of the 243 initialization process. 244 """ 245 if self._initialized: 246 return 0, None 247 if self._gold_properties.bypass_skia_gold_functionality: 248 logging.warning('Not actually initializing Gold due to ' 249 '--bypass-skia-gold-functionality being present.') 250 return 0, None 251 252 init_cmd = [ 253 GOLDCTL_BINARY, 254 'imgtest', 255 'init', 256 '--passfail', 257 '--instance', 258 self._instance, 259 '--corpus', 260 self._corpus, 261 '--keys-file', 262 self._keys_file, 263 '--work-dir', 264 self._working_dir, 265 '--failure-file', 266 self._triage_link_file, 267 '--commit', 268 self._gold_properties.git_revision, 269 ] 270 if self._bucket: 271 init_cmd.extend(['--bucket', self._bucket]) 272 if self._gold_properties.IsTryjobRun(): 273 init_cmd.extend([ 274 '--issue', 275 str(self._gold_properties.issue), 276 '--patchset', 277 str(self._gold_properties.patchset), 278 '--jobid', 279 str(self._gold_properties.job_id), 280 '--crs', 281 str(self._gold_properties.code_review_system), 282 '--cis', 283 str(self._gold_properties.continuous_integration_system), 284 ]) 285 286 rc, stdout = self._RunCmdForRcAndOutput(init_cmd) 287 if rc == 0: 288 self._initialized = True 289 return rc, stdout 290 291 def Compare(self, 292 name: str, 293 png_file: str, 294 inexact_matching_args: Optional[List[str]] = None, 295 optional_keys: Optional[Dict[str, str]] = None, 296 force_dryrun: bool = False) -> StepRetVal: 297 """Compares the given image to images known to Gold. 298 299 Triage links can later be retrieved using GetTriageLinks(). 300 301 Args: 302 name: The name of the image being compared. 303 png_file: A path to a PNG file containing the image to be compared. 304 inexact_matching_args: A list of strings containing extra command line 305 arguments to pass to Gold for inexact matching. Can be omitted to use 306 exact matching. 307 optional_keys: A dict containing optional key/value pairs to pass to Gold 308 for this comparison. Optional keys are keys unrelated to the 309 configuration the image was produced on, e.g. a comment or whether 310 Gold should treat the image as ignored. 311 force_dryrun: A boolean denoting whether dryrun should be forced on 312 regardless of whether this is a local comparison or not. 313 314 Returns: 315 A tuple (return_code, output). |return_code| is the return code of the 316 comparison process. |output| is the stdout + stderr of the comparison 317 process. 318 """ 319 if self._gold_properties.bypass_skia_gold_functionality: 320 logging.warning('Not actually comparing with Gold due to ' 321 '--bypass-skia-gold-functionality being present.') 322 return 0, None 323 324 compare_cmd = [ 325 GOLDCTL_BINARY, 326 'imgtest', 327 'add', 328 '--test-name', 329 name, 330 '--png-file', 331 png_file, 332 '--work-dir', 333 self._working_dir, 334 ] 335 if self._gold_properties.local_pixel_tests or force_dryrun: 336 compare_cmd.append('--dryrun') 337 if inexact_matching_args: 338 logging.info('Using inexact matching arguments for image %s: %s', name, 339 inexact_matching_args) 340 compare_cmd.extend(inexact_matching_args) 341 342 optional_keys = optional_keys or {} 343 for k, v in optional_keys.items(): 344 compare_cmd.extend([ 345 '--add-test-optional-key', 346 '%s:%s' % (k, v), 347 ]) 348 349 logging.info('Starting Gold triage link file clear') 350 start_time = time.time() 351 self._ClearTriageLinkFile() 352 logging.info('Gold triage link file clear took %fs', 353 time.time() - start_time) 354 logging.info('Starting Gold comparison command') 355 start_time = time.time() 356 rc, stdout = self._RunCmdForRcAndOutput(compare_cmd) 357 logging.info('Gold comparison command took %fs', time.time() - start_time) 358 359 self._comparison_results[name] = self.ComparisonResults() 360 if rc == 0: 361 self._comparison_results[name].triage_link_omission_reason = ( 362 'Comparison succeeded, no triage link') 363 elif self._gold_properties.IsTryjobRun(): 364 cl_triage_link = ('https://{instance}-gold.skia.org/cl/{crs}/{issue}') 365 cl_triage_link = cl_triage_link.format( 366 instance=self._instance, 367 crs=self._gold_properties.code_review_system, 368 issue=self._gold_properties.issue) 369 self._comparison_results[name].internal_triage_link = cl_triage_link 370 self._comparison_results[name].public_triage_link =\ 371 self._GeneratePublicTriageLink(cl_triage_link) 372 else: 373 try: 374 logging.info('Starting triage link file read') 375 start_time = time.time() 376 with open(self._triage_link_file) as tlf: 377 triage_link = tlf.read().strip() 378 logging.info('Triage link file read took %fs', time.time() - start_time) 379 if not triage_link: 380 self._comparison_results[name].triage_link_omission_reason = ( 381 'Gold did not provide a triage link. This is likely a bug on ' 382 "Gold's end.") 383 self._comparison_results[name].internal_triage_link = None 384 self._comparison_results[name].public_triage_link = None 385 else: 386 self._comparison_results[name].internal_triage_link = triage_link 387 self._comparison_results[name].public_triage_link =\ 388 self._GeneratePublicTriageLink(triage_link) 389 except IOError: 390 self._comparison_results[name].triage_link_omission_reason = ( 391 'Failed to read triage link from file') 392 return rc, stdout 393 394 def Diff(self, name: str, png_file: str, output_manager: Any) -> StepRetVal: 395 """Performs a local image diff against the closest known positive in Gold. 396 397 This is used for running tests on a workstation, where uploading data to 398 Gold for ingestion is not allowed, and thus the web UI is not available. 399 400 Image links can later be retrieved using Get*ImageLink(). 401 402 Args: 403 name: The name of the image being compared. 404 png_file: The path to a PNG file containing the image to be diffed. 405 output_manager: An output manager to use to store diff links. The 406 argument's type depends on what type a subclasses' _StoreDiffLinks 407 implementation expects. 408 409 Returns: 410 A tuple (return_code, output). |return_code| is the return code of the 411 diff process. |output| is the stdout + stderr of the diff process. 412 """ 413 # Instead of returning that everything is okay and putting in dummy links, 414 # just fail since this should only be called when running locally and 415 # --bypass-skia-gold-functionality is only meant for use on the bots. 416 if self._gold_properties.bypass_skia_gold_functionality: 417 raise RuntimeError( 418 '--bypass-skia-gold-functionality is not supported when running ' 419 'tests locally.') 420 421 output_dir = self._CreateDiffOutputDir(name) 422 # TODO(skbug.com/10611): Remove this temporary work dir and instead just use 423 # self._working_dir once `goldctl diff` stops clobbering the auth files in 424 # the provided work directory. 425 temp_work_dir = tempfile.mkdtemp() 426 # shutil.copytree() fails if the destination already exists, so use a 427 # subdirectory of the temporary directory. 428 temp_work_dir = os.path.join(temp_work_dir, 'diff_work_dir') 429 try: 430 shutil.copytree(self._working_dir, temp_work_dir) 431 diff_cmd = [ 432 GOLDCTL_BINARY, 433 'diff', 434 '--corpus', 435 self._corpus, 436 '--instance', 437 self._GetDiffGoldInstance(), 438 '--input', 439 png_file, 440 '--test', 441 name, 442 '--work-dir', 443 temp_work_dir, 444 '--out-dir', 445 output_dir, 446 ] 447 rc, stdout = self._RunCmdForRcAndOutput(diff_cmd) 448 self._StoreDiffLinks(name, output_manager, output_dir) 449 return rc, stdout 450 finally: 451 shutil.rmtree(os.path.realpath(os.path.join(temp_work_dir, '..'))) 452 453 def GetTriageLinks(self, name: str) -> Tuple[str, str]: 454 """Gets the triage links for the given image. 455 456 Args: 457 name: The name of the image to retrieve the triage link for. 458 459 Returns: 460 A tuple (public, internal). |public| is a string containing the triage 461 link for the public Gold instance if it is available, or None if it is not 462 available for some reason. |internal| is the same as |public|, but 463 containing a link to the internal Gold instance. The reason for links not 464 being available can be retrieved using GetTriageLinkOmissionReason. 465 """ 466 comparison_results = self._comparison_results.get(name, 467 self.ComparisonResults()) 468 return (comparison_results.public_triage_link, 469 comparison_results.internal_triage_link) 470 471 def GetTriageLinkOmissionReason(self, name: str) -> str: 472 """Gets the reason why a triage link is not available for an image. 473 474 Args: 475 name: The name of the image whose triage link does not exist. 476 477 Returns: 478 A string containing the reason why a triage link is not available. 479 """ 480 if name not in self._comparison_results: 481 return 'No image comparison performed for %s' % name 482 results = self._comparison_results[name] 483 # This method should not be called if there is a valid triage link. 484 assert results.public_triage_link is None 485 assert results.internal_triage_link is None 486 if results.triage_link_omission_reason: 487 return results.triage_link_omission_reason 488 if results.local_diff_given_image: 489 return 'Gold only used to do a local image diff' 490 raise RuntimeError( 491 'Somehow have a ComparisonResults instance for %s that should not ' 492 'exist' % name) 493 494 def GetGivenImageLink(self, name: str) -> str: 495 """Gets the link to the given image used for local diffing. 496 497 Args: 498 name: The name of the image that was diffed. 499 500 Returns: 501 A string containing the link to where the image is saved, or None if it 502 does not exist. 503 """ 504 assert name in self._comparison_results 505 return self._comparison_results[name].local_diff_given_image 506 507 def GetClosestImageLink(self, name: str) -> str: 508 """Gets the link to the closest known image used for local diffing. 509 510 Args: 511 name: The name of the image that was diffed. 512 513 Returns: 514 A string containing the link to where the image is saved, or None if it 515 does not exist. 516 """ 517 assert name in self._comparison_results 518 return self._comparison_results[name].local_diff_closest_image 519 520 def GetDiffImageLink(self, name: str) -> str: 521 """Gets the link to the diff between the given and closest images. 522 523 Args: 524 name: The name of the image that was diffed. 525 526 Returns: 527 A string containing the link to where the image is saved, or None if it 528 does not exist. 529 """ 530 assert name in self._comparison_results 531 return self._comparison_results[name].local_diff_diff_image 532 533 def _GeneratePublicTriageLink(self, internal_link: str) -> str: 534 """Generates a public triage link given an internal one. 535 536 Args: 537 internal_link: A string containing a triage link pointing to an internal 538 Gold instance. 539 540 Returns: 541 A string containing a triage link pointing to the public mirror of the 542 link pointed to by |internal_link|. 543 """ 544 return internal_link.replace('%s-gold' % self._instance, 545 '%s-public-gold' % self._instance) 546 547 def _ClearTriageLinkFile(self) -> None: 548 """Clears the contents of the triage link file. 549 550 This should be done before every comparison since goldctl appends to the 551 file instead of overwriting its contents, which results in multiple triage 552 links getting concatenated together if there are multiple failures. 553 """ 554 open(self._triage_link_file, 'w').close() 555 556 def _CreateDiffOutputDir(self, _name: str) -> str: 557 # We don't use self._local_png_directory here since we want it to be 558 # automatically cleaned up with the working directory. Any subclasses that 559 # want to keep it around can override this method. 560 return tempfile.mkdtemp(dir=self._working_dir) 561 562 def _GetDiffGoldInstance(self) -> str: 563 """Gets the Skia Gold instance to use for the Diff step. 564 565 This can differ based on how a particular instance is set up, mainly 566 depending on whether it is set up for internal results or not. 567 """ 568 # TODO(skbug.com/10610): Decide whether to use the public or 569 # non-public instance once authentication is fixed for the non-public 570 # instance. 571 return str(self._instance) + '-public' 572 573 def _StoreDiffLinks(self, image_name: str, output_manager: Any, 574 output_dir: str) -> None: 575 """Stores the local diff files as links. 576 577 The ComparisonResults entry for |image_name| should have its *_image fields 578 filled after this unless corresponding images were not found on disk. 579 580 Args: 581 image_name: A string containing the name of the image that was diffed. 582 output_manager: An output manager used used to surface links to users, 583 if necessary. The expected argument type depends on each subclasses' 584 implementation of this method. 585 output_dir: A string containing the path to the directory where diff 586 output image files where saved. 587 """ 588 raise NotImplementedError() 589 590 def _RequiresOutputManager(self) -> bool: 591 """Whether this session implementation requires an output manager.""" 592 return True 593 594 @staticmethod 595 def _RunCmdForRcAndOutput(cmd: List[str]) -> Tuple[int, str]: 596 """Runs |cmd| and returns its returncode and output. 597 598 Args: 599 cmd: A list containing the command line to run. 600 601 Returns: 602 A tuple (rc, output), where, |rc| is the returncode of the command and 603 |output| is the stdout + stderr of the command. 604 """ 605 raise NotImplementedError()