skia_gold_session_unittest.py (38130B)
1 #!/usr/bin/env vpython3 2 # Copyright 2020 The Chromium Authors 3 # Use of this source code is governed by a BSD-style license that can be 4 # found in the LICENSE file. 5 6 #pylint: disable=protected-access 7 8 import json 9 import os 10 import sys 11 import tempfile 12 from typing import Any 13 import unittest 14 15 if sys.version_info[0] == 2: 16 import mock 17 else: 18 import unittest.mock as mock 19 20 from pyfakefs import fake_filesystem_unittest 21 22 from skia_gold_common import skia_gold_properties 23 from skia_gold_common import skia_gold_session 24 from skia_gold_common import unittest_utils 25 26 createSkiaGoldArgs = unittest_utils.createSkiaGoldArgs 27 28 29 def assertArgWith(test: unittest.TestCase, arg_list: list, arg: Any, 30 value: Any) -> None: 31 i = arg_list.index(arg) 32 test.assertEqual(arg_list[i + 1], value) 33 34 35 class SkiaGoldSessionRunComparisonTest(fake_filesystem_unittest.TestCase): 36 """Tests the functionality of SkiaGoldSession.RunComparison.""" 37 38 def setUp(self) -> None: 39 self.setUpPyfakefs() 40 self._working_dir = tempfile.mkdtemp() 41 self._json_keys = tempfile.NamedTemporaryFile(delete=False).name 42 with open(self._json_keys, 'w') as f: 43 json.dump({}, f) 44 45 self.auth_patcher = mock.patch.object(skia_gold_session.SkiaGoldSession, 46 'Authenticate') 47 self.init_patcher = mock.patch.object(skia_gold_session.SkiaGoldSession, 48 'Initialize') 49 self.compare_patcher = mock.patch.object(skia_gold_session.SkiaGoldSession, 50 'Compare') 51 self.diff_patcher = mock.patch.object(skia_gold_session.SkiaGoldSession, 52 'Diff') 53 54 self.auth_mock = self.auth_patcher.start() 55 self.init_mock = self.init_patcher.start() 56 self.compare_mock = self.compare_patcher.start() 57 self.diff_mock = self.diff_patcher.start() 58 59 self.addCleanup(self.auth_patcher.stop) 60 self.addCleanup(self.init_patcher.stop) 61 self.addCleanup(self.compare_patcher.stop) 62 self.addCleanup(self.diff_patcher.stop) 63 64 def test_comparisonSuccess(self) -> None: 65 self.auth_mock.return_value = (0, None) 66 self.init_mock.return_value = (0, None) 67 self.compare_mock.return_value = (0, None) 68 sgp = skia_gold_properties.SkiaGoldProperties(createSkiaGoldArgs()) 69 session = skia_gold_session.SkiaGoldSession(self._working_dir, sgp, 70 self._json_keys, '', '') 71 status, _ = session.RunComparison('', '', None) 72 self.assertEqual(status, 73 skia_gold_session.SkiaGoldSession.StatusCodes.SUCCESS) 74 self.assertEqual(self.auth_mock.call_count, 1) 75 self.assertEqual(self.init_mock.call_count, 1) 76 self.assertEqual(self.compare_mock.call_count, 1) 77 self.assertEqual(self.diff_mock.call_count, 0) 78 79 def test_authFailure(self) -> None: 80 self.auth_mock.return_value = (1, 'Auth failed') 81 sgp = skia_gold_properties.SkiaGoldProperties(createSkiaGoldArgs()) 82 session = skia_gold_session.SkiaGoldSession(self._working_dir, sgp, 83 self._json_keys, '', '') 84 status, error = session.RunComparison('', '', None) 85 self.assertEqual(status, 86 skia_gold_session.SkiaGoldSession.StatusCodes.AUTH_FAILURE) 87 self.assertEqual(error, 'Auth failed') 88 self.assertEqual(self.auth_mock.call_count, 1) 89 self.assertEqual(self.init_mock.call_count, 0) 90 self.assertEqual(self.compare_mock.call_count, 0) 91 self.assertEqual(self.diff_mock.call_count, 0) 92 93 def test_initFailure(self) -> None: 94 self.auth_mock.return_value = (0, None) 95 self.init_mock.return_value = (1, 'Init failed') 96 sgp = skia_gold_properties.SkiaGoldProperties(createSkiaGoldArgs()) 97 session = skia_gold_session.SkiaGoldSession(self._working_dir, sgp, 98 self._json_keys, '', '') 99 status, error = session.RunComparison('', '', None) 100 self.assertEqual(status, 101 skia_gold_session.SkiaGoldSession.StatusCodes.INIT_FAILURE) 102 self.assertEqual(error, 'Init failed') 103 self.assertEqual(self.auth_mock.call_count, 1) 104 self.assertEqual(self.init_mock.call_count, 1) 105 self.assertEqual(self.compare_mock.call_count, 0) 106 self.assertEqual(self.diff_mock.call_count, 0) 107 108 def test_compareFailureRemote(self) -> None: 109 self.auth_mock.return_value = (0, None) 110 self.init_mock.return_value = (0, None) 111 self.compare_mock.return_value = (1, 'Compare failed') 112 args = createSkiaGoldArgs(local_pixel_tests=False) 113 sgp = skia_gold_properties.SkiaGoldProperties(args) 114 session = skia_gold_session.SkiaGoldSession(self._working_dir, sgp, 115 self._json_keys, '', '') 116 status, error = session.RunComparison('', '', None) 117 self.assertEqual( 118 status, 119 skia_gold_session.SkiaGoldSession.StatusCodes.COMPARISON_FAILURE_REMOTE) 120 self.assertEqual(error, 'Compare failed') 121 self.assertEqual(self.auth_mock.call_count, 1) 122 self.assertEqual(self.init_mock.call_count, 1) 123 self.assertEqual(self.compare_mock.call_count, 1) 124 self.assertEqual(self.diff_mock.call_count, 0) 125 126 def test_compareFailureLocal(self) -> None: 127 self.auth_mock.return_value = (0, None) 128 self.init_mock.return_value = (0, None) 129 self.compare_mock.return_value = (1, 'Compare failed') 130 self.diff_mock.return_value = (0, None) 131 args = createSkiaGoldArgs(local_pixel_tests=True) 132 sgp = skia_gold_properties.SkiaGoldProperties(args) 133 session = skia_gold_session.SkiaGoldSession(self._working_dir, sgp, 134 self._json_keys, '', '') 135 status, error = session.RunComparison('', '', 136 'Definitely an output manager') 137 self.assertEqual( 138 status, 139 skia_gold_session.SkiaGoldSession.StatusCodes.COMPARISON_FAILURE_LOCAL) 140 self.assertEqual(error, 'Compare failed') 141 self.assertEqual(self.auth_mock.call_count, 1) 142 self.assertEqual(self.init_mock.call_count, 1) 143 self.assertEqual(self.compare_mock.call_count, 1) 144 self.assertEqual(self.diff_mock.call_count, 1) 145 146 def test_compareInexactMatching(self) -> None: 147 self.auth_mock.return_value = (0, None) 148 self.init_mock.return_value = (0, None) 149 self.compare_mock.return_value = (0, None) 150 self.diff_mock.return_value = (0, None) 151 args = createSkiaGoldArgs(local_pixel_tests=False) 152 sgp = skia_gold_properties.SkiaGoldProperties(args) 153 session = skia_gold_session.SkiaGoldSession(self._working_dir, sgp, 154 self._json_keys, '', '') 155 status, _ = session.RunComparison('', 156 '', 157 None, 158 inexact_matching_args=['--inexact']) 159 self.assertEqual(status, 160 skia_gold_session.SkiaGoldSession.StatusCodes.SUCCESS) 161 self.assertEqual(self.auth_mock.call_count, 1) 162 self.assertEqual(self.init_mock.call_count, 1) 163 self.assertEqual(self.compare_mock.call_count, 1) 164 self.assertEqual(self.diff_mock.call_count, 0) 165 self.compare_mock.assert_called_with(name='', 166 png_file=mock.ANY, 167 inexact_matching_args=['--inexact'], 168 optional_keys=None, 169 force_dryrun=False) 170 171 def test_compareOptionalKeys(self) -> None: 172 self.auth_mock.return_value = (0, None) 173 self.init_mock.return_value = (0, None) 174 self.compare_mock.return_value = (0, None) 175 self.diff_mock.return_value = (0, None) 176 args = createSkiaGoldArgs(local_pixel_tests=False) 177 sgp = skia_gold_properties.SkiaGoldProperties(args) 178 session = skia_gold_session.SkiaGoldSession(self._working_dir, sgp, 179 self._json_keys, '', '') 180 status, _ = session.RunComparison('', 181 '', 182 None, 183 optional_keys={'foo': 'bar'}) 184 self.assertEqual(status, 185 skia_gold_session.SkiaGoldSession.StatusCodes.SUCCESS) 186 self.assertEqual(self.auth_mock.call_count, 1) 187 self.assertEqual(self.init_mock.call_count, 1) 188 self.assertEqual(self.compare_mock.call_count, 1) 189 self.assertEqual(self.diff_mock.call_count, 0) 190 self.compare_mock.assert_called_with(name='', 191 png_file=mock.ANY, 192 inexact_matching_args=None, 193 optional_keys={'foo': 'bar'}, 194 force_dryrun=False) 195 196 def test_compareForceDryrun(self) -> None: 197 self.auth_mock.return_value = (0, None) 198 self.init_mock.return_value = (0, None) 199 self.compare_mock.return_value = (0, None) 200 self.diff_mock.return_value = (0, None) 201 args = createSkiaGoldArgs(local_pixel_tests=False) 202 sgp = skia_gold_properties.SkiaGoldProperties(args) 203 session = skia_gold_session.SkiaGoldSession(self._working_dir, sgp, 204 self._json_keys, '', '') 205 status, _ = session.RunComparison('', '', None, force_dryrun=True) 206 self.assertEqual(status, 207 skia_gold_session.SkiaGoldSession.StatusCodes.SUCCESS) 208 self.assertEqual(self.auth_mock.call_count, 1) 209 self.assertEqual(self.init_mock.call_count, 1) 210 self.assertEqual(self.compare_mock.call_count, 1) 211 self.assertEqual(self.diff_mock.call_count, 0) 212 self.compare_mock.assert_called_with(name='', 213 png_file=mock.ANY, 214 inexact_matching_args=None, 215 optional_keys=None, 216 force_dryrun=True) 217 218 def test_diffFailure(self) -> None: 219 self.auth_mock.return_value = (0, None) 220 self.init_mock.return_value = (0, None) 221 self.compare_mock.return_value = (1, 'Compare failed') 222 self.diff_mock.return_value = (1, 'Diff failed') 223 args = createSkiaGoldArgs(local_pixel_tests=True) 224 sgp = skia_gold_properties.SkiaGoldProperties(args) 225 session = skia_gold_session.SkiaGoldSession(self._working_dir, sgp, 226 self._json_keys, '', '') 227 status, error = session.RunComparison('', '', 228 'Definitely an output manager') 229 self.assertEqual( 230 status, 231 skia_gold_session.SkiaGoldSession.StatusCodes.LOCAL_DIFF_FAILURE) 232 self.assertEqual(error, 'Diff failed') 233 self.assertEqual(self.auth_mock.call_count, 1) 234 self.assertEqual(self.init_mock.call_count, 1) 235 self.assertEqual(self.compare_mock.call_count, 1) 236 self.assertEqual(self.diff_mock.call_count, 1) 237 238 def test_noOutputManagerLocal(self) -> None: 239 self.auth_mock.return_value = (0, None) 240 self.init_mock.return_value = (0, None) 241 self.compare_mock.return_value = (1, 'Compare failed') 242 self.diff_mock.return_value = (0, None) 243 args = createSkiaGoldArgs(local_pixel_tests=True) 244 sgp = skia_gold_properties.SkiaGoldProperties(args) 245 session = skia_gold_session.SkiaGoldSession(self._working_dir, sgp, 246 self._json_keys, '', '') 247 status, error = session.RunComparison('', '', None) 248 self.assertEqual( 249 status, skia_gold_session.SkiaGoldSession.StatusCodes.NO_OUTPUT_MANAGER) 250 self.assertEqual(error, 'No output manager for local diff images') 251 self.assertEqual(self.auth_mock.call_count, 1) 252 self.assertEqual(self.compare_mock.call_count, 1) 253 self.assertEqual(self.diff_mock.call_count, 0) 254 255 256 class SkiaGoldSessionAuthenticateTest(fake_filesystem_unittest.TestCase): 257 """Tests the functionality of SkiaGoldSession.Authenticate.""" 258 259 def setUp(self) -> None: 260 self.setUpPyfakefs() 261 self._working_dir = tempfile.mkdtemp() 262 self._json_keys = tempfile.NamedTemporaryFile(delete=False).name 263 264 self.cmd_patcher = mock.patch.object(skia_gold_session.SkiaGoldSession, 265 '_RunCmdForRcAndOutput') 266 self.cmd_mock = self.cmd_patcher.start() 267 self.addCleanup(self.cmd_patcher.stop) 268 269 def test_commandOutputReturned(self) -> None: 270 self.cmd_mock.return_value = (1, 'Something bad :(') 271 args = createSkiaGoldArgs(git_revision='a') 272 sgp = skia_gold_properties.SkiaGoldProperties(args) 273 session = skia_gold_session.SkiaGoldSession(self._working_dir, sgp, 274 self._json_keys, '', '') 275 rc, stdout = session.Authenticate() 276 self.assertEqual(self.cmd_mock.call_count, 1) 277 self.assertEqual(rc, 1) 278 self.assertEqual(stdout, 'Something bad :(') 279 280 def test_bypassSkiaGoldFunctionality(self) -> None: 281 self.cmd_mock.return_value = (None, None) 282 args = createSkiaGoldArgs(git_revision='a', 283 bypass_skia_gold_functionality=True) 284 sgp = skia_gold_properties.SkiaGoldProperties(args) 285 session = skia_gold_session.SkiaGoldSession(self._working_dir, sgp, 286 self._json_keys, '', '') 287 rc, _ = session.Authenticate() 288 self.assertEqual(rc, 0) 289 self.cmd_mock.assert_not_called() 290 291 def test_shortCircuitAlreadyAuthenticated(self) -> None: 292 self.cmd_mock.return_value = (None, None) 293 args = createSkiaGoldArgs(git_revision='a') 294 sgp = skia_gold_properties.SkiaGoldProperties(args) 295 session = skia_gold_session.SkiaGoldSession(self._working_dir, sgp, 296 self._json_keys, '', '') 297 session._authenticated = True 298 rc, _ = session.Authenticate() 299 self.assertEqual(rc, 0) 300 self.cmd_mock.assert_not_called() 301 302 def test_successSetsShortCircuit(self) -> None: 303 self.cmd_mock.return_value = (0, None) 304 args = createSkiaGoldArgs(git_revision='a') 305 sgp = skia_gold_properties.SkiaGoldProperties(args) 306 session = skia_gold_session.SkiaGoldSession(self._working_dir, sgp, 307 self._json_keys, '', '') 308 self.assertFalse(session._authenticated) 309 rc, _ = session.Authenticate() 310 self.assertEqual(rc, 0) 311 self.assertTrue(session._authenticated) 312 self.cmd_mock.assert_called_once() 313 314 def test_failureDoesNotSetShortCircuit(self) -> None: 315 self.cmd_mock.return_value = (1, None) 316 args = createSkiaGoldArgs(git_revision='a') 317 sgp = skia_gold_properties.SkiaGoldProperties(args) 318 session = skia_gold_session.SkiaGoldSession(self._working_dir, sgp, 319 self._json_keys, '', '') 320 self.assertFalse(session._authenticated) 321 rc, _ = session.Authenticate() 322 self.assertEqual(rc, 1) 323 self.assertFalse(session._authenticated) 324 self.cmd_mock.assert_called_once() 325 326 def test_commandWithUseLuciTrue(self) -> None: 327 self.cmd_mock.return_value = (None, None) 328 args = createSkiaGoldArgs(git_revision='a') 329 sgp = skia_gold_properties.SkiaGoldProperties(args) 330 session = skia_gold_session.SkiaGoldSession(self._working_dir, sgp, 331 self._json_keys, '', '') 332 session.Authenticate(use_luci=True) 333 self.assertIn('--luci', self.cmd_mock.call_args[0][0]) 334 335 def test_commandWithUseLuciFalse(self) -> None: 336 self.cmd_mock.return_value = (None, None) 337 args = createSkiaGoldArgs(git_revision='a', local_pixel_tests=True) 338 sgp = skia_gold_properties.SkiaGoldProperties(args) 339 session = skia_gold_session.SkiaGoldSession(self._working_dir, sgp, 340 self._json_keys, '', '') 341 session.Authenticate(use_luci=False) 342 self.assertNotIn('--luci', self.cmd_mock.call_args[0][0]) 343 344 def test_commandWithUseLuciFalseNotLocal(self) -> None: 345 self.cmd_mock.return_value = (None, None) 346 args = createSkiaGoldArgs(git_revision='a', local_pixel_tests=False) 347 sgp = skia_gold_properties.SkiaGoldProperties(args) 348 session = skia_gold_session.SkiaGoldSession(self._working_dir, sgp, 349 self._json_keys, '', '') 350 with self.assertRaises(RuntimeError): 351 session.Authenticate(use_luci=False) 352 353 def test_commandWithUseLuciAndServiceAccount(self) -> None: 354 args = createSkiaGoldArgs(git_revision='a', local_pixel_tests=False) 355 sgp = skia_gold_properties.SkiaGoldProperties(args) 356 session = skia_gold_session.SkiaGoldSession(self._working_dir, sgp, 357 self._json_keys, '', '') 358 with self.assertRaises(AssertionError): 359 session.Authenticate(use_luci=True, service_account='a') 360 361 def test_commandWithServiceAccount(self) -> None: 362 self.cmd_mock.return_value = (None, None) 363 args = createSkiaGoldArgs(git_revision='a', local_pixel_tests=False) 364 sgp = skia_gold_properties.SkiaGoldProperties(args) 365 session = skia_gold_session.SkiaGoldSession(self._working_dir, sgp, 366 self._json_keys, '', '') 367 session.Authenticate(use_luci=False, service_account='service_account') 368 call_args = self.cmd_mock.call_args[0][0] 369 self.assertNotIn('--luci', call_args) 370 assertArgWith(self, call_args, '--service-account', 'service_account') 371 372 def test_commandCommonArgs(self) -> None: 373 self.cmd_mock.return_value = (None, None) 374 args = createSkiaGoldArgs(git_revision='a') 375 sgp = skia_gold_properties.SkiaGoldProperties(args) 376 session = skia_gold_session.SkiaGoldSession(self._working_dir, sgp, 377 self._json_keys, '', '') 378 session.Authenticate() 379 call_args = self.cmd_mock.call_args[0][0] 380 self.assertIn('auth', call_args) 381 assertArgWith(self, call_args, '--work-dir', self._working_dir) 382 383 384 class SkiaGoldSessionInitializeTest(fake_filesystem_unittest.TestCase): 385 """Tests the functionality of SkiaGoldSession.Initialize.""" 386 387 def setUp(self) -> None: 388 self.setUpPyfakefs() 389 self._working_dir = tempfile.mkdtemp() 390 self._json_keys = tempfile.NamedTemporaryFile(delete=False).name 391 392 self.cmd_patcher = mock.patch.object(skia_gold_session.SkiaGoldSession, 393 '_RunCmdForRcAndOutput') 394 self.cmd_mock = self.cmd_patcher.start() 395 self.addCleanup(self.cmd_patcher.stop) 396 397 def test_bypassSkiaGoldFunctionality(self) -> None: 398 self.cmd_mock.return_value = (None, None) 399 args = createSkiaGoldArgs(git_revision='a', 400 bypass_skia_gold_functionality=True) 401 sgp = skia_gold_properties.SkiaGoldProperties(args) 402 session = skia_gold_session.SkiaGoldSession(self._working_dir, sgp, 403 self._json_keys, '', '') 404 rc, _ = session.Initialize() 405 self.assertEqual(rc, 0) 406 self.cmd_mock.assert_not_called() 407 408 def test_shortCircuitAlreadyInitialized(self) -> None: 409 self.cmd_mock.return_value = (None, None) 410 args = createSkiaGoldArgs(git_revision='a') 411 sgp = skia_gold_properties.SkiaGoldProperties(args) 412 session = skia_gold_session.SkiaGoldSession(self._working_dir, sgp, 413 self._json_keys, '', '') 414 session._initialized = True 415 rc, _ = session.Initialize() 416 self.assertEqual(rc, 0) 417 self.cmd_mock.assert_not_called() 418 419 def test_successSetsShortCircuit(self) -> None: 420 self.cmd_mock.return_value = (0, None) 421 args = createSkiaGoldArgs(git_revision='a') 422 sgp = skia_gold_properties.SkiaGoldProperties(args) 423 session = skia_gold_session.SkiaGoldSession(self._working_dir, sgp, 424 self._json_keys, '', '') 425 self.assertFalse(session._initialized) 426 rc, _ = session.Initialize() 427 self.assertEqual(rc, 0) 428 self.assertTrue(session._initialized) 429 self.cmd_mock.assert_called_once() 430 431 def test_failureDoesNotSetShortCircuit(self) -> None: 432 self.cmd_mock.return_value = (1, None) 433 args = createSkiaGoldArgs(git_revision='a') 434 sgp = skia_gold_properties.SkiaGoldProperties(args) 435 session = skia_gold_session.SkiaGoldSession(self._working_dir, sgp, 436 self._json_keys, '', '') 437 self.assertFalse(session._initialized) 438 rc, _ = session.Initialize() 439 self.assertEqual(rc, 1) 440 self.assertFalse(session._initialized) 441 self.cmd_mock.assert_called_once() 442 443 def test_commandCommonArgs(self) -> None: 444 self.cmd_mock.return_value = (None, None) 445 args = createSkiaGoldArgs(git_revision='a') 446 sgp = skia_gold_properties.SkiaGoldProperties(args) 447 session = skia_gold_session.SkiaGoldSession(self._working_dir, 448 sgp, 449 self._json_keys, 450 'corpus', 451 instance='instance', 452 bucket='bucket') 453 session.Initialize() 454 call_args = self.cmd_mock.call_args[0][0] 455 self.assertIn('imgtest', call_args) 456 self.assertIn('init', call_args) 457 self.assertIn('--passfail', call_args) 458 assertArgWith(self, call_args, '--instance', 'instance') 459 assertArgWith(self, call_args, '--bucket', 'bucket') 460 assertArgWith(self, call_args, '--corpus', 'corpus') 461 # The keys file should have been copied to the working directory. 462 assertArgWith(self, call_args, '--keys-file', 463 os.path.join(self._working_dir, 'gold_keys.json')) 464 assertArgWith(self, call_args, '--work-dir', self._working_dir) 465 assertArgWith(self, call_args, '--failure-file', session._triage_link_file) 466 assertArgWith(self, call_args, '--commit', 'a') 467 468 def test_commandTryjobArgs(self) -> None: 469 self.cmd_mock.return_value = (None, None) 470 args = createSkiaGoldArgs(git_revision='a', 471 gerrit_issue=1, 472 gerrit_patchset=2, 473 buildbucket_id=3) 474 sgp = skia_gold_properties.SkiaGoldProperties(args) 475 session = skia_gold_session.SkiaGoldSession(self._working_dir, sgp, 476 self._json_keys, '', '') 477 session.Initialize() 478 call_args = self.cmd_mock.call_args[0][0] 479 assertArgWith(self, call_args, '--issue', '1') 480 assertArgWith(self, call_args, '--patchset', '2') 481 assertArgWith(self, call_args, '--jobid', '3') 482 assertArgWith(self, call_args, '--crs', 'gerrit') 483 assertArgWith(self, call_args, '--cis', 'buildbucket') 484 485 def test_commandTryjobArgsNonDefaultCrs(self) -> None: 486 self.cmd_mock.return_value = (None, None) 487 args = createSkiaGoldArgs(code_review_system='foo', 488 git_revision='a', 489 gerrit_issue=1, 490 gerrit_patchset=2, 491 buildbucket_id=3) 492 sgp = skia_gold_properties.SkiaGoldProperties(args) 493 session = skia_gold_session.SkiaGoldSession(self._working_dir, sgp, 494 self._json_keys, '', '') 495 session.Initialize() 496 call_args = self.cmd_mock.call_args[0][0] 497 assertArgWith(self, call_args, '--issue', '1') 498 assertArgWith(self, call_args, '--patchset', '2') 499 assertArgWith(self, call_args, '--jobid', '3') 500 assertArgWith(self, call_args, '--crs', 'foo') 501 assertArgWith(self, call_args, '--cis', 'buildbucket') 502 503 def test_commandTryjobArgsMissing(self) -> None: 504 self.cmd_mock.return_value = (None, None) 505 args = createSkiaGoldArgs(git_revision='a') 506 sgp = skia_gold_properties.SkiaGoldProperties(args) 507 session = skia_gold_session.SkiaGoldSession(self._working_dir, sgp, 508 self._json_keys, '', '') 509 session.Initialize() 510 call_args = self.cmd_mock.call_args[0][0] 511 self.assertNotIn('--issue', call_args) 512 self.assertNotIn('--patchset', call_args) 513 self.assertNotIn('--jobid', call_args) 514 self.assertNotIn('--crs', call_args) 515 self.assertNotIn('--cis', call_args) 516 517 518 class SkiaGoldSessionCompareTest(fake_filesystem_unittest.TestCase): 519 """Tests the functionality of SkiaGoldSession.Compare.""" 520 521 def setUp(self) -> None: 522 self.setUpPyfakefs() 523 self._working_dir = tempfile.mkdtemp() 524 self._json_keys = tempfile.NamedTemporaryFile(delete=False).name 525 526 self.cmd_patcher = mock.patch.object(skia_gold_session.SkiaGoldSession, 527 '_RunCmdForRcAndOutput') 528 self.cmd_mock = self.cmd_patcher.start() 529 self.addCleanup(self.cmd_patcher.stop) 530 531 def test_commandOutputReturned(self) -> None: 532 self.cmd_mock.return_value = (1, 'Something bad :(') 533 args = createSkiaGoldArgs(git_revision='a') 534 sgp = skia_gold_properties.SkiaGoldProperties(args) 535 session = skia_gold_session.SkiaGoldSession(self._working_dir, sgp, 536 self._json_keys, '', '') 537 rc, stdout = session.Compare('', '') 538 self.assertEqual(self.cmd_mock.call_count, 1) 539 self.assertEqual(rc, 1) 540 self.assertEqual(stdout, 'Something bad :(') 541 542 def test_bypassSkiaGoldFunctionality(self) -> None: 543 self.cmd_mock.return_value = (None, None) 544 args = createSkiaGoldArgs(git_revision='a', 545 bypass_skia_gold_functionality=True) 546 sgp = skia_gold_properties.SkiaGoldProperties(args) 547 session = skia_gold_session.SkiaGoldSession(self._working_dir, sgp, 548 self._json_keys, '', '') 549 rc, _ = session.Compare('', '') 550 self.assertEqual(rc, 0) 551 self.cmd_mock.assert_not_called() 552 553 def test_commandWithLocalPixelTestsTrue(self) -> None: 554 self.cmd_mock.return_value = (None, None) 555 args = createSkiaGoldArgs(git_revision='a', local_pixel_tests=True) 556 sgp = skia_gold_properties.SkiaGoldProperties(args) 557 session = skia_gold_session.SkiaGoldSession(self._working_dir, sgp, 558 self._json_keys, '', '') 559 session.Compare('', '') 560 self.assertIn('--dryrun', self.cmd_mock.call_args[0][0]) 561 562 def test_commandWithForceDryrunTrue(self) -> None: 563 self.cmd_mock.return_value = (None, None) 564 args = createSkiaGoldArgs(git_revision='a') 565 sgp = skia_gold_properties.SkiaGoldProperties(args) 566 session = skia_gold_session.SkiaGoldSession(self._working_dir, sgp, 567 self._json_keys, '', '') 568 session.Compare('', '', force_dryrun=True) 569 self.assertIn('--dryrun', self.cmd_mock.call_args[0][0]) 570 571 def test_commandWithLocalPixelTestsFalse(self) -> None: 572 self.cmd_mock.return_value = (None, None) 573 args = createSkiaGoldArgs(git_revision='a', local_pixel_tests=False) 574 sgp = skia_gold_properties.SkiaGoldProperties(args) 575 session = skia_gold_session.SkiaGoldSession(self._working_dir, sgp, 576 self._json_keys, '', '') 577 session.Compare('', '') 578 self.assertNotIn('--dryrun', self.cmd_mock.call_args[0][0]) 579 580 def test_commandWithInexactArgs(self) -> None: 581 self.cmd_mock.return_value = (None, None) 582 args = createSkiaGoldArgs(git_revision='a') 583 sgp = skia_gold_properties.SkiaGoldProperties(args) 584 session = skia_gold_session.SkiaGoldSession(self._working_dir, sgp, 585 self._json_keys, '', '') 586 session.Compare('', '', inexact_matching_args=['--inexact', 'foobar']) 587 self.assertIn('--inexact', self.cmd_mock.call_args[0][0]) 588 self.assertIn('foobar', self.cmd_mock.call_args[0][0]) 589 590 def test_commandCommonArgs(self) -> None: 591 self.cmd_mock.return_value = (None, None) 592 args = createSkiaGoldArgs(git_revision='a') 593 sgp = skia_gold_properties.SkiaGoldProperties(args) 594 session = skia_gold_session.SkiaGoldSession(self._working_dir, 595 sgp, 596 self._json_keys, 597 'corpus', 598 instance='instance') 599 session.Compare('name', 'png_file') 600 call_args = self.cmd_mock.call_args[0][0] 601 self.assertIn('imgtest', call_args) 602 self.assertIn('add', call_args) 603 assertArgWith(self, call_args, '--test-name', 'name') 604 assertArgWith(self, call_args, '--png-file', 'png_file') 605 assertArgWith(self, call_args, '--work-dir', self._working_dir) 606 607 def test_noLinkOnSuccess(self) -> None: 608 self.cmd_mock.return_value = (0, None) 609 args = createSkiaGoldArgs(git_revision='a') 610 sgp = skia_gold_properties.SkiaGoldProperties(args) 611 session = skia_gold_session.SkiaGoldSession(self._working_dir, sgp, 612 self._json_keys, '', '') 613 rc, _ = session.Compare('name', 'png_file') 614 self.assertEqual(rc, 0) 615 comparison_result = session._comparison_results['name'] 616 self.assertEqual(comparison_result.public_triage_link, None) 617 self.assertEqual(comparison_result.internal_triage_link, None) 618 self.assertNotEqual(comparison_result.triage_link_omission_reason, None) 619 620 def test_clLinkOnTrybot(self) -> None: 621 self.cmd_mock.return_value = (1, None) 622 args = createSkiaGoldArgs(git_revision='a', 623 gerrit_issue=1, 624 gerrit_patchset=2, 625 buildbucket_id=3) 626 sgp = skia_gold_properties.SkiaGoldProperties(args) 627 session = skia_gold_session.SkiaGoldSession(self._working_dir, 628 sgp, 629 self._json_keys, 630 '', 631 instance='instance') 632 rc, _ = session.Compare('name', 'png_file') 633 self.assertEqual(rc, 1) 634 comparison_result = session._comparison_results['name'] 635 self.assertNotEqual(comparison_result.public_triage_link, None) 636 self.assertNotEqual(comparison_result.internal_triage_link, None) 637 internal_link = 'https://instance-gold.skia.org/cl/gerrit/1' 638 public_link = 'https://instance-public-gold.skia.org/cl/gerrit/1' 639 self.assertEqual(comparison_result.internal_triage_link, internal_link) 640 self.assertEqual(comparison_result.public_triage_link, public_link) 641 self.assertEqual(comparison_result.triage_link_omission_reason, None) 642 self.assertEqual(session.GetTriageLinks('name'), 643 (public_link, internal_link)) 644 645 def test_individualLinkOnCi(self) -> None: 646 args = createSkiaGoldArgs(git_revision='a') 647 sgp = skia_gold_properties.SkiaGoldProperties(args) 648 session = skia_gold_session.SkiaGoldSession(self._working_dir, 649 sgp, 650 self._json_keys, 651 '', 652 instance='foobar') 653 654 internal_link = 'foobar-gold.skia.org' 655 public_link = 'foobar-public-gold.skia.org' 656 657 def WriteTriageLinkFile(_): 658 with open(session._triage_link_file, 'w') as f: 659 f.write(internal_link) 660 return (1, None) 661 662 self.cmd_mock.side_effect = WriteTriageLinkFile 663 rc, _ = session.Compare('name', 'png_file') 664 self.assertEqual(rc, 1) 665 comparison_result = session._comparison_results['name'] 666 self.assertNotEqual(comparison_result.public_triage_link, None) 667 self.assertNotEqual(comparison_result.internal_triage_link, None) 668 self.assertEqual(comparison_result.internal_triage_link, internal_link) 669 self.assertEqual(comparison_result.public_triage_link, public_link) 670 self.assertEqual(comparison_result.triage_link_omission_reason, None) 671 self.assertEqual(session.GetTriageLinks('name'), 672 (public_link, internal_link)) 673 674 def test_validOmissionOnMissingLink(self) -> None: 675 args = createSkiaGoldArgs(git_revision='a') 676 sgp = skia_gold_properties.SkiaGoldProperties(args) 677 session = skia_gold_session.SkiaGoldSession(self._working_dir, sgp, 678 self._json_keys, '', '') 679 680 def WriteTriageLinkFile(_): 681 with open(session._triage_link_file, 'w'): 682 pass 683 return (1, None) 684 685 self.cmd_mock.side_effect = WriteTriageLinkFile 686 rc, _ = session.Compare('name', 'png_file') 687 self.assertEqual(rc, 1) 688 comparison_result = session._comparison_results['name'] 689 self.assertEqual(comparison_result.public_triage_link, None) 690 self.assertEqual(comparison_result.internal_triage_link, None) 691 self.assertIn('Gold did not provide a triage link', 692 comparison_result.triage_link_omission_reason) 693 694 def test_validOmissionOnIoError(self) -> None: 695 self.cmd_mock.return_value = (1, None) 696 args = createSkiaGoldArgs(git_revision='a') 697 sgp = skia_gold_properties.SkiaGoldProperties(args) 698 session = skia_gold_session.SkiaGoldSession(self._working_dir, sgp, 699 self._json_keys, '', '') 700 701 def DeleteTriageLinkFile(_): 702 os.remove(session._triage_link_file) 703 return (1, None) 704 705 self.cmd_mock.side_effect = DeleteTriageLinkFile 706 rc, _ = session.Compare('name', 'png_file') 707 self.assertEqual(rc, 1) 708 comparison_result = session._comparison_results['name'] 709 self.assertEqual(comparison_result.public_triage_link, None) 710 self.assertEqual(comparison_result.internal_triage_link, None) 711 self.assertNotEqual(comparison_result.triage_link_omission_reason, None) 712 self.assertIn('Failed to read', 713 comparison_result.triage_link_omission_reason) 714 715 def test_optionalKeysPassedToGoldctl(self) -> None: 716 self.cmd_mock.return_value = (None, None) 717 args = createSkiaGoldArgs(git_revision='a', local_pixel_tests=True) 718 sgp = skia_gold_properties.SkiaGoldProperties(args) 719 session = skia_gold_session.SkiaGoldSession(self._working_dir, sgp, 720 self._json_keys, '', '') 721 session.Compare('', '', optional_keys={'foo': 'bar'}) 722 assertArgWith(self, self.cmd_mock.call_args[0][0], 723 '--add-test-optional-key', 'foo:bar') 724 725 726 class SkiaGoldSessionDiffTest(fake_filesystem_unittest.TestCase): 727 """Tests the functionality of SkiaGoldSession.Diff.""" 728 729 def setUp(self) -> None: 730 self.setUpPyfakefs() 731 self._working_dir = tempfile.mkdtemp() 732 self._json_keys = tempfile.NamedTemporaryFile(delete=False).name 733 734 self.cmd_patcher = mock.patch.object(skia_gold_session.SkiaGoldSession, 735 '_RunCmdForRcAndOutput') 736 self.cmd_mock = self.cmd_patcher.start() 737 self.addCleanup(self.cmd_patcher.stop) 738 739 @mock.patch.object(skia_gold_session.SkiaGoldSession, '_StoreDiffLinks') 740 def test_commandOutputReturned(self, _) -> None: 741 self.cmd_mock.return_value = (1, 'Something bad :(') 742 args = createSkiaGoldArgs(git_revision='a', local_pixel_tests=False) 743 sgp = skia_gold_properties.SkiaGoldProperties(args) 744 session = skia_gold_session.SkiaGoldSession(self._working_dir, sgp, 745 self._json_keys, '', '') 746 rc, stdout = session.Diff('', '', None) 747 self.assertEqual(self.cmd_mock.call_count, 1) 748 self.assertEqual(rc, 1) 749 self.assertEqual(stdout, 'Something bad :(') 750 751 def test_bypassSkiaGoldFunctionality(self) -> None: 752 self.cmd_mock.return_value = (None, None) 753 args = createSkiaGoldArgs(git_revision='a', 754 bypass_skia_gold_functionality=True) 755 sgp = skia_gold_properties.SkiaGoldProperties(args) 756 session = skia_gold_session.SkiaGoldSession(self._working_dir, sgp, 757 self._json_keys, '', '') 758 with self.assertRaises(RuntimeError): 759 session.Diff('', '', None) 760 761 762 class SkiaGoldSessionTriageLinkOmissionTest(fake_filesystem_unittest.TestCase): 763 """Tests the functionality of SkiaGoldSession.GetTriageLinkOmissionReason.""" 764 765 def setUp(self) -> None: 766 self.setUpPyfakefs() 767 self._working_dir = tempfile.mkdtemp() 768 769 def _CreateSession(self) -> skia_gold_session.SkiaGoldSession: 770 sgp = skia_gold_properties.SkiaGoldProperties(createSkiaGoldArgs()) 771 json_keys = tempfile.NamedTemporaryFile(delete=False).name 772 session = skia_gold_session.SkiaGoldSession(self._working_dir, sgp, 773 json_keys, '', '') 774 session._comparison_results = { 775 'foo': skia_gold_session.SkiaGoldSession.ComparisonResults(), 776 } 777 return session 778 779 def test_noComparison(self) -> None: 780 session = self._CreateSession() 781 session._comparison_results = {} 782 reason = session.GetTriageLinkOmissionReason('foo') 783 self.assertEqual(reason, 'No image comparison performed for foo') 784 785 def test_validReason(self) -> None: 786 session = self._CreateSession() 787 session._comparison_results['foo'].triage_link_omission_reason = 'bar' 788 reason = session.GetTriageLinkOmissionReason('foo') 789 self.assertEqual(reason, 'bar') 790 791 def test_onlyLocal(self) -> None: 792 session = self._CreateSession() 793 session._comparison_results['foo'].local_diff_given_image = 'bar' 794 reason = session.GetTriageLinkOmissionReason('foo') 795 self.assertEqual(reason, 'Gold only used to do a local image diff') 796 797 def test_onlyWithoutTriageLink(self) -> None: 798 session = self._CreateSession() 799 comparison_result = session._comparison_results['foo'] 800 comparison_result.public_triage_link = 'bar' 801 with self.assertRaises(AssertionError): 802 session.GetTriageLinkOmissionReason('foo') 803 comparison_result.public_triage_link = None 804 comparison_result.internal_triage_link = 'bar' 805 with self.assertRaises(AssertionError): 806 session.GetTriageLinkOmissionReason('foo') 807 808 def test_resultsShouldNotExist(self) -> None: 809 session = self._CreateSession() 810 with self.assertRaises(RuntimeError): 811 session.GetTriageLinkOmissionReason('foo') 812 813 814 if __name__ == '__main__': 815 unittest.main(verbosity=2)