basic_tests.py (24601B)
1 #!/usr/bin/env python3 2 # This Source Code Form is subject to the terms of the Mozilla Public 3 # License, v. 2.0. If a copy of the MPL was not distributed with this 4 # file, You can obtain one at https://mozilla.org/MPL/2.0/. 5 6 7 import base64 8 import io 9 import json 10 import os 11 import subprocess 12 import sys 13 import tempfile 14 import time 15 import traceback 16 17 from mozlog import formatters, handlers, structuredlog 18 from PIL import Image, ImageChops, ImageDraw 19 from pixelmatch.contrib.PIL import pixelmatch 20 from selenium import webdriver 21 from selenium.common.exceptions import TimeoutException, WebDriverException 22 from selenium.webdriver.common.by import By 23 from selenium.webdriver.firefox.options import Options 24 from selenium.webdriver.firefox.service import Service 25 from selenium.webdriver.remote.webdriver import WebDriver 26 from selenium.webdriver.remote.webelement import WebElement 27 from selenium.webdriver.support import expected_conditions as EC 28 from selenium.webdriver.support.ui import WebDriverWait 29 30 31 class SnapTestsBase: 32 _INSTANCE = None 33 34 def __init__(self, exp): 35 self._INSTANCE = os.environ.get("TEST_SNAP_INSTANCE") 36 37 self._PROFILE_PATH = f"~/snap/{self._INSTANCE}/common/.mozilla/firefox/" 38 self._LIB_PATH = rf"/snap/{self._INSTANCE}/current/usr/lib/firefox/libxul.so" 39 # This needs to be the snap-based symlink geckodriver to properly setup 40 # the Snap environment 41 self._EXE_PATH = rf"/snap/bin/{self._INSTANCE}.geckodriver" 42 43 # This needs to be the full path to the binary because at the moment of 44 # its execution it will already be under the Snap environment, and 45 # running "snap" command in this context will fail with Permission denied 46 # 47 # This can be trivially verified by the following shell script being used 48 # as binary_location/_BIN_PATH below: 49 # 50 # 8<-8<-8<-8<-8<-8<-8<-8<-8<-8<-8<-8<- 51 # #!/bin/sh 52 # TMPDIR=${TMPDIR:-/tmp} 53 # env > $TMPDIR/snap-test.txt 54 # 8<-8<-8<-8<-8<-8<-8<-8<-8<-8<-8<-8<- 55 # 56 # One should see output generated in the instance-specific temp dir 57 # /tmp/snap-private-tmp/snap.{}/tmp/snap-test.txt 58 # denoting that everything properly runs under Snap as expected. 59 self._BIN_PATH = rf"/snap/{self._INSTANCE}/current/usr/lib/firefox/firefox" 60 61 snap_profile_path = tempfile.mkdtemp( 62 prefix="snap-tests", 63 dir=os.path.expanduser(self._PROFILE_PATH), 64 ) 65 66 driver_service_args = [] 67 if self.need_allow_system_access(): 68 driver_service_args += ["--allow-system-access"] 69 70 driver_service = Service( 71 executable_path=self._EXE_PATH, 72 log_output=os.path.join( 73 os.environ.get("ARTIFACT_DIR", ""), "geckodriver.log" 74 ), 75 service_args=driver_service_args, 76 ) 77 78 options = Options() 79 if "TEST_GECKODRIVER_TRACE" in os.environ.keys(): 80 options.log.level = "trace" 81 options.binary_location = self._BIN_PATH 82 if not "TEST_NO_HEADLESS" in os.environ.keys(): 83 options.add_argument("--headless") 84 if "MOZ_AUTOMATION" in os.environ.keys(): 85 os.environ["MOZ_LOG_FILE"] = os.path.join( 86 os.environ.get("ARTIFACT_DIR"), "gecko.log" 87 ) 88 options.add_argument("-profile") 89 options.add_argument(snap_profile_path) 90 self._driver = webdriver.Firefox(service=driver_service, options=options) 91 92 self._logger = structuredlog.StructuredLogger(self.__class__.__name__) 93 self._logger.add_handler( 94 handlers.StreamHandler(sys.stdout, formatters.TbplFormatter()) 95 ) 96 97 test_filter = "test_{}".format(os.environ.get("TEST_FILTER", "")) 98 object_methods = [ 99 method_name 100 for method_name in dir(self) 101 if callable(getattr(self, method_name)) 102 and method_name.startswith(test_filter) 103 ] 104 105 self._logger.suite_start(object_methods) 106 107 assert self._dir is not None 108 109 self._update_channel = None 110 self._version_major = None 111 self._snap_core_base = None 112 113 self._is_debug_build = None 114 if self.is_debug_build(): 115 self._logger.info("Running against a DEBUG build") 116 else: 117 self._logger.info("Running against a OPT build") 118 119 self._driver.maximize_window() 120 121 self._wait = WebDriverWait(self._driver, self.get_timeout()) 122 self._longwait = WebDriverWait(self._driver, 60) 123 124 with open(exp) as j: 125 self._expectations = json.load(j) 126 127 # exit code ; will be set to 1 at first assertion failure 128 ec = 0 129 first_tab = self._driver.window_handles[0] 130 channel = self.update_channel() 131 if self.is_esr_128(): 132 channel = "esr-128" 133 134 core_base = self.snap_core_base() 135 channel_and_core = f"{channel}core{core_base}" 136 self._logger.info(f"Channel & Core: {channel_and_core}") 137 138 for m in object_methods: 139 tabs_before = set() 140 tabs_after = set() 141 self._logger.test_start(m) 142 expectations = ( 143 self._expectations[m] 144 if not channel_and_core in self._expectations[m] 145 else self._expectations[m][channel_and_core] 146 ) 147 self._driver.switch_to.window(first_tab) 148 149 try: 150 tabs_before = set(self._driver.window_handles) 151 rv = getattr(self, m)(expectations) 152 assert rv is not None, "test returned no value" 153 154 tabs_after = set(self._driver.window_handles) 155 self._logger.info(f"tabs_after OK {tabs_after}") 156 157 self._driver.switch_to.parent_frame() 158 if rv: 159 self._logger.test_end(m, status="OK") 160 else: 161 self._logger.test_end(m, status="FAIL") 162 except Exception as ex: 163 ec = 1 164 test_status = "ERROR" 165 if isinstance(ex, AssertionError): 166 test_status = "FAIL" 167 elif isinstance(ex, TimeoutException): 168 test_status = "TIMEOUT" 169 170 test_message = repr(ex) 171 page_source = self._driver.page_source 172 self._logger.info(f"page source:\n-->\n{page_source}\n<--\n") 173 self.save_screenshot( 174 f"screenshot_{m.lower()}_{test_status.lower()}.png" 175 ) 176 self._driver.switch_to.parent_frame() 177 self.save_screenshot( 178 f"screenshot_{m.lower()}_{test_status.lower()}_parent.png" 179 ) 180 self._logger.test_end(m, status=test_status, message=test_message) 181 traceback.print_exc() 182 183 tabs_after = set(self._driver.window_handles) 184 self._logger.info(f"tabs_after EXCEPTION {tabs_after}") 185 finally: 186 self._logger.info(f"tabs_before {tabs_before}") 187 tabs_opened = tabs_after - tabs_before 188 self._logger.info(f"opened {len(tabs_opened)} tabs") 189 self._logger.info(f"opened {tabs_opened} tabs") 190 closed = 0 191 for tab in tabs_opened: 192 self._logger.info(f"switch to {tab}") 193 self._driver.switch_to.window(tab) 194 self._logger.info(f"close {tab}") 195 self._driver.close() 196 closed += 1 197 self._logger.info( 198 f"wait EC.number_of_windows_to_be({len(tabs_after) - closed})" 199 ) 200 self._wait.until( 201 EC.number_of_windows_to_be(len(tabs_after) - closed) 202 ) 203 204 self._driver.switch_to.window(first_tab) 205 206 if not "TEST_NO_QUIT" in os.environ.keys(): 207 self._driver.quit() 208 209 self._logger.info(f"Exiting with {ec}") 210 self._logger.suite_end() 211 sys.exit(ec) 212 213 def get_screenshot_destination(self, name): 214 final_name = name 215 if "MOZ_AUTOMATION" in os.environ.keys(): 216 final_name = os.path.join(os.environ.get("ARTIFACT_DIR"), name) 217 return final_name 218 219 def save_screenshot(self, name): 220 final_name = self.get_screenshot_destination(name) 221 self._logger.info(f"Saving screenshot '{name}' to '{final_name}'") 222 try: 223 self._driver.save_screenshot(final_name) 224 except WebDriverException as ex: 225 self._logger.info(f"Saving screenshot FAILED due to {ex}") 226 227 def get_timeout(self): 228 if "TEST_TIMEOUT" in os.environ.keys(): 229 return int(os.getenv("TEST_TIMEOUT")) 230 else: 231 return 5 232 233 def maybe_collect_reference(self): 234 return "TEST_COLLECT_REFERENCE" in os.environ.keys() 235 236 def open_tab(self, url): 237 opened_tabs = len(self._driver.window_handles) 238 239 self._driver.switch_to.new_window("tab") 240 self._wait.until(EC.number_of_windows_to_be(opened_tabs + 1)) 241 self._driver.get(url) 242 243 return self._driver.current_window_handle 244 245 def is_debug_build(self): 246 if self._is_debug_build is None: 247 self._is_debug_build = ( 248 "with debug_info" 249 in subprocess.check_output(["file", self._LIB_PATH]).decode() 250 ) 251 return self._is_debug_build 252 253 def need_allow_system_access(self): 254 geckodriver_output = subprocess.check_output([ 255 self._EXE_PATH, 256 "--help", 257 ]).decode() 258 return "--allow-system-access" in geckodriver_output 259 260 def update_channel(self): 261 if self._update_channel is None: 262 self._driver.set_context("chrome") 263 self._update_channel = self._driver.execute_script( 264 "return Services.prefs.getStringPref('app.update.channel');" 265 ) 266 self._logger.info(f"Update channel: {self._update_channel}") 267 self._driver.set_context("content") 268 return self._update_channel 269 270 def snap_core_base(self): 271 if self._snap_core_base is None: 272 self._driver.set_context("chrome") 273 self._snap_core_base = self._driver.execute_script( 274 "return Services.sysinfo.getProperty('distroVersion');" 275 ) 276 self._logger.info(f"Snap Core: {self._snap_core_base}") 277 self._driver.set_context("content") 278 return self._snap_core_base 279 280 def version(self): 281 self._driver.set_context("chrome") 282 version = self._driver.execute_script("return AppConstants.MOZ_APP_VERSION;") 283 self._driver.set_context("content") 284 return version 285 286 def version_major(self): 287 if self._version_major is None: 288 self._driver.set_context("chrome") 289 self._version_major = self._driver.execute_script( 290 "return AppConstants.MOZ_APP_VERSION.split('.')[0];" 291 ) 292 self._logger.info(f"Version major: {self._version_major}") 293 self._driver.set_context("content") 294 return self._version_major 295 296 def is_esr_128(self): 297 return self.update_channel() == "esr" and self.version_major() == "128" 298 299 def assert_rendering(self, exp, element_or_driver): 300 # wait a bit for things to settle down 301 time.sleep(0.5) 302 303 # Convert as RGB otherwise we cannot get difference 304 png_bytes = ( 305 element_or_driver.screenshot_as_png 306 if isinstance(element_or_driver, WebElement) 307 else ( 308 element_or_driver.get_screenshot_as_png() 309 if isinstance(element_or_driver, WebDriver) 310 else base64.b64decode(element_or_driver) 311 ) 312 ) 313 svg_png = Image.open(io.BytesIO(png_bytes)).convert("RGB") 314 svg_png_cropped = svg_png.crop((0, 35, svg_png.width - 20, svg_png.height - 10)) 315 316 if self.maybe_collect_reference(): 317 new_ref = "new_{}".format(exp["reference"]) 318 new_ref_file = self.get_screenshot_destination(new_ref) 319 self._logger.info( 320 f"Collecting new reference screenshot: {new_ref} => {new_ref_file}" 321 ) 322 323 with open(new_ref_file, "wb") as current_screenshot: 324 svg_png_cropped.save(current_screenshot) 325 326 return 327 328 svg_ref = Image.open(os.path.join(self._dir, exp["reference"])).convert("RGB") 329 diff = ImageChops.difference(svg_ref, svg_png_cropped) 330 331 bbox = diff.getbbox() 332 333 mismatch = 0 334 try: 335 mismatch = pixelmatch( 336 svg_png_cropped, svg_ref, diff, includeAA=False, threshold=0.15 337 ) 338 if mismatch == 0: 339 return 340 self._logger.info(f"Non empty differences from pixelmatch: {mismatch}") 341 except ValueError as ex: 342 self._logger.info(f"Problem at pixelmatch: {ex}") 343 current_rendering_png = "pixelmatch_current_rendering_{}".format( 344 exp["reference"] 345 ) 346 with open( 347 self.get_screenshot_destination(current_rendering_png), "wb" 348 ) as current_screenshot: 349 svg_png_cropped.save(current_screenshot) 350 351 reference_rendering_png = "pixelmatch_reference_rendering_{}".format( 352 exp["reference"] 353 ) 354 with open( 355 self.get_screenshot_destination(reference_rendering_png), "wb" 356 ) as current_screenshot: 357 svg_ref.save(current_screenshot) 358 359 if bbox is not None: 360 (left, upper, right, lower) = bbox 361 assert mismatch > 0, "Really mismatching" 362 363 bbox_w = right - left 364 bbox_h = lower - upper 365 366 diff_px_on_bbox = round((mismatch * 1.0 / (bbox_w * bbox_h)) * 100, 3) 367 allowance = exp["allowance"] if "allowance" in exp else 0.15 368 self._logger.info( 369 f"Bbox: {bbox_w}x{bbox_h} => {diff_px_on_bbox}% ({allowance}% allowed)" 370 ) 371 372 if diff_px_on_bbox <= allowance: 373 return 374 375 (diff_r, diff_g, diff_b) = diff.getextrema() 376 377 draw_ref = ImageDraw.Draw(svg_ref) 378 draw_ref.rectangle(bbox, outline="red") 379 380 draw_rend = ImageDraw.Draw(svg_png_cropped) 381 draw_rend.rectangle(bbox, outline="red") 382 383 draw_diff = ImageDraw.Draw(diff) 384 draw_diff.rectangle(bbox, outline="red") 385 386 # Some differences have been found, let's verify 387 self._logger.info(f"Non empty differences bbox: {bbox}") 388 389 buffered = io.BytesIO() 390 diff.save(buffered, format="PNG") 391 392 if "TEST_DUMP_DIFF" in os.environ.keys(): 393 diff_b64 = base64.b64encode(buffered.getvalue()) 394 self._logger.info( 395 "data:image/png;base64,{}".format(diff_b64.decode("utf-8")) 396 ) 397 398 differences_png = "differences_{}".format(exp["reference"]) 399 with open( 400 self.get_screenshot_destination(differences_png), "wb" 401 ) as diff_screenshot: 402 diff_screenshot.write(buffered.getvalue()) 403 404 current_rendering_png = "current_rendering_{}".format(exp["reference"]) 405 with open( 406 self.get_screenshot_destination(current_rendering_png), "wb" 407 ) as current_screenshot: 408 svg_png_cropped.save(current_screenshot) 409 410 reference_rendering_png = "reference_rendering_{}".format(exp["reference"]) 411 with open( 412 self.get_screenshot_destination(reference_rendering_png), "wb" 413 ) as current_screenshot: 414 svg_ref.save(current_screenshot) 415 416 (left, upper, right, lower) = bbox 417 assert right >= left, f"Inconsistent boundaries right={right} left={left}" 418 assert lower >= upper, ( 419 f"Inconsistent boundaries lower={lower} upper={upper}" 420 ) 421 if ((right - left) <= 2) or ((lower - upper) <= 2): 422 self._logger.info("Difference is a <= 2 pixels band, ignoring") 423 return 424 425 assert diff_px_on_bbox <= allowance, ( 426 "Mismatching screenshots for {}".format(exp["reference"]) 427 ) 428 429 430 class SnapTests(SnapTestsBase): 431 def __init__(self, exp): 432 self._dir = "basic_tests" 433 super().__init__(exp) 434 435 def test_snap_core_base(self, exp): 436 assert self.snap_core_base() in ["22", "24"], "Core base should be 22 or 24" 437 438 return True 439 440 def test_about_support(self, exp): 441 self.open_tab("about:support") 442 443 version_box = self._wait.until( 444 EC.visibility_of_element_located((By.ID, "version-box")) 445 ) 446 self._wait.until(lambda d: len(version_box.text) > 0) 447 self._logger.info(f"about:support version: {version_box.text}") 448 assert version_box.text == exp["version_box"], "version text should match" 449 450 distributionid_box = self._wait.until( 451 EC.visibility_of_element_located((By.ID, "distributionid-box")) 452 ) 453 self._wait.until(lambda d: len(distributionid_box.text) > 0) 454 self._logger.info(f"about:support distribution ID: {distributionid_box.text}") 455 assert distributionid_box.text == exp["distribution_id"], ( 456 "distribution_id should match" 457 ) 458 459 windowing_protocol = self._driver.execute_script( 460 "return document.querySelector('th[data-l10n-id=\"graphics-window-protocol\"').parentNode.lastChild.textContent;" 461 ) 462 self._logger.info(f"about:support windowing protocol: {windowing_protocol}") 463 assert windowing_protocol == "wayland", "windowing protocol should be wayland" 464 465 return True 466 467 def test_about_buildconfig(self, exp): 468 self.open_tab("about:buildconfig") 469 470 source_link = self._wait.until( 471 EC.visibility_of_element_located((By.CSS_SELECTOR, "a")) 472 ) 473 self._wait.until(lambda d: len(source_link.text) > 0) 474 self._logger.info(f"about:buildconfig source: {source_link.text}") 475 assert source_link.text.startswith(exp["source_repo"]), ( 476 "source repo should exists and match" 477 ) 478 479 build_flags_box = self._wait.until( 480 EC.visibility_of_element_located((By.CSS_SELECTOR, "p:last-child")) 481 ) 482 self._wait.until(lambda d: len(build_flags_box.text) > 0) 483 self._logger.info(f"about:support buildflags: {build_flags_box.text}") 484 assert build_flags_box.text.find(exp["official"]) >= 0, ( 485 "official build flag should be there" 486 ) 487 488 return True 489 490 def test_youtube(self, exp): 491 # Skip because unreliable 492 return True 493 494 self.open_tab("https://www.youtube.com/channel/UCYfdidRxbB8Qhf0Nx7ioOYw") 495 496 # Wait so we leave time to breathe and not be classified as a bot 497 time.sleep(5) 498 499 # Wait for the consent dialog and accept it 500 self._logger.info("Wait for consent form") 501 try: 502 self._wait.until( 503 EC.visibility_of_element_located(( 504 By.CSS_SELECTOR, 505 "button[aria-label*=Accept]", 506 )) 507 ).click() 508 except TimeoutException: 509 self._logger.info("Wait for consent form: timed out, maybe it is not here") 510 511 # Wait so we leave time to breathe and not be classified as a bot 512 time.sleep(3) 513 514 # Wait for the cable TV dialog and accept it 515 self._logger.info("Wait for cable proposal") 516 try: 517 self._wait.until( 518 EC.visibility_of_element_located(( 519 By.CSS_SELECTOR, 520 "button[aria-label*=Dismiss]", 521 )) 522 ).click() 523 except TimeoutException: 524 self._logger.info( 525 "Wait for cable proposal: timed out, maybe it is not here" 526 ) 527 528 # Wait so we leave time to breathe and not be classified as a bot 529 time.sleep(3) 530 531 # Find first video and click it 532 self._logger.info("Wait for one video") 533 self._wait.until( 534 EC.visibility_of_element_located((By.ID, "video-title-link")) 535 ).click() 536 537 # Wait so we leave time to breathe and not be classified as a bot 538 time.sleep(3) 539 540 # Wait for duration to be set to something 541 self._logger.info("Wait for video to start") 542 video = None 543 try: 544 video = self._longwait.until( 545 EC.visibility_of_element_located((By.CLASS_NAME, "html5-main-video")) 546 ) 547 self._longwait.until( 548 lambda d: type(video.get_property("duration")) is float 549 ) 550 self._logger.info( 551 "video duration: {}".format(video.get_property("duration")) 552 ) 553 assert video.get_property("duration") > exp["duration"], ( 554 "youtube video should have duration" 555 ) 556 557 self._wait.until( 558 lambda d: video.get_property("currentTime") > exp["playback"] 559 ) 560 self._logger.info( 561 "video played: {}".format(video.get_property("currentTime")) 562 ) 563 assert video.get_property("currentTime") > exp["playback"], ( 564 "youtube video should perform playback" 565 ) 566 except TimeoutException as ex: 567 self._logger.info("video detection timed out") 568 self._logger.info(f"video: {video}") 569 if video: 570 self._logger.info( 571 "video duration: {}".format(video.get_property("duration")) 572 ) 573 raise ex 574 575 return True 576 577 def wait_for_enable_drm(self): 578 rv = True 579 self._driver.set_context("chrome") 580 self._driver.execute_script( 581 "Services.prefs.setBoolPref('media.gmp-manager.updateEnabled', true);" 582 ) 583 584 enable_drm_button = self._wait.until( 585 EC.visibility_of_element_located(( 586 By.CSS_SELECTOR, 587 ".notification-button[label='Enable DRM']", 588 )) 589 ) 590 self._logger.info("Enabling DRMs") 591 enable_drm_button.click() 592 self._wait.until( 593 EC.invisibility_of_element_located(( 594 By.CSS_SELECTOR, 595 ".notification-button[label='Enable DRM']", 596 )) 597 ) 598 599 self._logger.info("Installing DRMs") 600 self._wait.until( 601 EC.visibility_of_element_located(( 602 By.CSS_SELECTOR, 603 ".infobar[value='drmContentCDMInstalling']", 604 )) 605 ) 606 607 self._logger.info("Waiting for DRMs installation to complete") 608 self._longwait.until( 609 EC.invisibility_of_element_located(( 610 By.CSS_SELECTOR, 611 ".infobar[value='drmContentCDMInstalling']", 612 )) 613 ) 614 615 self._driver.set_context("content") 616 return rv 617 618 def test_youtube_film(self, exp): 619 # Bug 1885473: require sign-in? 620 return True 621 622 self.open_tab("https://www.youtube.com/watch?v=i4FSx9LXVSE") 623 if not self.wait_for_enable_drm(): 624 self._logger.info("Skipped on ESR because cannot enable DRM") 625 return True 626 627 # Wait for duration to be set to something 628 self._logger.info("Wait for video to start") 629 video = self._wait.until( 630 EC.visibility_of_element_located(( 631 By.CSS_SELECTOR, 632 "video.html5-main-video", 633 )) 634 ) 635 self._wait.until(lambda d: type(video.get_property("duration")) is float) 636 self._logger.info("video duration: {}".format(video.get_property("duration"))) 637 assert video.get_property("duration") > exp["duration"], ( 638 "youtube video should have duration" 639 ) 640 641 self._driver.execute_script("arguments[0].click();", video) 642 video.send_keys("k") 643 644 self._wait.until(lambda d: video.get_property("currentTime") > exp["playback"]) 645 self._logger.info("video played: {}".format(video.get_property("currentTime"))) 646 assert video.get_property("currentTime") > exp["playback"], ( 647 "youtube video should perform playback" 648 ) 649 650 return True 651 652 653 if __name__ == "__main__": 654 SnapTests(exp=sys.argv[1])