reftest.rs (37467B)
1 /* This Source Code Form is subject to the terms of the Mozilla Public 2 * License, v. 2.0. If a copy of the MPL was not distributed with this 3 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ 4 5 use crate::{WindowWrapper, NotifierEvent}; 6 use base64::Engine as _; 7 use image::load as load_piston_image; 8 use image::png::PNGEncoder; 9 use image::{ColorType, ImageFormat}; 10 use crate::parse_function::parse_function; 11 use crate::png::save_flipped; 12 use std::{cmp, env}; 13 use std::fmt::{Display, Error, Formatter}; 14 use std::fs::File; 15 use std::io::{BufRead, BufReader}; 16 use std::path::{Path, PathBuf}; 17 use std::process::Command; 18 use std::sync::mpsc::Receiver; 19 use webrender::RenderResults; 20 use webrender::api::*; 21 use webrender::render_api::*; 22 use webrender::api::units::*; 23 use crate::wrench::{Wrench, WrenchThing}; 24 use crate::yaml_frame_reader::YamlFrameReader; 25 26 27 const OPTION_DISABLE_SUBPX: &str = "disable-subpixel"; 28 const OPTION_DISABLE_AA: &str = "disable-aa"; 29 const OPTION_ALLOW_MIPMAPS: &str = "allow-mipmaps"; 30 31 pub struct ReftestOptions { 32 // These override values that are lower. 33 pub allow_max_difference: usize, 34 pub allow_num_differences: usize, 35 } 36 37 impl ReftestOptions { 38 pub fn default() -> Self { 39 ReftestOptions { 40 allow_max_difference: 0, 41 allow_num_differences: 0, 42 } 43 } 44 } 45 46 #[derive(Debug, Copy, Clone)] 47 pub enum ReftestOp { 48 /// Expect that the images match the reference 49 Equal, 50 /// Expect that the images *don't* match the reference 51 NotEqual, 52 /// Expect that drawing the reference at different tiles sizes gives the same pixel exact result. 53 Accurate, 54 /// Expect that drawing the reference at different tiles sizes gives a *different* pixel exact result. 55 Inaccurate, 56 } 57 58 impl Display for ReftestOp { 59 fn fmt(&self, f: &mut Formatter) -> Result<(), Error> { 60 write!( 61 f, 62 "{}", 63 match *self { 64 ReftestOp::Equal => "==".to_owned(), 65 ReftestOp::NotEqual => "!=".to_owned(), 66 ReftestOp::Accurate => "**".to_owned(), 67 ReftestOp::Inaccurate => "!*".to_owned(), 68 } 69 ) 70 } 71 } 72 73 #[derive(Debug)] 74 enum ExtraCheck { 75 DrawCalls(usize), 76 AlphaTargets(usize), 77 ColorTargets(usize), 78 } 79 80 impl ExtraCheck { 81 fn run(&self, results: &[RenderResults]) -> bool { 82 match *self { 83 ExtraCheck::DrawCalls(x) => 84 x == results.last().unwrap().stats.total_draw_calls, 85 ExtraCheck::AlphaTargets(x) => 86 x == results.last().unwrap().stats.alpha_target_count, 87 ExtraCheck::ColorTargets(x) => 88 x == results.last().unwrap().stats.color_target_count, 89 } 90 } 91 } 92 93 pub struct RefTestFuzzy { 94 max_difference: usize, 95 num_differences: usize, 96 } 97 98 pub struct Reftest { 99 op: ReftestOp, 100 test: Vec<PathBuf>, 101 reference: PathBuf, 102 font_render_mode: Option<FontRenderMode>, 103 fuzziness: Vec<RefTestFuzzy>, 104 extra_checks: Vec<ExtraCheck>, 105 allow_mipmaps: bool, 106 force_subpixel_aa_where_possible: Option<bool>, 107 max_surface_override: Option<usize>, 108 } 109 110 impl Reftest { 111 /// Check the positive case (expecting equality) and report details if different 112 fn check_and_report_equality_failure( 113 &self, 114 comparison: ReftestImageComparison, 115 test: &ReftestImage, 116 reference: &ReftestImage, 117 ) -> bool { 118 match comparison { 119 ReftestImageComparison::Equal => { 120 true 121 } 122 ReftestImageComparison::NotEqual { difference_histogram, max_difference, count_different } => { 123 // Each entry in the sorted self.fuzziness list represents a bucket which 124 // allows at most num_differences pixels with a difference of at most 125 // max_difference -- but with the caveat that a difference which is small 126 // enough to be less than a max_difference of an earlier bucket, must be 127 // counted against that bucket. 128 // 129 // Thus the test will fail if the number of pixels with a difference 130 // > fuzzy[j-1].max_difference and <= fuzzy[j].max_difference 131 // exceeds fuzzy[j].num_differences. 132 // 133 // (For the first entry, consider fuzzy[j-1] to allow zero pixels of zero 134 // difference). 135 // 136 // For example, say we have this histogram of differences: 137 // 138 // | [0] [1] [2] [3] [4] [5] [6] ... [255] 139 // ------+------------------------------------------ 140 // Hist. | 0 3 2 1 6 2 0 ... 0 141 // 142 // Ie. image comparison found 3 pixels that differ by 1, 2 that differ by 2, etc. 143 // (Note that entry 0 is always zero, we don't count matching pixels.) 144 // 145 // First we calculate an inclusive prefix sum: 146 // 147 // | [0] [1] [2] [3] [4] [5] [6] ... [255] 148 // ------+------------------------------------------ 149 // Hist. | 0 3 2 1 6 2 0 ... 0 150 // Sum | 0 3 5 6 12 14 14 ... 14 151 // 152 // Let's say the fuzzy statements are: 153 // Fuzzy( 2, 6 ) -- allow up to 6 pixels that differ by 2 or less 154 // Fuzzy( 4, 8 ) -- allow up to 8 pixels that differ by 4 or less _but_ 155 // also by more than 2 (= by 3 or 4). 156 // 157 // The first check is Sum[2] <= max 6 which passes: 5 <= 6. 158 // The second check is Sum[4] - Sum[2] <= max 8 which passes: 12-5 <= 8. 159 // Finally we check if there are any pixels that exceed the max difference (4) 160 // by checking Sum[255] - Sum[4] which shows there are 14-12 == 2 so we fail. 161 162 let prefix_sum = difference_histogram.iter() 163 .scan(0, |sum, i| { *sum += i; Some(*sum) }) 164 .collect::<Vec<_>>(); 165 166 // check each fuzzy statement for violations. 167 assert_eq!(0, difference_histogram[0]); 168 assert_eq!(0, prefix_sum[0]); 169 170 // loop invariant: this is the max_difference of the previous iteration's 'fuzzy' 171 let mut previous_max_diff = 0; 172 173 // loop invariant: this is the number of pixels to ignore as they have been counted 174 // against previous iterations' fuzzy statements. 175 let mut previous_sum_fail = 0; // == prefix_sum[previous_max_diff] 176 177 let mut is_failing = false; 178 let mut fail_text = String::new(); 179 180 for fuzzy in &self.fuzziness { 181 let fuzzy_max_difference = cmp::min(255, fuzzy.max_difference); 182 let num_differences = prefix_sum[fuzzy_max_difference] - previous_sum_fail; 183 if num_differences > fuzzy.num_differences { 184 fail_text.push_str( 185 &format!("{} differences > {} and <= {} (allowed {}); ", 186 num_differences, 187 previous_max_diff, fuzzy_max_difference, 188 fuzzy.num_differences)); 189 is_failing = true; 190 } 191 previous_max_diff = fuzzy_max_difference; 192 previous_sum_fail = prefix_sum[previous_max_diff]; 193 } 194 // do we have any pixels with a difference above the highest allowed 195 // max difference? if so, we fail the test: 196 let num_differences = prefix_sum[255] - previous_sum_fail; 197 if num_differences > 0 { 198 fail_text.push_str( 199 &format!("{} num_differences > {} and <= {} (allowed {}); ", 200 num_differences, 201 previous_max_diff, 255, 202 0)); 203 is_failing = true; 204 } 205 206 if is_failing { 207 println!( 208 "REFTEST TEST-UNEXPECTED-FAIL | {} | \ 209 image comparison, max difference: {}, number of differing pixels: {} | {}", 210 self, 211 max_difference, 212 count_different, 213 fail_text, 214 ); 215 println!("REFTEST IMAGE 1 (TEST): {}", test.clone().create_data_uri()); 216 println!( 217 "REFTEST IMAGE 2 (REFERENCE): {}", 218 reference.clone().create_data_uri() 219 ); 220 println!("REFTEST TEST-END | {}", self); 221 222 false 223 } else { 224 true 225 } 226 } 227 } 228 } 229 230 /// Report details of the negative case 231 fn report_unexpected_equality(&self) { 232 println!("REFTEST TEST-UNEXPECTED-FAIL | {} | image comparison", self); 233 println!("REFTEST TEST-END | {}", self); 234 } 235 } 236 237 impl Display for Reftest { 238 fn fmt(&self, f: &mut Formatter) -> Result<(), Error> { 239 let paths: Vec<String> = self.test.iter().map(|t| t.display().to_string()).collect(); 240 write!( 241 f, 242 "{} {} {}", 243 paths.join(", "), 244 self.op, 245 self.reference.display() 246 ) 247 } 248 } 249 250 #[derive(Clone)] 251 pub struct ReftestImage { 252 pub data: Vec<u8>, 253 pub size: DeviceIntSize, 254 } 255 256 #[derive(Debug, Clone)] 257 pub enum ReftestImageComparison { 258 Equal, 259 NotEqual { 260 /// entry[j] = number of pixels with a difference of exactly j 261 difference_histogram: Vec<usize>, 262 max_difference: usize, 263 count_different: usize, 264 }, 265 } 266 267 impl ReftestImage { 268 pub fn compare(&self, other: &ReftestImage) -> ReftestImageComparison { 269 assert_eq!(self.size, other.size); 270 assert_eq!(self.data.len(), other.data.len()); 271 assert_eq!(self.data.len() % 4, 0); 272 273 let mut histogram = [0usize; 256]; 274 let mut count = 0; 275 let mut max = 0; 276 277 for (a, b) in self.data.chunks(4).zip(other.data.chunks(4)) { 278 if a != b { 279 let pixel_max = a.iter() 280 .zip(b.iter()) 281 .map(|(x, y)| (*x as isize - *y as isize).abs() as usize) 282 .max() 283 .unwrap(); 284 285 count += 1; 286 assert!(pixel_max < 256, "pixel values are not 8 bit, update the histogram binning code"); 287 // deliberately avoid counting pixels that match -- 288 // histogram[0] stays at zero. 289 // this helps our prefix sum later during analysis to 290 // only count actual differences. 291 histogram[pixel_max as usize] += 1; 292 max = cmp::max(max, pixel_max); 293 } 294 } 295 296 if count != 0 { 297 ReftestImageComparison::NotEqual { 298 difference_histogram: histogram.to_vec(), 299 max_difference: max, 300 count_different: count, 301 } 302 } else { 303 ReftestImageComparison::Equal 304 } 305 } 306 307 pub fn create_data_uri(mut self) -> String { 308 let width = self.size.width; 309 let height = self.size.height; 310 311 // flip image vertically (texture is upside down) 312 let orig_pixels = self.data.clone(); 313 let stride = width as usize * 4; 314 for y in 0 .. height as usize { 315 let dst_start = y * stride; 316 let src_start = (height as usize - y - 1) * stride; 317 let src_slice = &orig_pixels[src_start .. src_start + stride]; 318 (&mut self.data[dst_start .. dst_start + stride]) 319 .clone_from_slice(&src_slice[.. stride]); 320 } 321 322 let mut png: Vec<u8> = vec![]; 323 { 324 let encoder = PNGEncoder::new(&mut png); 325 encoder 326 .encode(&self.data[..], width as u32, height as u32, ColorType::Rgba8) 327 .expect("Unable to encode PNG!"); 328 } 329 let png_base64 = base64::engine::general_purpose::STANDARD.encode(&png); 330 format!("data:image/png;base64,{}", png_base64) 331 } 332 } 333 334 struct ReftestManifest { 335 reftests: Vec<Reftest>, 336 } 337 impl ReftestManifest { 338 fn new(manifest: &Path, environment: &ReftestEnvironment, options: &ReftestOptions) -> ReftestManifest { 339 let dir = manifest.parent().unwrap(); 340 let f = 341 File::open(manifest).unwrap_or_else(|_| panic!("couldn't open manifest: {}", manifest.display())); 342 let file = BufReader::new(&f); 343 344 let mut reftests = Vec::new(); 345 346 for line in file.lines() { 347 let l = line.unwrap(); 348 349 // strip the comments 350 let s = &l[0 .. l.find('#').unwrap_or(l.len())]; 351 let s = s.trim(); 352 if s.is_empty() { 353 continue; 354 } 355 356 let tokens: Vec<&str> = s.split_whitespace().collect(); 357 358 let mut fuzziness = Vec::new(); 359 let mut op = None; 360 let mut font_render_mode = None; 361 let mut extra_checks = vec![]; 362 let mut allow_mipmaps = false; 363 let mut force_subpixel_aa_where_possible = None; 364 let mut max_surface_override = None; 365 366 let mut parse_command = |token: &str| -> bool { 367 match token { 368 function if function.starts_with("force_subpixel_aa_where_possible(") => { 369 let (_, args, _) = parse_function(function); 370 force_subpixel_aa_where_possible = Some(args[0].parse().unwrap()); 371 } 372 function if function.starts_with("fuzzy-range(") || 373 function.starts_with("fuzzy-range-if(") => { 374 let (_, mut args, _) = parse_function(function); 375 if function.starts_with("fuzzy-range-if(") { 376 if !environment.parse_condition(args.remove(0)).expect("unknown condition") { 377 return true; 378 } 379 fuzziness.clear(); 380 } 381 let num_range = args.len() / 2; 382 for range in 0..num_range { 383 let mut max = args[range * 2 ]; 384 let mut num = args[range * 2 + 1]; 385 if max.starts_with("<=") { // trim_start_matches would allow <=<=123 386 max = &max[2..]; 387 } 388 if num.starts_with('*') { 389 num = &num[1..]; 390 } 391 let max_difference = max.parse().unwrap(); 392 let num_differences = num.parse().unwrap(); 393 fuzziness.push(RefTestFuzzy { max_difference, num_differences }); 394 } 395 } 396 function if function.starts_with("fuzzy(") || 397 function.starts_with("fuzzy-if(") => { 398 let (_, mut args, _) = parse_function(function); 399 if function.starts_with("fuzzy-if(") { 400 if !environment.parse_condition(args.remove(0)).expect("unknown condition") { 401 return true; 402 } 403 fuzziness.clear(); 404 } 405 let max_difference = args[0].parse().unwrap(); 406 let num_differences = args[1].parse().unwrap(); 407 assert!(fuzziness.is_empty()); // if this fires, consider fuzzy-range instead 408 fuzziness.push(RefTestFuzzy { max_difference, num_differences }); 409 } 410 function if function.starts_with("draw_calls(") => { 411 let (_, args, _) = parse_function(function); 412 extra_checks.push(ExtraCheck::DrawCalls(args[0].parse().unwrap())); 413 } 414 function if function.starts_with("alpha_targets(") => { 415 let (_, args, _) = parse_function(function); 416 extra_checks.push(ExtraCheck::AlphaTargets(args[0].parse().unwrap())); 417 } 418 function if function.starts_with("color_targets(") => { 419 let (_, args, _) = parse_function(function); 420 extra_checks.push(ExtraCheck::ColorTargets(args[0].parse().unwrap())); 421 } 422 function if function.starts_with("max_surface_size(") => { 423 let (_, args, _) = parse_function(function); 424 max_surface_override = Some(args[0].parse().unwrap()); 425 } 426 options if options.starts_with("options(") => { 427 let (_, args, _) = parse_function(options); 428 if args.iter().any(|arg| arg == &OPTION_DISABLE_SUBPX) { 429 font_render_mode = Some(FontRenderMode::Alpha); 430 } 431 if args.iter().any(|arg| arg == &OPTION_DISABLE_AA) { 432 font_render_mode = Some(FontRenderMode::Mono); 433 } 434 if args.iter().any(|arg| arg == &OPTION_ALLOW_MIPMAPS) { 435 allow_mipmaps = true; 436 } 437 } 438 _ => return false, 439 } 440 true 441 }; 442 443 let mut paths = vec![]; 444 for (i, token) in tokens.iter().enumerate() { 445 match *token { 446 "include" => { 447 assert!(i == 0, "include must be by itself"); 448 let include = dir.join(tokens[1]); 449 450 reftests.append( 451 &mut ReftestManifest::new(include.as_path(), environment, options).reftests, 452 ); 453 454 break; 455 } 456 "==" => { 457 op = Some(ReftestOp::Equal); 458 } 459 "!=" => { 460 op = Some(ReftestOp::NotEqual); 461 } 462 "**" => { 463 op = Some(ReftestOp::Accurate); 464 } 465 "!*" => { 466 op = Some(ReftestOp::Inaccurate); 467 } 468 cond if cond.starts_with("if(") => { 469 let (_, args, _) = parse_function(cond); 470 if environment.parse_condition(args[0]).expect("unknown condition") { 471 for command in &args[1..] { 472 parse_command(command); 473 } 474 } 475 } 476 command if parse_command(command) => {} 477 _ => { 478 match environment.parse_condition(*token) { 479 Some(true) => {} 480 Some(false) => break, 481 _ => paths.push(dir.join(*token)), 482 } 483 } 484 } 485 } 486 487 // Don't try to add tests for include lines. 488 if op.is_none() { 489 assert!(paths.is_empty(), "paths = {:?}", paths); 490 continue; 491 } 492 let op = op.unwrap(); 493 494 // The reference is the last path provided. If multiple paths are 495 // passed for the test, they render sequentially before being 496 // compared to the reference, which is useful for testing 497 // invalidation. 498 let reference = paths.pop().unwrap(); 499 let test = paths; 500 501 if environment.platform == "android" { 502 // Add some fuzz on mobile as we do for non-wrench reftests. 503 // First remove the ranges with difference <= 2, otherwise they might cause the 504 // test to fail before the new range is picked up. 505 fuzziness.retain(|fuzzy| fuzzy.max_difference > 2); 506 fuzziness.push(RefTestFuzzy { max_difference: 2, num_differences: std::usize::MAX }); 507 } 508 509 // to avoid changing the meaning of existing tests, the case of 510 // only a single (or no) 'fuzzy' keyword means we use the max 511 // of that fuzzy and options.allow_.. (we don't want that to 512 // turn into a test that allows fuzzy.allow_ *plus* options.allow_): 513 match fuzziness.len() { 514 0 => fuzziness.push(RefTestFuzzy { 515 max_difference: options.allow_max_difference, 516 num_differences: options.allow_num_differences }), 517 1 => { 518 let fuzzy = &mut fuzziness[0]; 519 fuzzy.max_difference = cmp::max(fuzzy.max_difference, options.allow_max_difference); 520 fuzzy.num_differences = cmp::max(fuzzy.num_differences, options.allow_num_differences); 521 }, 522 _ => { 523 // ignore options, use multiple fuzzy keywords instead. make sure 524 // the list is sorted to speed up counting violations. 525 fuzziness.sort_by(|a, b| a.max_difference.cmp(&b.max_difference)); 526 for pair in fuzziness.windows(2) { 527 if pair[0].max_difference == pair[1].max_difference { 528 println!("Warning: repeated fuzzy of max_difference {} ignored.", 529 pair[1].max_difference); 530 } 531 } 532 } 533 } 534 535 reftests.push(Reftest { 536 op, 537 test, 538 reference, 539 font_render_mode, 540 fuzziness, 541 extra_checks, 542 allow_mipmaps, 543 force_subpixel_aa_where_possible, 544 max_surface_override, 545 }); 546 } 547 548 ReftestManifest { reftests } 549 } 550 551 fn find(&self, prefix: &Path) -> Vec<&Reftest> { 552 self.reftests 553 .iter() 554 .filter(|x| { 555 x.test.iter().any(|t| t.starts_with(prefix)) || x.reference.starts_with(prefix) 556 }) 557 .collect() 558 } 559 } 560 561 struct YamlRenderOutput { 562 image: ReftestImage, 563 results: RenderResults, 564 } 565 566 struct ReftestEnvironment { 567 pub platform: &'static str, 568 pub version: Option<semver::Version>, 569 pub mode: &'static str, 570 } 571 572 impl ReftestEnvironment { 573 fn new(wrench: &Wrench, window: &WindowWrapper) -> Self { 574 Self { 575 platform: Self::platform(wrench, window), 576 version: Self::version(wrench, window), 577 mode: Self::mode(), 578 } 579 } 580 581 fn has(&self, condition: &str) -> bool { 582 if self.platform == condition || self.mode == condition { 583 return true; 584 } 585 if let (Some(v), Ok(r)) = (&self.version, &semver::VersionReq::parse(condition)) { 586 if r.matches(v) { 587 return true; 588 } 589 } 590 let envkey = format!("WRENCH_REFTEST_CONDITION_{}", condition.to_uppercase()); 591 env::var(envkey).is_ok() 592 } 593 594 fn platform(_wrench: &Wrench, window: &WindowWrapper) -> &'static str { 595 if window.is_software() { 596 "swgl" 597 } else if cfg!(target_os = "windows") { 598 "win" 599 } else if cfg!(target_os = "linux") { 600 "linux" 601 } else if cfg!(target_os = "macos") { 602 "mac" 603 } else if cfg!(target_os = "android") { 604 "android" 605 } else { 606 "other" 607 } 608 } 609 610 fn version(_wrench: &Wrench, window: &WindowWrapper) -> Option<semver::Version> { 611 if window.is_software() { 612 None 613 } else if cfg!(target_os = "macos") { 614 use std::str; 615 let version_bytes = Command::new("defaults") 616 .arg("read") 617 .arg("loginwindow") 618 .arg("SystemVersionStampAsString") 619 .output() 620 .expect("Failed to get macOS version") 621 .stdout; 622 let mut version_string = str::from_utf8(&version_bytes) 623 .expect("Failed to read macOS version") 624 .trim() 625 .to_string(); 626 // On some machines this produces just the major.minor and on 627 // some machines this gives major.minor.patch. But semver requires 628 // the patch so we fake one if it's not there. 629 if version_string.chars().filter(|c| *c == '.').count() == 1 { 630 version_string.push_str(".0"); 631 } 632 Some(semver::Version::parse(&version_string) 633 .unwrap_or_else(|_| panic!("Failed to parse macOS version {}", version_string))) 634 } else { 635 None 636 } 637 } 638 639 fn mode() -> &'static str { 640 if cfg!(debug_assertions) { 641 "debug" 642 } else { 643 "release" 644 } 645 } 646 647 fn parse_condition(&self, token: &str) -> Option<bool> { 648 match token { 649 platform if platform.starts_with("skip_on(") => { 650 // e.g. skip_on(android,debug) will skip only when 651 // running on a debug android build. 652 let (_, args, _) = parse_function(platform); 653 Some(!args.iter().all(|arg| self.has(arg))) 654 } 655 platform if platform.starts_with("env(") => { 656 // non-negated version of skip_on for nested conditions 657 let (_, args, _) = parse_function(platform); 658 Some(args.iter().all(|arg| self.has(arg))) 659 } 660 platform if platform.starts_with("platform(") => { 661 let (_, args, _) = parse_function(platform); 662 // Skip due to platform not matching 663 Some(args.iter().any(|arg| arg == &self.platform)) 664 } 665 op if op.starts_with("not(") => { 666 let (_, args, _) = parse_function(op); 667 Some(!self.parse_condition(args[0]).expect("unknown condition")) 668 } 669 op if op.starts_with("or(") => { 670 let (_, args, _) = parse_function(op); 671 Some(args.iter().any(|arg| self.parse_condition(arg).expect("unknown condition"))) 672 } 673 op if op.starts_with("and(") => { 674 let (_, args, _) = parse_function(op); 675 Some(args.iter().all(|arg| self.parse_condition(arg).expect("unknown condition"))) 676 } 677 _ => None, 678 } 679 } 680 } 681 682 pub struct ReftestHarness<'a> { 683 wrench: &'a mut Wrench, 684 window: &'a mut WindowWrapper, 685 rx: &'a Receiver<NotifierEvent>, 686 environment: ReftestEnvironment, 687 } 688 impl<'a> ReftestHarness<'a> { 689 pub fn new(wrench: &'a mut Wrench, window: &'a mut WindowWrapper, rx: &'a Receiver<NotifierEvent>) -> Self { 690 let environment = ReftestEnvironment::new(wrench, window); 691 ReftestHarness { wrench, window, rx, environment } 692 } 693 694 pub fn run(mut self, base_manifest: &Path, reftests: Option<&Path>, options: &ReftestOptions) -> usize { 695 let manifest = ReftestManifest::new(base_manifest, &self.environment, options); 696 let reftests = manifest.find(reftests.unwrap_or(&PathBuf::new())); 697 698 let mut total_passing = 0; 699 let mut failing = Vec::new(); 700 701 for t in reftests { 702 if self.run_reftest(t) { 703 total_passing += 1; 704 } else { 705 failing.push(t); 706 } 707 } 708 709 println!( 710 "REFTEST INFO | {} passing, {} failing", 711 total_passing, 712 failing.len() 713 ); 714 715 if !failing.is_empty() { 716 println!("\nReftests with unexpected results:"); 717 718 for reftest in &failing { 719 println!("\t{}", reftest); 720 } 721 } 722 723 failing.len() 724 } 725 726 fn run_reftest(&mut self, t: &Reftest) -> bool { 727 let test_name = t.to_string(); 728 println!("REFTEST {}", test_name); 729 profile_scope!("wrench reftest", text: &test_name); 730 731 self.wrench 732 .api 733 .send_debug_cmd( 734 DebugCommand::ClearCaches(ClearCache::all()) 735 ); 736 737 let quality_settings = QualitySettings { 738 force_subpixel_aa_where_possible: t.force_subpixel_aa_where_possible.unwrap_or_default(), 739 }; 740 741 self.wrench.set_quality_settings(quality_settings); 742 743 if let Some(max_surface_override) = t.max_surface_override { 744 self.wrench 745 .api 746 .send_debug_cmd( 747 DebugCommand::SetMaximumSurfaceSize(Some(max_surface_override)) 748 ); 749 } 750 751 let window_size = self.window.get_inner_size(); 752 let reference_image = match t.reference.extension().unwrap().to_str().unwrap() { 753 "yaml" => None, 754 "png" => Some(self.load_image(t.reference.as_path(), ImageFormat::Png)), 755 other => panic!("Unknown reftest extension: {}", other), 756 }; 757 let test_size = reference_image.as_ref().map_or(window_size, |img| img.size); 758 759 // The reference can be smaller than the window size, in which case 760 // we only compare the intersection. 761 // 762 // Note also that, when we have multiple test scenes in sequence, we 763 // want to test the picture caching machinery. But since picture caching 764 // only takes effect after the result has been the same several frames in 765 // a row, we need to render the scene multiple times. 766 let mut images = vec![]; 767 let mut results = vec![]; 768 769 match t.op { 770 ReftestOp::Equal | ReftestOp::NotEqual => { 771 // For equality tests, render each test image and store result 772 for filename in t.test.iter() { 773 let output = self.render_yaml( 774 filename, 775 test_size, 776 t.font_render_mode, 777 t.allow_mipmaps, 778 ); 779 images.push(output.image); 780 results.push(output.results); 781 } 782 } 783 ReftestOp::Accurate | ReftestOp::Inaccurate => { 784 // For accuracy tests, render the reference yaml at an arbitrary series 785 // of tile sizes, and compare to the reference drawn at normal tile size. 786 let tile_sizes = [ 787 DeviceIntSize::new(128, 128), 788 DeviceIntSize::new(256, 256), 789 DeviceIntSize::new(512, 512), 790 ]; 791 792 for tile_size in &tile_sizes { 793 self.wrench 794 .api 795 .send_debug_cmd( 796 DebugCommand::SetPictureTileSize(Some(*tile_size)) 797 ); 798 799 let output = self.render_yaml( 800 &t.reference, 801 test_size, 802 t.font_render_mode, 803 t.allow_mipmaps, 804 ); 805 images.push(output.image); 806 results.push(output.results); 807 } 808 809 self.wrench 810 .api 811 .send_debug_cmd( 812 DebugCommand::SetPictureTileSize(None) 813 ); 814 } 815 } 816 817 let reference = if let Some(image) = reference_image { 818 let save_all_png = false; // flip to true to update all the tests! 819 if save_all_png { 820 let img = images.last().unwrap(); 821 save_flipped(&t.reference, img.data.clone(), img.size); 822 } 823 image 824 } else { 825 let output = self.render_yaml( 826 &t.reference, 827 test_size, 828 t.font_render_mode, 829 t.allow_mipmaps, 830 ); 831 output.image 832 }; 833 834 if let Some(_) = t.max_surface_override { 835 self.wrench 836 .api 837 .send_debug_cmd( 838 DebugCommand::SetMaximumSurfaceSize(None) 839 ); 840 } 841 842 for extra_check in t.extra_checks.iter() { 843 if !extra_check.run(&results) { 844 println!( 845 "REFTEST TEST-UNEXPECTED-FAIL | {} | Failing Check: {:?} | Actual Results: {:?}", 846 t, 847 extra_check, 848 results, 849 ); 850 println!("REFTEST TEST-END | {}", t); 851 return false; 852 } 853 } 854 855 match t.op { 856 ReftestOp::Equal => { 857 // Ensure that the final image matches the reference 858 let test = images.pop().unwrap(); 859 let comparison = test.compare(&reference); 860 t.check_and_report_equality_failure( 861 comparison, 862 &test, 863 &reference, 864 ) 865 } 866 ReftestOp::NotEqual => { 867 // Ensure that the final image *doesn't* match the reference 868 let test = images.pop().unwrap(); 869 let comparison = test.compare(&reference); 870 match comparison { 871 ReftestImageComparison::Equal => { 872 t.report_unexpected_equality(); 873 false 874 } 875 ReftestImageComparison::NotEqual { .. } => { 876 true 877 } 878 } 879 } 880 ReftestOp::Accurate => { 881 // Ensure that *all* images match the reference 882 for test in images.drain(..) { 883 let comparison = test.compare(&reference); 884 885 if !t.check_and_report_equality_failure( 886 comparison, 887 &test, 888 &reference, 889 ) { 890 return false; 891 } 892 } 893 894 true 895 } 896 ReftestOp::Inaccurate => { 897 // Ensure that at least one of the images doesn't match the reference 898 let all_same = images.iter().all(|image| { 899 match image.compare(&reference) { 900 ReftestImageComparison::Equal => true, 901 ReftestImageComparison::NotEqual { .. } => false, 902 } 903 }); 904 905 if all_same { 906 t.report_unexpected_equality(); 907 } 908 909 !all_same 910 } 911 } 912 } 913 914 fn load_image(&mut self, filename: &Path, format: ImageFormat) -> ReftestImage { 915 let file = BufReader::new(File::open(filename).unwrap()); 916 let img_raw = load_piston_image(file, format).unwrap(); 917 let img = img_raw.flipv().to_rgba(); 918 let size = img.dimensions(); 919 ReftestImage { 920 data: img.into_raw(), 921 size: DeviceIntSize::new(size.0 as i32, size.1 as i32), 922 } 923 } 924 925 fn render_yaml( 926 &mut self, 927 filename: &Path, 928 size: DeviceIntSize, 929 font_render_mode: Option<FontRenderMode>, 930 allow_mipmaps: bool, 931 ) -> YamlRenderOutput { 932 let mut reader = YamlFrameReader::new(filename); 933 reader.set_font_render_mode(font_render_mode); 934 reader.allow_mipmaps(allow_mipmaps); 935 reader.do_frame(self.wrench); 936 937 self.wrench.api.flush_scene_builder(); 938 939 // wait for the frame 940 self.rx.recv().unwrap(); 941 let results = self.wrench.render(); 942 943 let window_size = self.window.get_inner_size(); 944 assert!( 945 size.width <= window_size.width && 946 size.height <= window_size.height, 947 "size={:?} ws={:?}", size, window_size 948 ); 949 950 // taking the bottom left sub-rectangle 951 let rect = FramebufferIntRect::from_origin_and_size( 952 FramebufferIntPoint::new(0, window_size.height - size.height), 953 FramebufferIntSize::new(size.width, size.height), 954 ); 955 let pixels = self.wrench.renderer.read_pixels_rgba8(rect); 956 self.window.swap_buffers(); 957 958 let write_debug_images = false; 959 if write_debug_images { 960 let debug_path = filename.with_extension("yaml.png"); 961 save_flipped(debug_path, pixels.clone(), size); 962 } 963 964 reader.deinit(self.wrench); 965 966 YamlRenderOutput { 967 image: ReftestImage { data: pixels, size }, 968 results, 969 } 970 } 971 }