perf.rs (11423B)
1 /* This Source Code Form is subject to the terms of the Mozilla Public 2 * License, v. 2.0. If a copy of the MPL was not distributed with this 3 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ 4 5 use crate::NotifierEvent; 6 use crate::WindowWrapper; 7 use std::collections::{HashMap, HashSet}; 8 use std::fs::File; 9 use std::io::{BufRead, BufReader}; 10 use std::io::{Read, Write}; 11 use std::path::{Path, PathBuf}; 12 use std::sync::mpsc::Receiver; 13 use crate::wrench::{Wrench, WrenchThing}; 14 use crate::yaml_frame_reader::YamlFrameReader; 15 use webrender::DebugFlags; 16 use webrender::render_api::DebugCommand; 17 18 const COLOR_DEFAULT: &str = "\x1b[0m"; 19 const COLOR_RED: &str = "\x1b[31m"; 20 const COLOR_GREEN: &str = "\x1b[32m"; 21 const COLOR_MAGENTA: &str = "\x1b[95m"; 22 23 const MIN_SAMPLE_COUNT: usize = 50; 24 const SAMPLE_EXCLUDE_COUNT: usize = 10; 25 26 pub struct Benchmark { 27 pub test: PathBuf, 28 } 29 30 pub struct BenchmarkManifest { 31 pub benchmarks: Vec<Benchmark>, 32 } 33 34 impl BenchmarkManifest { 35 pub fn new(manifest: &Path) -> BenchmarkManifest { 36 let dir = manifest.parent().unwrap(); 37 let f = 38 File::open(manifest).unwrap_or_else(|_| panic!("couldn't open manifest: {}", manifest.display())); 39 let file = BufReader::new(&f); 40 41 let mut benchmarks = Vec::new(); 42 43 for line in file.lines() { 44 let l = line.unwrap(); 45 46 // strip the comments 47 let s = &l[0 .. l.find('#').unwrap_or(l.len())]; 48 let s = s.trim(); 49 if s.is_empty() { 50 continue; 51 } 52 53 let mut items = s.split_whitespace(); 54 55 match items.next() { 56 Some("include") => { 57 let include = dir.join(items.next().unwrap()); 58 59 benchmarks.append(&mut BenchmarkManifest::new(include.as_path()).benchmarks); 60 } 61 Some(name) => { 62 let test = dir.join(name); 63 benchmarks.push(Benchmark { test }); 64 } 65 _ => panic!(), 66 }; 67 } 68 69 BenchmarkManifest { 70 benchmarks, 71 } 72 } 73 } 74 75 #[derive(Clone, Serialize, Deserialize)] 76 struct TestProfileRange { 77 min: u64, 78 avg: u64, 79 max: u64, 80 } 81 82 #[derive(Clone, Serialize, Deserialize)] 83 struct TestProfile { 84 name: String, 85 backend_time_ns: TestProfileRange, 86 composite_time_ns: TestProfileRange, 87 paint_time_ns: TestProfileRange, 88 draw_calls: usize, 89 } 90 91 impl TestProfile { 92 fn csv_header() -> String { 93 "name,\ 94 backend_time_ns min, avg, max,\ 95 composite_time_ns min, avg, max,\ 96 paint_time_ns min, avg, max,\ 97 draw_calls\n".to_string() 98 } 99 100 fn convert_to_csv(&self) -> String { 101 format!("{},\ 102 {},{},{},\ 103 {},{},{},\ 104 {},{},{},\ 105 {}\n", 106 self.name, 107 self.backend_time_ns.min, self.backend_time_ns.avg, self.backend_time_ns.max, 108 self.composite_time_ns.min, self.composite_time_ns.avg, self.composite_time_ns.max, 109 self.paint_time_ns.min, self.paint_time_ns.avg, self.paint_time_ns.max, 110 self.draw_calls) 111 } 112 } 113 114 #[derive(Serialize, Deserialize)] 115 struct Profile { 116 tests: Vec<TestProfile>, 117 } 118 119 impl Profile { 120 fn new() -> Profile { 121 Profile { tests: Vec::new() } 122 } 123 124 fn add(&mut self, profile: TestProfile) { 125 self.tests.push(profile); 126 } 127 128 fn save(&self, filename: &str, as_csv: bool) { 129 let mut file = File::create(&filename).unwrap(); 130 if as_csv { 131 file.write_all(&TestProfile::csv_header().into_bytes()).unwrap(); 132 for test in &self.tests { 133 file.write_all(&test.convert_to_csv().into_bytes()).unwrap(); 134 } 135 } else { 136 let s = serde_json::to_string_pretty(self).unwrap(); 137 file.write_all(&s.into_bytes()).unwrap(); 138 file.write_all(b"\n").unwrap(); 139 } 140 } 141 142 fn load(filename: &str) -> Profile { 143 let mut file = File::open(&filename).unwrap(); 144 let mut string = String::new(); 145 file.read_to_string(&mut string).unwrap(); 146 serde_json::from_str(&string).expect("Unable to load profile!") 147 } 148 149 fn build_set_and_map_of_tests(&self) -> (HashSet<String>, HashMap<String, TestProfile>) { 150 let mut hash_set = HashSet::new(); 151 let mut hash_map = HashMap::new(); 152 153 for test in &self.tests { 154 hash_set.insert(test.name.clone()); 155 hash_map.insert(test.name.clone(), test.clone()); 156 } 157 158 (hash_set, hash_map) 159 } 160 } 161 162 pub struct PerfHarness<'a> { 163 wrench: &'a mut Wrench, 164 window: &'a mut WindowWrapper, 165 rx: Receiver<NotifierEvent>, 166 warmup_frames: usize, 167 sample_count: usize, 168 } 169 170 impl<'a> PerfHarness<'a> { 171 pub fn new(wrench: &'a mut Wrench, 172 window: &'a mut WindowWrapper, 173 rx: Receiver<NotifierEvent>, 174 warmup_frames: Option<usize>, 175 sample_count: Option<usize>) -> Self { 176 PerfHarness { 177 wrench, 178 window, 179 rx, 180 warmup_frames: warmup_frames.unwrap_or(0usize), 181 sample_count: sample_count.unwrap_or(MIN_SAMPLE_COUNT), 182 } 183 } 184 185 pub fn run(mut self, base_manifest: &Path, filename: &str, as_csv: bool) { 186 let manifest = BenchmarkManifest::new(base_manifest); 187 188 let mut profile = Profile::new(); 189 190 for t in manifest.benchmarks { 191 let stats = self.render_yaml(t.test.as_path()); 192 profile.add(stats); 193 } 194 195 profile.save(filename, as_csv); 196 } 197 198 fn render_yaml(&mut self, filename: &Path) -> TestProfile { 199 let mut reader = YamlFrameReader::new(filename); 200 201 // Loop until we get a reasonable number of CPU and GPU 202 // frame profiles. Then take the mean. 203 let mut cpu_frame_profiles = Vec::new(); 204 let mut gpu_frame_profiles = Vec::new(); 205 206 let mut debug_flags = DebugFlags::empty(); 207 debug_flags.set(DebugFlags::GPU_TIME_QUERIES | DebugFlags::GPU_SAMPLE_QUERIES, true); 208 self.wrench.api.send_debug_cmd(DebugCommand::SetFlags(debug_flags)); 209 210 let mut frame_count = 0; 211 212 while cpu_frame_profiles.len() < self.sample_count || 213 gpu_frame_profiles.len() < self.sample_count 214 { 215 reader.do_frame(self.wrench); 216 self.rx.recv().unwrap(); 217 self.wrench.render(); 218 self.window.swap_buffers(); 219 let (cpu_profiles, gpu_profiles) = self.wrench.get_frame_profiles(); 220 if frame_count >= self.warmup_frames { 221 cpu_frame_profiles.extend(cpu_profiles); 222 gpu_frame_profiles.extend(gpu_profiles); 223 } 224 frame_count += 1; 225 } 226 227 // Ensure the draw calls match in every sample. 228 let draw_calls = cpu_frame_profiles[0].draw_calls; 229 let draw_calls_same = 230 cpu_frame_profiles 231 .iter() 232 .all(|s| s.draw_calls == draw_calls); 233 234 // this can be normal in cases where some elements are cached (eg. linear 235 // gradients), but print a warning in case it's not (which could make the 236 // benchmark produce unexpected results). 237 if !draw_calls_same { 238 println!("Warning: not every frame has the same number of draw calls"); 239 } 240 241 let composite_time_ns = extract_sample(&mut cpu_frame_profiles, |a| a.composite_time_ns); 242 let paint_time_ns = extract_sample(&mut gpu_frame_profiles, |a| a.paint_time_ns); 243 let backend_time_ns = extract_sample(&mut cpu_frame_profiles, |a| a.backend_time_ns); 244 245 TestProfile { 246 name: filename.to_str().unwrap().to_string(), 247 composite_time_ns, 248 paint_time_ns, 249 backend_time_ns, 250 draw_calls, 251 } 252 } 253 } 254 255 // returns min, average, max, after removing the lowest and highest SAMPLE_EXCLUDE_COUNT 256 // samples (each). 257 fn extract_sample<F, T>(profiles: &mut [T], f: F) -> TestProfileRange 258 where 259 F: Fn(&T) -> u64, 260 { 261 let mut samples: Vec<u64> = profiles.iter().map(f).collect(); 262 samples.sort_unstable(); 263 let useful_samples = &samples[SAMPLE_EXCLUDE_COUNT .. samples.len() - SAMPLE_EXCLUDE_COUNT]; 264 let total_time: u64 = useful_samples.iter().sum(); 265 TestProfileRange { 266 min: useful_samples[0], 267 avg: total_time / useful_samples.len() as u64, 268 max: useful_samples[useful_samples.len()-1] 269 } 270 } 271 272 fn select_color(base: f32, value: f32) -> &'static str { 273 let tolerance = base * 0.1; 274 if (value - base).abs() < tolerance { 275 COLOR_DEFAULT 276 } else if value > base { 277 COLOR_RED 278 } else { 279 COLOR_GREEN 280 } 281 } 282 283 pub fn compare(first_filename: &str, second_filename: &str) { 284 let profile0 = Profile::load(first_filename); 285 let profile1 = Profile::load(second_filename); 286 287 let (set0, map0) = profile0.build_set_and_map_of_tests(); 288 let (set1, map1) = profile1.build_set_and_map_of_tests(); 289 290 print!("+------------------------------------------------"); 291 println!("+--------------+------------------+------------------+"); 292 print!("| Test name "); 293 println!("| Draw Calls | Composite (ms) | Paint (ms) |"); 294 print!("+------------------------------------------------"); 295 println!("+--------------+------------------+------------------+"); 296 297 for test_name in set0.symmetric_difference(&set1) { 298 println!( 299 "| {}{:47}{}|{:14}|{:18}|{:18}|", 300 COLOR_MAGENTA, 301 test_name, 302 COLOR_DEFAULT, 303 " -", 304 " -", 305 " -" 306 ); 307 } 308 309 for test_name in set0.intersection(&set1) { 310 let test0 = &map0[test_name]; 311 let test1 = &map1[test_name]; 312 313 let composite_time0 = test0.composite_time_ns.avg as f32 / 1000000.0; 314 let composite_time1 = test1.composite_time_ns.avg as f32 / 1000000.0; 315 316 let paint_time0 = test0.paint_time_ns.avg as f32 / 1000000.0; 317 let paint_time1 = test1.paint_time_ns.avg as f32 / 1000000.0; 318 319 let draw_calls_color = match test0.draw_calls.cmp(&test1.draw_calls) { 320 std::cmp::Ordering::Equal => COLOR_DEFAULT, 321 std::cmp::Ordering::Greater => COLOR_GREEN, 322 std::cmp::Ordering::Less => COLOR_RED, 323 }; 324 325 let composite_time_color = select_color(composite_time0, composite_time1); 326 let paint_time_color = select_color(paint_time0, paint_time1); 327 328 let draw_call_string = format!(" {} -> {}", test0.draw_calls, test1.draw_calls); 329 let composite_time_string = format!(" {:.2} -> {:.2}", composite_time0, composite_time1); 330 let paint_time_string = format!(" {:.2} -> {:.2}", paint_time0, paint_time1); 331 332 println!( 333 "| {:47}|{}{:14}{}|{}{:18}{}|{}{:18}{}|", 334 test_name, 335 draw_calls_color, 336 draw_call_string, 337 COLOR_DEFAULT, 338 composite_time_color, 339 composite_time_string, 340 COLOR_DEFAULT, 341 paint_time_color, 342 paint_time_string, 343 COLOR_DEFAULT 344 ); 345 } 346 347 print!("+------------------------------------------------"); 348 println!("+--------------+------------------+------------------+"); 349 }