program_cache.rs (12631B)
1 /* This Source Code Form is subject to the terms of the Mozilla Public 2 * License, v. 2.0. If a copy of the MPL was not distributed with this 3 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ 4 5 use nsstring::nsAString; 6 use rayon::ThreadPool; 7 use std::cell::RefCell; 8 use std::ffi::OsString; 9 use std::fs::{create_dir_all, read_dir, read_to_string, File}; 10 use std::io::{Error, ErrorKind}; 11 use std::io::{Read, Write}; 12 use std::path::{Path, PathBuf}; 13 use std::rc::Rc; 14 use std::sync::Arc; 15 use webrender::{ProgramBinary, ProgramCache, ProgramCacheObserver, ProgramSourceDigest}; 16 17 const MAX_LOAD_TIME_MS: u64 = 400; 18 19 fn hash_bytes(v: &[u8]) -> u64 { 20 use rustc_hash::FxHasher; 21 use std::hash::{Hash, Hasher}; 22 let mut state = FxHasher::default(); 23 v.hash(&mut state); 24 state.finish() 25 } 26 27 fn deserialize_program_binary(path: &Path) -> Result<Arc<ProgramBinary>, Error> { 28 let mut buf = vec![]; 29 let mut file = File::open(path)?; 30 file.read_to_end(&mut buf)?; 31 32 if buf.len() <= 8 + 4 { 33 return Err(Error::new(ErrorKind::InvalidData, "File size is too small")); 34 } 35 let magic = &buf[0..4]; 36 let hash = &buf[4..8 + 4]; 37 let data = &buf[8 + 4..]; 38 39 // Check if magic + version are correct. 40 let mv: u32 = bincode::deserialize(&magic).unwrap(); 41 if mv != MAGIC_AND_VERSION { 42 return Err(Error::new( 43 ErrorKind::InvalidData, 44 "File data is invalid (magic+version)", 45 )); 46 } 47 48 // Check if hash is correct 49 let hash: u64 = bincode::deserialize(&hash).unwrap(); 50 let hash_data = hash_bytes(&data); 51 if hash != hash_data { 52 return Err(Error::new(ErrorKind::InvalidData, "File data is invalid (hash)")); 53 } 54 55 // Deserialize ProgramBinary 56 let binary = match bincode::deserialize(&data) { 57 Ok(binary) => binary, 58 Err(_) => { 59 return Err(Error::new( 60 ErrorKind::InvalidData, 61 "Failed to deserialize ProgramBinary", 62 )) 63 }, 64 }; 65 66 Ok(Arc::new(binary)) 67 } 68 69 #[cfg(target_os = "windows")] 70 fn get_cache_path_from_prof_path(prof_path: &nsAString) -> Option<PathBuf> { 71 if prof_path.is_empty() { 72 // Empty means that we do not use disk cache. 73 return None; 74 } 75 76 use std::os::windows::prelude::*; 77 78 let prof_path = OsString::from_wide(prof_path.as_ref()); 79 let mut cache_path = PathBuf::from(&prof_path); 80 cache_path.push("shader-cache"); 81 82 Some(cache_path) 83 } 84 85 #[cfg(not(target_os = "windows"))] 86 fn get_cache_path_from_prof_path(prof_path: &nsAString) -> Option<PathBuf> { 87 if prof_path.is_empty() { 88 // Empty means that we do not use disk cache. 89 return None; 90 } 91 92 let utf8 = String::from_utf16(prof_path.as_ref()).unwrap(); 93 let prof_path = OsString::from(utf8); 94 let mut cache_path = PathBuf::from(&prof_path); 95 cache_path.push("shader-cache"); 96 97 Some(cache_path) 98 } 99 100 struct WrProgramBinaryDiskCache { 101 cache_path: PathBuf, 102 workers: Arc<ThreadPool>, 103 cached_shader_filenames: Vec<OsString>, 104 } 105 106 // Magic number + version. Increment the version when the binary format changes. 107 const MAGIC: u32 = 0xB154AD30; // BI-SHADE + version. 108 const VERSION: u32 = 2; 109 const MAGIC_AND_VERSION: u32 = MAGIC + VERSION; 110 111 const WHITELIST_FILENAME: &str = "startup_shaders"; 112 const WHITELIST_SEPARATOR: &str = "\n"; 113 114 /// Helper to convert a closure returning a `Result` to one that returns void. 115 /// This allows the enclosed code to use the question-mark operator in a 116 /// context where the calling function doesn't expect a `Result`. 117 #[allow(unused_must_use)] 118 fn result_to_void<F: FnOnce() -> Result<(), ()>>(f: F) { 119 f(); 120 } 121 122 impl WrProgramBinaryDiskCache { 123 #[allow(dead_code)] 124 fn new(cache_path: PathBuf, workers: &Arc<ThreadPool>) -> Self { 125 WrProgramBinaryDiskCache { 126 cache_path, 127 workers: Arc::clone(workers), 128 cached_shader_filenames: Vec::new(), 129 } 130 } 131 132 /// Saves the specified binaries to the on-disk shader cache. 133 fn save_shaders_to_disk(&mut self, entries: Vec<Arc<ProgramBinary>>) { 134 info!("Saving binaries to on-disk shader cache"); 135 136 // Write the entries to disk on a worker thread. 137 for entry in entries { 138 let file_name = entry.source_digest().to_string(); 139 let file_path = self.cache_path.join(&file_name); 140 141 self.workers.spawn(move || { 142 result_to_void(move || { 143 info!("Writing shader: {}", file_name); 144 145 use std::time::Instant; 146 let start = Instant::now(); 147 148 let data: Vec<u8> = 149 bincode::serialize(&*entry).map_err(|e| error!("shader-cache: Failed to serialize: {}", e))?; 150 151 let mut file = 152 File::create(&file_path).map_err(|e| error!("shader-cache: Failed to create file: {}", e))?; 153 154 // Write magic + version. 155 let mv = MAGIC_AND_VERSION; 156 let mv = bincode::serialize(&mv).unwrap(); 157 assert!(mv.len() == 4); 158 file.write_all(&mv) 159 .map_err(|e| error!("shader-cache: Failed to write magic + version: {}", e))?; 160 161 // Write hash 162 let hash = hash_bytes(&data); 163 let hash = bincode::serialize(&hash).unwrap(); 164 assert!(hash.len() == 8); 165 file.write_all(&hash) 166 .map_err(|e| error!("shader-cache: Failed to write hash: {}", e))?; 167 168 // Write serialized data 169 file.write_all(&data) 170 .map_err(|e| error!("shader-cache: Failed to write program binary: {}", e))?; 171 172 info!("Wrote shader {} in {:?}", file_name, start.elapsed()); 173 Ok(()) 174 }) 175 }); 176 } 177 } 178 179 /// Writes the whitelist containing the set of startup shaders to disk. 180 fn set_startup_shaders(&mut self, entries: Vec<Arc<ProgramBinary>>) { 181 let whitelist = entries 182 .iter() 183 .map(|e| e.source_digest().to_string()) 184 .collect::<Vec<String>>() 185 .join(WHITELIST_SEPARATOR); 186 187 let mut whitelist_path = self.cache_path.clone(); 188 whitelist_path.push(WHITELIST_FILENAME); 189 self.workers.spawn(move || { 190 result_to_void(move || { 191 info!("Writing startup shader whitelist"); 192 File::create(&whitelist_path) 193 .and_then(|mut file| file.write_all(whitelist.as_bytes())) 194 .map_err(|e| error!("shader-cache: Failed to write startup whitelist: {}", e))?; 195 Ok(()) 196 }) 197 }); 198 } 199 200 pub fn try_load_shader_from_disk(&mut self, filename: &str, program_cache: &Rc<ProgramCache>) { 201 if let Some(index) = self.cached_shader_filenames.iter().position(|e| e == filename) { 202 let mut path = self.cache_path.clone(); 203 path.push(filename); 204 205 self.cached_shader_filenames.swap_remove(index); 206 207 info!("Loading shader: {}", filename); 208 209 match deserialize_program_binary(&path) { 210 Ok(program) => { 211 program_cache.load_program_binary(program); 212 }, 213 Err(err) => { 214 error!("shader-cache: Failed to deserialize program binary: {}", err); 215 }, 216 }; 217 } else { 218 info!("shader-cache: Program binary not found in disk cache"); 219 } 220 } 221 222 pub fn try_load_startup_shaders_from_disk(&mut self, program_cache: &Rc<ProgramCache>) { 223 use std::time::Instant; 224 let start = Instant::now(); 225 226 // Load and parse the whitelist if it exists 227 let mut whitelist_path = self.cache_path.clone(); 228 whitelist_path.push(WHITELIST_FILENAME); 229 let whitelist = match read_to_string(&whitelist_path) { 230 Ok(whitelist) => whitelist 231 .split(WHITELIST_SEPARATOR) 232 .map(|s| s.to_string()) 233 .collect::<Vec<String>>(), 234 Err(err) => { 235 info!("shader-cache: Could not read startup whitelist: {}", err); 236 Vec::new() 237 }, 238 }; 239 info!("Loaded startup shader whitelist in {:?}", start.elapsed()); 240 241 self.cached_shader_filenames = read_dir(&self.cache_path) 242 .map_err(|err| { 243 error!( 244 "shader-cache: Error reading directory whilst loading startup shaders: {}", 245 err 246 ) 247 }) 248 .into_iter() 249 .flatten() 250 .filter_map(Result::ok) 251 .map(|entry| entry.file_name()) 252 .filter(|filename| filename != WHITELIST_FILENAME) 253 .collect::<Vec<OsString>>(); 254 255 // Load whitelisted program binaries if they exist 256 for entry in &whitelist { 257 self.try_load_shader_from_disk(&entry, program_cache); 258 259 let elapsed = start.elapsed(); 260 info!("Loaded shader in {:?}", elapsed); 261 let elapsed_ms = (elapsed.as_secs() * 1_000) + elapsed.subsec_millis() as u64; 262 263 if elapsed_ms > MAX_LOAD_TIME_MS { 264 // Loading the startup shaders is taking too long, so bail out now. 265 // Additionally clear the list of remaining shaders cached on disk, 266 // so that we do not attempt to load any on demand during rendering. 267 error!("shader-cache: Timed out before finishing loads"); 268 self.cached_shader_filenames.clear(); 269 break; 270 } 271 } 272 } 273 } 274 275 pub struct WrProgramCacheObserver { 276 disk_cache: Rc<RefCell<WrProgramBinaryDiskCache>>, 277 } 278 279 impl WrProgramCacheObserver { 280 #[allow(dead_code)] 281 fn new(disk_cache: Rc<RefCell<WrProgramBinaryDiskCache>>) -> Self { 282 WrProgramCacheObserver { disk_cache } 283 } 284 } 285 286 impl ProgramCacheObserver for WrProgramCacheObserver { 287 fn save_shaders_to_disk(&self, entries: Vec<Arc<ProgramBinary>>) { 288 self.disk_cache.borrow_mut().save_shaders_to_disk(entries); 289 } 290 291 fn set_startup_shaders(&self, entries: Vec<Arc<ProgramBinary>>) { 292 self.disk_cache.borrow_mut().set_startup_shaders(entries); 293 } 294 295 fn try_load_shader_from_disk(&self, digest: &ProgramSourceDigest, program_cache: &Rc<ProgramCache>) { 296 let filename = digest.to_string(); 297 self.disk_cache 298 .borrow_mut() 299 .try_load_shader_from_disk(&filename, program_cache); 300 } 301 302 fn notify_program_binary_failed(&self, _program_binary: &Arc<ProgramBinary>) { 303 error!("shader-cache: Failed program_binary"); 304 } 305 } 306 307 pub struct WrProgramCache { 308 pub program_cache: Rc<ProgramCache>, 309 disk_cache: Option<Rc<RefCell<WrProgramBinaryDiskCache>>>, 310 } 311 312 impl WrProgramCache { 313 pub fn new(prof_path: &nsAString, workers: &Arc<ThreadPool>) -> Self { 314 let cache_path = get_cache_path_from_prof_path(prof_path); 315 let use_disk_cache = cache_path.as_ref().map_or(false, |p| create_dir_all(p).is_ok()); 316 let (disk_cache, program_cache_observer) = if use_disk_cache { 317 let cache = Rc::new(RefCell::new(WrProgramBinaryDiskCache::new( 318 cache_path.unwrap(), 319 workers, 320 ))); 321 let obs = Box::new(WrProgramCacheObserver::new(Rc::clone(&cache))) as Box<dyn ProgramCacheObserver>; 322 (Some(cache), Some(obs)) 323 } else { 324 (None, None) 325 }; 326 let program_cache = ProgramCache::new(program_cache_observer); 327 328 WrProgramCache { 329 program_cache, 330 disk_cache, 331 } 332 } 333 334 pub fn rc_get(&self) -> &Rc<ProgramCache> { 335 &self.program_cache 336 } 337 338 pub fn try_load_startup_shaders_from_disk(&self) { 339 if let Some(ref disk_cache) = self.disk_cache { 340 disk_cache 341 .borrow_mut() 342 .try_load_startup_shaders_from_disk(&self.program_cache); 343 } else { 344 error!("shader-cache: Shader disk cache is not supported"); 345 } 346 } 347 } 348 349 pub fn remove_disk_cache(prof_path: &nsAString) -> Result<(), Error> { 350 use std::time::Instant; 351 352 if let Some(cache_path) = get_cache_path_from_prof_path(prof_path) { 353 if cache_path.exists() { 354 let start = Instant::now(); 355 remove_dir_all::remove_dir_all(&cache_path)?; 356 info!("removed all disk cache shaders in {:?}", start.elapsed()); 357 } 358 } 359 Ok(()) 360 }