main.rs (21966B)
1 use std::{ 2 borrow::Cow, 3 collections::{BTreeMap, BTreeSet}, 4 env::set_current_dir, 5 path::PathBuf, 6 process::{ExitCode, Stdio}, 7 }; 8 9 use clap::Parser; 10 use ezcmd::EasyCommand; 11 use itertools::Itertools; 12 use joinery::JoinableIterator; 13 use miette::{ensure, miette, Context, Diagnostic, IntoDiagnostic, Report, SourceSpan}; 14 use regex::Regex; 15 16 use crate::{ 17 fs::{create_dir_all, remove_file, FileRoot}, 18 process::which, 19 }; 20 21 mod fs; 22 mod process; 23 mod test_split; 24 25 /// Vendor WebGPU CTS tests from a local Git checkout of [our `gpuweb/cts` fork]. 26 /// 27 /// WPT tests are generated into `testing/web-platform/mozilla/tests/webgpu/`. If the set of tests 28 /// changes upstream, make sure that the generated output still matches up with test expectation 29 /// metadata in `testing/web-platform/mozilla/meta/webgpu/`. 30 /// 31 /// [our `gpuweb/cts` fork]: https://github.com/mozilla/gpuweb-cts 32 #[derive(Debug, Parser)] 33 struct CliArgs { 34 /// A path to the top-level directory of your WebGPU CTS checkout. 35 cts_checkout_path: PathBuf, 36 } 37 38 fn main() -> ExitCode { 39 env_logger::builder() 40 .filter_level(log::LevelFilter::Info) 41 .parse_default_env() 42 .init(); 43 44 let args = CliArgs::parse(); 45 46 match run(args) { 47 Ok(()) => ExitCode::SUCCESS, 48 Err(e) => { 49 log::error!("{e:?}"); 50 ExitCode::FAILURE 51 } 52 } 53 } 54 55 fn run(args: CliArgs) -> miette::Result<()> { 56 let CliArgs { cts_checkout_path } = args; 57 58 let cts_ckt = FileRoot::new("cts", cts_checkout_path).unwrap(); 59 60 let npm_bin = which("npm", "NPM binary")?; 61 62 let node_bin = which("node", "Node.js binary")?; 63 64 set_current_dir(&*cts_ckt) 65 .into_diagnostic() 66 .wrap_err("failed to change working directory to CTS checkout")?; 67 log::debug!("changed CWD to {cts_ckt}"); 68 69 let mut npm_ci_cmd = EasyCommand::simple(&npm_bin, ["ci"]); 70 log::info!( 71 "ensuring a clean {} directory with {npm_ci_cmd}…", 72 cts_ckt.child("node_modules"), 73 ); 74 npm_ci_cmd.run().into_diagnostic()?; 75 76 let test_listing_join_handle = { 77 let mut cmd = EasyCommand::new_with(node_bin, |cmd| { 78 cmd.args(["tools/run_node", "--list", "webgpu:*"]) 79 .stderr(Stdio::inherit()) 80 }); 81 log::info!("requesting exhaustive list of tests in a separate thread using {cmd}…"); 82 std::thread::spawn(move || { 83 let stdout = cmd.output().into_diagnostic()?.stdout; 84 85 String::from_utf8(stdout) 86 .into_diagnostic() 87 .context("failed to read output of exhaustive test listing command") 88 }) 89 }; 90 91 let out_wpt_dir = cts_ckt.regen_dir("out-wpt", |out_wpt_dir| { 92 let mut npm_run_wpt_cmd = EasyCommand::simple(&npm_bin, ["run", "wpt"]); 93 log::info!("generating WPT test cases into {out_wpt_dir} with {npm_run_wpt_cmd}…"); 94 npm_run_wpt_cmd.run().into_diagnostic() 95 })?; 96 97 let cts_https_html_path = out_wpt_dir.child("cts-withsomeworkers.https.html"); 98 99 { 100 for file_name in ["cts-chunked2sec.https.html", "cts.https.html"] { 101 let file_name = out_wpt_dir.child(file_name); 102 log::info!("removing extraneous {file_name}…"); 103 remove_file(&*file_name)?; 104 } 105 } 106 107 log::info!("analyzing {cts_https_html_path}…"); 108 let cts_https_html_content = fs::read_to_string(&*cts_https_html_path)?; 109 let cts_boilerplate_short_timeout; 110 let cts_boilerplate_long_timeout; 111 let cts_cases; 112 #[derive(Clone, Copy, Debug, Eq, Ord, PartialEq, PartialOrd)] 113 enum WorkerType { 114 Dedicated, 115 Service, 116 Shared, 117 } 118 119 impl WorkerType { 120 const DEDICATED: &str = "dedicated"; 121 const SERVICE: &str = "service"; 122 const SHARED: &str = "shared"; 123 124 pub(crate) fn new(s: &str) -> Option<Self> { 125 match s { 126 Self::DEDICATED => Some(WorkerType::Dedicated), 127 Self::SERVICE => Some(WorkerType::Service), 128 Self::SHARED => Some(WorkerType::Shared), 129 _ => None, 130 } 131 } 132 133 pub(crate) fn as_str(&self) -> &'static str { 134 match self { 135 Self::Dedicated => Self::DEDICATED, 136 Self::Service => Self::SERVICE, 137 Self::Shared => Self::SHARED, 138 } 139 } 140 } 141 { 142 { 143 let (boilerplate, cases_start) = { 144 let cases_start_idx = cts_https_html_content 145 .find("<meta name=variant") 146 .ok_or_else(|| miette!("no test cases found; this is unexpected!"))?; 147 cts_https_html_content.split_at(cases_start_idx) 148 }; 149 150 { 151 if !boilerplate.is_empty() { 152 #[derive(Debug, Diagnostic, thiserror::Error)] 153 #[error("last character before test cases was not a newline; bug, or weird?")] 154 #[diagnostic(severity("warning"))] 155 struct Oops { 156 #[label( 157 "this character ({:?}) was expected to be a newline, so that {}", 158 source_code.chars().last().unwrap(), 159 "the test spec. following it is on its own line" 160 )] 161 span: SourceSpan, 162 #[source_code] 163 source_code: String, 164 } 165 ensure!( 166 boilerplate.ends_with('\n'), 167 Oops { 168 span: SourceSpan::from(0..boilerplate.len()), 169 source_code: cts_https_html_content, 170 } 171 ); 172 } 173 174 // NOTE: Adding `_mozilla` is necessary because [that's how it's mounted][source]. 175 // 176 // [source]: https://searchfox.org/mozilla-central/rev/cd2121e7d83af1b421c95e8c923db70e692dab5f/testing/web-platform/mozilla/README#1-4] 177 log::info!(concat!( 178 " …fixing `script` paths in WPT boilerplate ", 179 "so they work as Mozilla-private WPT tests…" 180 )); 181 let expected_wpt_script_tag = 182 "<script type=module src=/webgpu/common/runtime/wpt.js></script>"; 183 ensure!( 184 boilerplate.contains(expected_wpt_script_tag), 185 format!( 186 concat!( 187 "failed to find expected `script` tag for `wpt.js` ", 188 "({:?}); did something change upstream?" 189 ), 190 expected_wpt_script_tag 191 ), 192 ); 193 let mut boilerplate = boilerplate.replacen( 194 expected_wpt_script_tag, 195 "<script type=module src=/_mozilla/webgpu/common/runtime/wpt.js></script>", 196 1, 197 ); 198 199 cts_boilerplate_short_timeout = boilerplate.clone(); 200 201 let timeout_insert_idx = { 202 let meta_charset_utf8 = "\n<meta charset=utf-8>\n"; 203 let meta_charset_utf8_idx = 204 boilerplate.find(meta_charset_utf8).ok_or_else(|| { 205 miette!( 206 "could not find {:?} in document; did something change upstream?", 207 meta_charset_utf8 208 ) 209 })?; 210 meta_charset_utf8_idx + meta_charset_utf8.len() 211 }; 212 boilerplate.insert_str( 213 timeout_insert_idx, 214 concat!( 215 r#"<meta name="timeout" content="long">"#, 216 " <!-- TODO: narrow to only where it's needed, see ", 217 "https://bugzilla.mozilla.org/show_bug.cgi?id=1850537", 218 " -->\n" 219 ), 220 ); 221 cts_boilerplate_long_timeout = boilerplate 222 }; 223 224 log::info!(" …parsing test variants in {cts_https_html_path}…"); 225 let mut parsing_failed = false; 226 let meta_variant_regex = Regex::new(concat!( 227 "^", 228 "<meta name=variant content='", 229 r"\?", 230 r"(:?worker=(?P<worker_type>\w+)&)?", 231 r"q=(?P<test_path>[^']*?:\*)", 232 "'>", 233 "$" 234 )) 235 .unwrap(); 236 cts_cases = cases_start 237 .split_terminator('\n') 238 .filter_map(|line| { 239 if line.is_empty() { 240 // Empty separator lines exist between groups of different `worker_type`s. 241 return None; 242 } 243 let captures = meta_variant_regex.captures(line); 244 if captures.is_none() { 245 parsing_failed = true; 246 log::error!("line is not a test case: {line:?}"); 247 } 248 let captures = captures?; 249 250 let test_path = captures["test_path"].to_owned(); 251 252 let worker_type = 253 captures 254 .name("worker_type") 255 .map(|wt| wt.as_str()) 256 .and_then(|wt| match WorkerType::new(wt) { 257 Some(wt) => Some(wt), 258 None => { 259 parsing_failed = true; 260 log::error!("unrecognized `worker` type {wt:?}"); 261 None 262 } 263 }); 264 265 Some((test_path, worker_type, line)) 266 }) 267 .collect::<Vec<_>>(); 268 ensure!( 269 !parsing_failed, 270 "one or more test case lines failed to parse, fix it and try again" 271 ); 272 }; 273 log::trace!("\"original\" HTML boilerplate:\n\n{cts_boilerplate_short_timeout}"); 274 275 ensure!( 276 !cts_cases.is_empty(), 277 "no test cases found; this is unexpected!" 278 ); 279 log::info!(" …found {} test cases", cts_cases.len()); 280 } 281 282 let test_listing_buf; 283 let mut tests_to_split = { 284 log::info!("generating index of tests to split…"); 285 286 let test_split_config = { 287 use test_split::*; 288 [( 289 "webgpu:api,operation,command_buffer,image_copy:mip_levels", 290 Config { 291 new_sibling_basename: "image_copy__mip_levels", 292 split_by: SplitBy::first_param( 293 "initMethod", 294 SplitParamsTo::SeparateTestsInSameFile, 295 ), 296 }, 297 )] 298 }; 299 300 let mut tests_to_split = test_split_config 301 .into_iter() 302 .map(|(test_path, config)| (test_path, test_split::Entry::from_config(config))) 303 .collect::<BTreeMap<_, _>>(); 304 305 log::debug!("blocking on list of tests…"); 306 test_listing_buf = test_listing_join_handle 307 .join() 308 .expect("failed to get value from test listing thread") 309 .unwrap(); 310 log::info!("building index from list of tests…"); 311 for full_path in test_listing_buf.lines() { 312 let (subtest_path, params) = split_at_nth_colon(2, full_path) 313 .wrap_err_with(|| "failed to parse configured split entry")?; 314 if let Some(entry) = tests_to_split.get_mut(subtest_path) { 315 entry.process_listing_line(params)?; 316 } 317 } 318 test_split::assert_seen("test listing output", tests_to_split.iter(), |seen| { 319 &seen.listing 320 }); 321 322 tests_to_split 323 }; 324 325 cts_ckt.regen_dir(out_wpt_dir.join("cts"), |cts_tests_dir| { 326 log::info!("re-distributing tests into single file per test path…"); 327 let mut failed_writing = false; 328 let mut cts_cases_by_spec_file_dir = BTreeMap::<_, BTreeMap<_, BTreeSet<_>>>::new(); 329 for (path, worker_type, meta) in cts_cases { 330 macro_rules! insert { 331 ($path:expr, $meta:expr $(,)?) => {{ 332 let dir = cts_tests_dir.child($path); 333 if !cts_cases_by_spec_file_dir 334 .entry(dir) 335 .or_default() 336 .entry(worker_type) 337 .or_default() 338 .insert($meta) 339 { 340 log::warn!("duplicate entry {meta:?} detected") 341 } 342 }}; 343 } 344 345 // Context: We want to mirror CTS upstream's `src/webgpu/**/*.spec.ts` paths as 346 // entire WPT tests, with each subtest being a WPT variant. Here's a diagram of 347 // a CTS path to explain why the logic below is correct: 348 // 349 // ```sh 350 // webgpu:this,is,the,spec.ts,file,path:test_in_file:… 351 // \____/ \___________________________/^\__________/ 352 // test `*.spec.ts` file path | | 353 // \__________________________________/| | 354 // | | | 355 // We want this… | …but not this. CTS upstream generates 356 // | this too, but we don't want to divide 357 // second ':' character here---/ here (yet). 358 // ``` 359 let (test_path, _cases) = match split_at_nth_colon(2, &path) { 360 Ok(ok) => ok, 361 Err(e) => { 362 failed_writing = true; 363 log::error!("{e}"); 364 continue; 365 } 366 }; 367 let (test_group_path, _test_name) = test_path.rsplit_once(':').unwrap(); 368 let mut test_group_path_components = test_group_path.split([':', ',']); 369 370 if let Some(entry) = tests_to_split.get_mut(test_path) { 371 let test_split::Entry { seen, ref config } = entry; 372 let test_split::Config { 373 new_sibling_basename, 374 split_by, 375 } = config; 376 377 let file_path = { 378 test_group_path_components.next_back(); 379 test_group_path_components 380 .chain([*new_sibling_basename]) 381 .join_with("/") 382 .to_string() 383 }; 384 385 seen.wpt_files = true; 386 387 match split_by { 388 test_split::SplitBy::FirstParam { 389 expected_name, 390 split_to, 391 observed_values, 392 } => match split_to { 393 test_split::SplitParamsTo::SeparateTestsInSameFile => { 394 for value in observed_values { 395 let new_meta = meta.replace( 396 &*path, 397 &format!("{test_path}:{expected_name}={value};*"), 398 ); 399 assert_ne!(meta, new_meta); 400 insert!(&file_path, new_meta.into()); 401 } 402 } 403 }, 404 } 405 } else { 406 insert!( 407 &test_group_path_components.join_with("/").to_string(), 408 meta.into() 409 ) 410 }; 411 } 412 413 test_split::assert_seen("WPT test output", tests_to_split.iter(), |seen| { 414 &seen.wpt_files 415 }); 416 417 struct WptEntry<'a> { 418 cases: BTreeSet<Cow<'a, str>>, 419 timeout_length: TimeoutLength, 420 } 421 #[derive(Clone, Copy, Debug)] 422 enum TimeoutLength { 423 Short, 424 Long, 425 } 426 let split_cases = { 427 let mut split_cases = BTreeMap::new(); 428 fn insert_with_default_name<'a>( 429 split_cases: &mut BTreeMap<fs::Child<'a>, WptEntry<'a>>, 430 spec_file_dir: fs::Child<'a>, 431 cases: BTreeMap<Option<WorkerType>, BTreeSet<Cow<'a, str>>>, 432 timeout_length: TimeoutLength, 433 ) { 434 for (worker_type, cases) in cases { 435 let file_stem = worker_type.map(|wt| wt.as_str()).unwrap_or("cts"); 436 let path = spec_file_dir.child(format!("{file_stem}.https.html")); 437 assert!(split_cases 438 .insert( 439 path, 440 WptEntry { 441 cases, 442 timeout_length 443 } 444 ) 445 .is_none()); 446 } 447 } 448 { 449 let dld_path = 450 &cts_tests_dir.child("webgpu/api/validation/state/device_lost/destroy"); 451 let (spec_file_dir, cases) = cts_cases_by_spec_file_dir 452 .remove_entry(dld_path) 453 .expect("no `device_lost/destroy` tests found; did they move?"); 454 insert_with_default_name( 455 &mut split_cases, 456 spec_file_dir, 457 cases, 458 TimeoutLength::Short, 459 ); 460 } 461 for (spec_file_dir, cases) in cts_cases_by_spec_file_dir { 462 insert_with_default_name( 463 &mut split_cases, 464 spec_file_dir, 465 cases, 466 TimeoutLength::Long, 467 ); 468 } 469 split_cases 470 }; 471 472 for (path, entry) in split_cases { 473 let dir = path.parent().expect("no parent found for "); 474 match create_dir_all(dir) { 475 Ok(()) => log::trace!("made directory {}", dir.display()), 476 Err(e) => { 477 failed_writing = true; 478 log::error!("{e:#}"); 479 continue; 480 } 481 } 482 let file_contents = { 483 let WptEntry { 484 cases, 485 timeout_length, 486 } = entry; 487 let content = match timeout_length { 488 TimeoutLength::Short => &cts_boilerplate_short_timeout, 489 TimeoutLength::Long => &cts_boilerplate_long_timeout, 490 }; 491 let mut content = content.as_bytes().to_vec(); 492 for meta in cases { 493 content.extend(meta.as_bytes()); 494 content.extend(b"\n"); 495 } 496 content 497 }; 498 match fs::write(&path, &file_contents) 499 .wrap_err_with(|| miette!("failed to write output to path {path:?}")) 500 { 501 Ok(()) => log::debug!(" …wrote {path}"), 502 Err(e) => { 503 failed_writing = true; 504 log::error!("{e:#}"); 505 } 506 } 507 } 508 ensure!( 509 !failed_writing, 510 "failed to write one or more WPT test files; see above output for more details" 511 ); 512 log::debug!(" …finished writing new WPT test files!"); 513 514 log::info!(" …removing {cts_https_html_path}, now that it's been divided up…"); 515 remove_file(&cts_https_html_path)?; 516 517 log::info!("moving ready-to-go WPT test files into `cts`…"); 518 519 let webgpu_dir = out_wpt_dir.child("webgpu"); 520 let ready_to_go_tests = wax::Glob::new("**/*.{html,{any,sub,worker}.js}") 521 .unwrap() 522 .walk(&webgpu_dir) 523 .map_ok(|entry| webgpu_dir.child(entry.into_path())) 524 .collect::<Result<Vec<_>, _>>() 525 .map_err(Report::msg) 526 .wrap_err_with(|| { 527 format!("failed to walk {webgpu_dir} for ready-to-go WPT test files") 528 })?; 529 530 log::trace!(" …will move the following: {ready_to_go_tests:#?}"); 531 532 for file in ready_to_go_tests { 533 let path_relative_to_webgpu_dir = file.strip_prefix(&webgpu_dir).unwrap(); 534 let dst_path = cts_tests_dir.child(path_relative_to_webgpu_dir); 535 log::trace!("…moving {file} to {dst_path}…"); 536 ensure!( 537 !fs::try_exists(&dst_path)?, 538 "internal error: duplicate path found while moving ready-to-go test {} to {}", 539 file, 540 dst_path, 541 ); 542 fs::create_dir_all(dst_path.parent().unwrap()).wrap_err_with(|| { 543 format!( 544 concat!( 545 "failed to create destination parent dirs. ", 546 "while recursively moving from {} to {}", 547 ), 548 file, dst_path, 549 ) 550 })?; 551 fs::rename(&file, &dst_path) 552 .wrap_err_with(|| format!("failed to move {file} to {dst_path}"))?; 553 } 554 log::debug!(" …finished moving ready-to-go WPT test files"); 555 556 Ok(()) 557 })?; 558 559 log::info!("All done! Now get your CTS _ON_! :)"); 560 561 Ok(()) 562 } 563 564 fn split_at_nth_colon(nth: usize, path: &str) -> miette::Result<(&str, &str)> { 565 path.match_indices(':') 566 .nth(nth) 567 .map(|(idx, s)| (&path[..idx], &path[idx + s.len()..])) 568 .ok_or_else(move || { 569 miette::diagnostic!("failed to split at colon {nth} from CTS path `{path}`").into() 570 }) 571 }