serve.py (56074B)
1 # mypy: allow-untyped-defs 2 3 import abc 4 import argparse 5 import importlib 6 import json 7 import logging 8 import multiprocessing 9 import os 10 import platform 11 import subprocess 12 import sys 13 import threading 14 import time 15 import traceback 16 import urllib 17 import uuid 18 from collections import defaultdict, OrderedDict 19 from io import IOBase 20 from itertools import chain, product 21 from html5lib import html5parser 22 from typing import ClassVar, List, Optional, Set, Tuple 23 24 from localpaths import repo_root # type: ignore 25 26 from manifest.sourcefile import read_script_metadata, js_meta_re, parse_variants # type: ignore 27 from wptserve import server as wptserve, handlers 28 from wptserve import stash 29 from wptserve import config 30 from wptserve.handlers import filesystem_path, wrap_pipeline 31 from wptserve.response import ResponseHeaders 32 from wptserve.utils import get_port, HTTPException, http2_compatible 33 from pywebsocket3 import standalone as pywebsocket 34 35 36 EDIT_HOSTS_HELP = ("Please ensure all the necessary WPT subdomains " 37 "are mapped to a loopback device in /etc/hosts.\n" 38 "See https://web-platform-tests.org/running-tests/from-local-system.html#system-setup " 39 "for instructions.") 40 41 42 def replace_end(s, old, new): 43 """ 44 Given a string `s` that ends with `old`, replace that occurrence of `old` 45 with `new`. 46 """ 47 assert s.endswith(old) 48 return s[:-len(old)] + new 49 50 51 def domains_are_distinct(a, b): 52 a_parts = a.split(".") 53 b_parts = b.split(".") 54 min_length = min(len(a_parts), len(b_parts)) 55 slice_index = -1 * min_length 56 57 return a_parts[slice_index:] != b_parts[slice_index:] 58 59 60 def inject_script(html, script_tag): 61 # Tokenize and find the position of the first content (e.g. after the 62 # doctype, html, and head opening tags if present but before any other tags). 63 token_types = html5parser.tokenTypes 64 after_tags = {"html", "head"} 65 before_tokens = {token_types["EndTag"], token_types["EmptyTag"], 66 token_types["Characters"]} 67 error_tokens = {token_types["ParseError"]} 68 69 tokenizer = html5parser._tokenizer.HTMLTokenizer(html) 70 stream = tokenizer.stream 71 offset = 0 72 error = False 73 for item in tokenizer: 74 if item["type"] == token_types["StartTag"]: 75 if not item["name"].lower() in after_tags: 76 break 77 elif item["type"] in before_tokens: 78 break 79 elif item["type"] in error_tokens: 80 error = True 81 break 82 offset = stream.chunkOffset 83 else: 84 error = True 85 86 if not error and stream.prevNumCols or stream.prevNumLines: 87 # We're outside the first chunk, so we don't know what to do 88 error = True 89 90 if error: 91 return html 92 else: 93 return html[:offset] + script_tag + html[offset:] 94 95 96 class WrapperHandler(metaclass=abc.ABCMeta): 97 98 headers: ClassVar[List[Tuple[str, str]]] = [] 99 100 def __init__(self, base_path=None, url_base="/"): 101 self.base_path = base_path 102 self.url_base = url_base 103 self.handler = handlers.handler(self.handle_request) 104 105 def __call__(self, request, response): 106 self.handler(request, response) 107 108 def handle_request(self, request, response): 109 headers = self.headers + handlers.load_headers( 110 request, self._get_filesystem_path(request)) 111 for header_name, header_value in headers: 112 response.headers.set(header_name, header_value) 113 114 self.check_exposure(request) 115 116 path = self._get_path(request.url_parts.path, True) 117 query = request.url_parts.query 118 if query: 119 query = "?" + query 120 meta = "\n".join(self._get_meta(request)) 121 script = "\n".join(self._get_script(request)) 122 response.content = self.wrapper % {"meta": meta, "script": script, "path": path, "query": query} 123 wrap_pipeline(path, request, response) 124 125 def _get_path(self, path, resource_path): 126 """Convert the path from an incoming request into a path corresponding to an "unwrapped" 127 resource e.g. the file on disk that will be loaded in the wrapper. 128 129 :param path: Path from the HTTP request 130 :param resource_path: Boolean used to control whether to get the path for the resource that 131 this wrapper will load or the associated file on disk. 132 Typically these are the same but may differ when there are multiple 133 layers of wrapping e.g. for a .any.worker.html input the underlying disk file is 134 .any.js but the top level html file loads a resource with a 135 .any.worker.js extension, which itself loads the .any.js file. 136 If True return the path to the resource that the wrapper will load, 137 otherwise return the path to the underlying file on disk.""" 138 for item in self.path_replace: 139 if len(item) == 2: 140 src, dest = item 141 else: 142 assert len(item) == 3 143 src = item[0] 144 dest = item[2 if resource_path else 1] 145 if path.endswith(src): 146 path = replace_end(path, src, dest) 147 return path 148 149 def _get_filesystem_path(self, request): 150 """Get the path of the underlying resource file on disk.""" 151 return self._get_path(filesystem_path(self.base_path, request, self.url_base), False) 152 153 def _get_metadata(self, request): 154 """Get an iterator over script metadata based on // META comments in the 155 associated js file. 156 157 :param request: The Request being processed. 158 """ 159 path = self._get_filesystem_path(request) 160 try: 161 with open(path, "rb") as f: 162 yield from read_script_metadata(f, js_meta_re) 163 except OSError: 164 raise HTTPException(404) 165 166 def _get_meta(self, request): 167 """Get an iterator over strings to inject into the wrapper document 168 based on // META comments in the associated js file. 169 170 :param request: The Request being processed. 171 """ 172 for key, value in self._get_metadata(request): 173 replacement = self._meta_replacement(key, value) 174 if replacement: 175 yield replacement 176 177 def _get_script(self, request): 178 """Get an iterator over strings to inject into the wrapper document 179 based on // META comments in the associated js file. 180 181 :param request: The Request being processed. 182 """ 183 for key, value in self._get_metadata(request): 184 replacement = self._script_replacement(key, value) 185 if replacement: 186 yield replacement 187 188 @abc.abstractproperty 189 def path_replace(self): 190 # A list containing a mix of 2 item tuples with (input suffix, output suffix) 191 # and 3-item tuples with (input suffix, filesystem suffix, resource suffix) 192 # for the case where we want a different path in the generated resource to 193 # the actual path on the filesystem (e.g. when there is another handler 194 # that will wrap the file). 195 return None 196 197 @abc.abstractproperty 198 def wrapper(self): 199 # String template with variables path and meta for wrapper document 200 return None 201 202 @abc.abstractmethod 203 def _meta_replacement(self, key, value): 204 # Get the string to insert into the wrapper document, given 205 # a specific metadata key: value pair. 206 pass 207 208 def check_exposure(self, request): 209 # Raise an exception if this handler shouldn't be exposed after all. 210 pass 211 212 213 class HtmlWrapperHandler(WrapperHandler): 214 global_type: ClassVar[Optional[str]] = None 215 headers = [("Content-Type", "text/html")] 216 217 def check_exposure(self, request): 218 if self.global_type is not None: 219 global_variants = "" 220 for (key, value) in self._get_metadata(request): 221 if key == "global": 222 global_variants = value 223 break 224 225 if self.global_type not in parse_variants(global_variants): 226 raise HTTPException(404, "This test cannot be loaded in %s mode" % 227 self.global_type) 228 229 def _meta_replacement(self, key, value): 230 if key == "timeout": 231 if value == "long": 232 return '<meta name="timeout" content="long">' 233 if key == "title": 234 value = value.replace("&", "&").replace("<", "<") 235 return "<title>%s</title>" % value 236 return None 237 238 def _script_replacement(self, key, value): 239 if key == "script": 240 attribute = value.replace("&", "&").replace('"', """) 241 return '<script src="%s"></script>' % attribute 242 return None 243 244 245 class HtmlScriptInjectorHandlerWrapper: 246 def __init__(self, inject="", wrap=None): 247 self.inject = inject 248 self.wrap = wrap 249 250 def __call__(self, request, response): 251 self.wrap(request, response) 252 # If the response content type isn't html, don't modify it. 253 if not isinstance(response.headers, ResponseHeaders) or response.headers.get("Content-Type")[0] != b"text/html": 254 return response 255 256 # Skip injection on custom streaming responses. 257 if not isinstance(response.content, (bytes, str, IOBase)) and not hasattr(response, "read"): 258 return response 259 260 response.content = inject_script( 261 b"".join(response.iter_content(read_file=True)), 262 b"<script>\n" + 263 self.inject + b"\n" + 264 (b"// Remove the injected script tag from the DOM.\n" 265 b"document.currentScript.remove();\n" 266 b"</script>\n")) 267 return response 268 269 270 class WorkersHandler(HtmlWrapperHandler): 271 global_type = "dedicatedworker" 272 path_replace = [(".any.worker.html", ".any.js", ".any.worker.js"), 273 (".worker.html", ".worker.js")] 274 wrapper = """<!doctype html> 275 <meta charset=utf-8> 276 %(meta)s 277 <script src="/resources/testharness.js"></script> 278 <script src="/resources/testharnessreport.js"></script> 279 <div id=log></div> 280 <script> 281 fetch_tests_from_worker(new Worker("%(path)s%(query)s")); 282 </script> 283 """ 284 285 286 class WorkerModulesHandler(HtmlWrapperHandler): 287 global_type = "dedicatedworker-module" 288 path_replace = [(".any.worker-module.html", ".any.js", ".any.worker-module.js"), 289 (".worker.html", ".worker.js")] 290 wrapper = """<!doctype html> 291 <meta charset=utf-8> 292 %(meta)s 293 <script src="/resources/testharness.js"></script> 294 <script src="/resources/testharnessreport.js"></script> 295 <div id=log></div> 296 <script> 297 fetch_tests_from_worker(new Worker("%(path)s%(query)s", { type: "module" })); 298 </script> 299 """ 300 301 302 class WindowHandler(HtmlWrapperHandler): 303 path_replace = [(".window.html", ".window.js")] 304 wrapper = """<!doctype html> 305 <meta charset=utf-8> 306 %(meta)s 307 <script src="/resources/testharness.js"></script> 308 <script src="/resources/testharnessreport.js"></script> 309 %(script)s 310 <div id=log></div> 311 <script src="%(path)s"></script> 312 """ 313 314 class ExtensionHandler(HtmlWrapperHandler): 315 path_replace = [(".extension.html", ".extension.js")] 316 wrapper = """<!doctype html> 317 <meta charset=utf-8> 318 %(meta)s 319 <script src="/resources/testharness.js"></script> 320 <script src="/resources/testharnessreport.js"></script> 321 <script src="/resources/testdriver.js?feature=extensions"></script> 322 <script src="/resources/testdriver-vendor.js"></script> 323 <script src="/resources/web-extensions-helper.js"></script> 324 %(script)s 325 <div id=log></div> 326 <script src="%(path)s"></script> 327 """ 328 329 330 class WindowModulesHandler(HtmlWrapperHandler): 331 global_type = "window-module" 332 path_replace = [(".any.window-module.html", ".any.js")] 333 wrapper = """<!doctype html> 334 <meta charset=utf-8> 335 %(meta)s 336 <script src="/resources/testharness.js"></script> 337 <script src="/resources/testharnessreport.js"></script> 338 %(script)s 339 <div id=log></div> 340 <script type=module src="%(path)s"></script> 341 """ 342 343 344 class AnyHtmlHandler(HtmlWrapperHandler): 345 global_type = "window" 346 path_replace = [(".any.html", ".any.js")] 347 wrapper = """<!doctype html> 348 <meta charset=utf-8> 349 %(meta)s 350 <script> 351 self.GLOBAL = { 352 isWindow: function() { return true; }, 353 isWorker: function() { return false; }, 354 isShadowRealm: function() { return false; }, 355 }; 356 </script> 357 <script src="/resources/testharness.js"></script> 358 <script src="/resources/testharnessreport.js"></script> 359 %(script)s 360 <div id=log></div> 361 <script src="%(path)s"></script> 362 """ 363 364 365 class SharedWorkersHandler(HtmlWrapperHandler): 366 global_type = "sharedworker" 367 path_replace = [(".any.sharedworker.html", ".any.js", ".any.worker.js")] 368 wrapper = """<!doctype html> 369 <meta charset=utf-8> 370 %(meta)s 371 <script src="/resources/testharness.js"></script> 372 <script src="/resources/testharnessreport.js"></script> 373 <div id=log></div> 374 <script> 375 fetch_tests_from_worker(new SharedWorker("%(path)s%(query)s")); 376 </script> 377 """ 378 379 380 class SharedWorkerModulesHandler(HtmlWrapperHandler): 381 global_type = "sharedworker-module" 382 path_replace = [(".any.sharedworker-module.html", ".any.js", ".any.worker-module.js")] 383 wrapper = """<!doctype html> 384 <meta charset=utf-8> 385 %(meta)s 386 <script src="/resources/testharness.js"></script> 387 <script src="/resources/testharnessreport.js"></script> 388 <div id=log></div> 389 <script> 390 fetch_tests_from_worker(new SharedWorker("%(path)s%(query)s", { type: "module" })); 391 </script> 392 """ 393 394 395 class ServiceWorkersHandler(HtmlWrapperHandler): 396 global_type = "serviceworker" 397 path_replace = [(".any.serviceworker.html", ".any.js", ".any.worker.js")] 398 wrapper = """<!doctype html> 399 <meta charset=utf-8> 400 %(meta)s 401 <script src="/resources/testharness.js"></script> 402 <script src="/resources/testharnessreport.js"></script> 403 <div id=log></div> 404 <script> 405 (async function() { 406 const scope = 'does/not/exist'; 407 let reg = await navigator.serviceWorker.getRegistration(scope); 408 if (reg) await reg.unregister(); 409 reg = await navigator.serviceWorker.register("%(path)s%(query)s", {scope}); 410 fetch_tests_from_worker(reg.installing); 411 })(); 412 </script> 413 """ 414 415 416 class ServiceWorkerModulesHandler(HtmlWrapperHandler): 417 global_type = "serviceworker-module" 418 path_replace = [(".any.serviceworker-module.html", 419 ".any.js", ".any.worker-module.js")] 420 wrapper = """<!doctype html> 421 <meta charset=utf-8> 422 %(meta)s 423 <script src="/resources/testharness.js"></script> 424 <script src="/resources/testharnessreport.js"></script> 425 <div id=log></div> 426 <script> 427 (async function() { 428 const scope = 'does/not/exist'; 429 let reg = await navigator.serviceWorker.getRegistration(scope); 430 if (reg) await reg.unregister(); 431 reg = await navigator.serviceWorker.register( 432 "%(path)s%(query)s", 433 { scope, type: 'module' }, 434 ); 435 fetch_tests_from_worker(reg.installing); 436 })(); 437 </script> 438 """ 439 440 441 class ShadowRealmInWindowHandler(HtmlWrapperHandler): 442 global_type = "shadowrealm-in-window" 443 path_replace = [(".any.shadowrealm-in-window.html", ".any.js")] 444 445 wrapper = """<!doctype html> 446 <meta charset=utf-8> 447 %(meta)s 448 <script src="/resources/testharness.js"></script> 449 <script src="/resources/testharnessreport.js"></script> 450 <script src="/resources/testharness-shadowrealm-outer.js"></script> 451 <script> 452 (async function() { 453 const r = new ShadowRealm(); 454 await shadowRealmEvalAsync(r, ` 455 await import("/resources/testharness-shadowrealm-inner.js"); 456 await import("/resources/testharness.js"); 457 `); 458 r.evaluate("setShadowRealmGlobalProperties")(location.search, fetchAdaptor); 459 460 await shadowRealmEvalAsync(r, ` 461 %(script)s 462 await import("%(path)s"); 463 `); 464 465 await fetch_tests_from_shadow_realm(r); 466 done(); 467 })().catch(e => setup(() => { throw e; })); 468 </script> 469 """ 470 471 def _script_replacement(self, key, value): 472 if key == "script": 473 return 'await import("%s");' % value 474 return None 475 476 477 class ShadowRealmInShadowRealmHandler(HtmlWrapperHandler): 478 global_type = "shadowrealm-in-shadowrealm" 479 path_replace = [(".any.shadowrealm-in-shadowrealm.html", ".any.js")] 480 481 wrapper = """<!doctype html> 482 <meta charset=utf-8> 483 %(meta)s 484 <script src="/resources/testharness.js"></script> 485 <script src="/resources/testharnessreport.js"></script> 486 <script src="/resources/testharness-shadowrealm-outer.js"></script> 487 <script> 488 (async function() { 489 const outer = new ShadowRealm(); 490 outer.evaluate(` 491 var inner = new ShadowRealm(); 492 `); 493 await shadowRealmEvalAsync(outer, ` 494 await import("/resources/testharness-shadowrealm-outer.js"); 495 await shadowRealmEvalAsync(inner, \\` 496 await import("/resources/testharness-shadowrealm-inner.js"); 497 await import("/resources/testharness.js"); 498 \\`); 499 `); 500 501 outer.evaluate(` 502 inner.evaluate("setShadowRealmGlobalProperties") 503 `)(location.search, fetchAdaptor); 504 505 await shadowRealmEvalAsync(outer, ` 506 await shadowRealmEvalAsync(inner, \\` 507 %(script)s 508 await import("%(path)s"); 509 \\`); 510 `); 511 512 outer.evaluate(` 513 function begin_shadow_realm_tests(windowCallback) { 514 inner.evaluate("begin_shadow_realm_tests")(windowCallback); 515 } 516 `); 517 await fetch_tests_from_shadow_realm(outer); 518 done(); 519 })().catch(e => setup(() => { throw e; })); 520 </script> 521 """ 522 523 def _script_replacement(self, key, value): 524 if key == "script": 525 return 'await import("%s");' % value 526 return None 527 528 529 class ShadowRealmInDedicatedWorkerHandler(WorkersHandler): 530 global_type = "shadowrealm-in-dedicatedworker" 531 path_replace = [(".any.shadowrealm-in-dedicatedworker.html", 532 ".any.js", 533 ".any.worker-shadowrealm.js")] 534 535 536 class ShadowRealmInSharedWorkerHandler(SharedWorkersHandler): 537 global_type = "shadowrealm-in-sharedworker" 538 path_replace = [(".any.shadowrealm-in-sharedworker.html", 539 ".any.js", 540 ".any.worker-shadowrealm.js")] 541 542 543 class ShadowRealmInServiceWorkerHandler(ServiceWorkersHandler): 544 global_type = "shadowrealm-in-serviceworker" 545 path_replace = [(".https.any.shadowrealm-in-serviceworker.html", 546 ".any.js", 547 ".any.serviceworker-shadowrealm.js")] 548 549 550 class ShadowRealmInAudioWorkletHandler(HtmlWrapperHandler): 551 global_type = "shadowrealm-in-audioworklet" 552 path_replace = [(".https.any.shadowrealm-in-audioworklet.html", ".any.js", 553 ".any.audioworklet-shadowrealm.js")] 554 555 wrapper = """<!doctype html> 556 <meta charset=utf-8> 557 %(meta)s 558 <script src="/resources/testharness.js"></script> 559 <script src="/resources/testharnessreport.js"></script> 560 <script src="/resources/testharness-shadowrealm-outer.js"></script> 561 <script> 562 (async function() { 563 const context = new AudioContext(); 564 await context.audioWorklet.addModule( 565 "/resources/testharness-shadowrealm-outer.js"); 566 await context.audioWorklet.addModule( 567 "/resources/testharness-shadowrealm-audioworkletprocessor.js"); 568 await context.audioWorklet.addModule("%(path)s%(query)s"); 569 const node = new AudioWorkletNode(context, "test-runner"); 570 setupFakeFetchOverMessagePort(node.port); 571 fetch_tests_from_worker(node.port); 572 })(); 573 </script> 574 """ 575 576 577 class BaseWorkerHandler(WrapperHandler): 578 headers = [("Content-Type", "text/javascript")] 579 580 def _meta_replacement(self, key, value): 581 return None 582 583 @abc.abstractmethod 584 def _create_script_import(self, attribute): 585 # Take attribute (a string URL to a JS script) and return JS source to import the script 586 # into the worker. 587 pass 588 589 def _script_replacement(self, key, value): 590 if key == "script": 591 attribute = value.replace("\\", "\\\\").replace('"', '\\"') 592 return self._create_script_import(attribute) 593 if key == "title": 594 value = value.replace("\\", "\\\\").replace('"', '\\"') 595 return 'self.META_TITLE = "%s";' % value 596 return None 597 598 599 class ClassicWorkerHandler(BaseWorkerHandler): 600 path_replace = [(".any.worker.js", ".any.js")] 601 wrapper = """%(meta)s 602 self.GLOBAL = { 603 isWindow: function() { return false; }, 604 isWorker: function() { return true; }, 605 isShadowRealm: function() { return false; }, 606 }; 607 importScripts("/resources/testharness.js"); 608 %(script)s 609 importScripts("%(path)s"); 610 done(); 611 """ 612 613 def _create_script_import(self, attribute): 614 return 'importScripts("%s")' % attribute 615 616 617 class ModuleWorkerHandler(BaseWorkerHandler): 618 path_replace = [(".any.worker-module.js", ".any.js")] 619 wrapper = """%(meta)s 620 self.GLOBAL = { 621 isWindow: function() { return false; }, 622 isWorker: function() { return true; }, 623 isShadowRealm: function() { return false; }, 624 }; 625 import "/resources/testharness.js"; 626 %(script)s 627 import "%(path)s"; 628 done(); 629 """ 630 631 def _create_script_import(self, attribute): 632 return 'import "%s";' % attribute 633 634 635 class ShadowRealmWorkerWrapperHandler(BaseWorkerHandler): 636 path_replace = [(".any.worker-shadowrealm.js", ".any.js")] 637 wrapper = """%(meta)s 638 importScripts("/resources/testharness-shadowrealm-outer.js"); 639 (async function() { 640 const postMessageFunc = await getPostMessageFunc(); 641 try { 642 const r = new ShadowRealm(); 643 await shadowRealmEvalAsync(r, ` 644 await import("/resources/testharness-shadowrealm-inner.js"); 645 await import("/resources/testharness.js"); 646 `); 647 r.evaluate("setShadowRealmGlobalProperties")("%(query)s", fetchAdaptor); 648 649 await shadowRealmEvalAsync(r, ` 650 %(script)s 651 await import("%(path)s"); 652 `); 653 654 function forwardMessage(msgJSON) { 655 postMessageFunc(JSON.parse(msgJSON)); 656 } 657 r.evaluate('begin_shadow_realm_tests')(forwardMessage); 658 } catch (e) { 659 postMessageFunc(createSetupErrorResult(e)); 660 } 661 })(); 662 """ 663 664 def _create_script_import(self, attribute): 665 return 'await import("%s");' % attribute 666 667 668 class ShadowRealmServiceWorkerWrapperHandler(BaseWorkerHandler): 669 path_replace = [(".any.serviceworker-shadowrealm.js", ".any.js")] 670 wrapper = """%(meta)s 671 importScripts("/resources/testharness-shadowrealm-outer.js"); 672 673 (async function () { 674 const postMessageFunc = await getPostMessageFunc(); 675 try { 676 const r = new ShadowRealm(); 677 setupFakeDynamicImportInShadowRealm(r, fetchAdaptor); 678 679 await shadowRealmEvalAsync(r, ` 680 await fakeDynamicImport("/resources/testharness-shadowrealm-inner.js"); 681 await fakeDynamicImport("/resources/testharness.js"); 682 `); 683 r.evaluate("setShadowRealmGlobalProperties")("%(query)s", fetchAdaptor); 684 685 await shadowRealmEvalAsync(r, ` 686 %(script)s 687 await fakeDynamicImport("%(path)s"); 688 `); 689 690 function forwardMessage(msgJSON) { 691 postMessageFunc(JSON.parse(msgJSON)); 692 } 693 r.evaluate("begin_shadow_realm_tests")(forwardMessage); 694 } catch (e) { 695 postMessageFunc(createSetupErrorResult(e)); 696 } 697 })(); 698 """ 699 700 def _create_script_import(self, attribute): 701 return 'await fakeDynamicImport("%s");' % attribute 702 703 704 class ShadowRealmAudioWorkletWrapperHandler(BaseWorkerHandler): 705 path_replace = [(".any.audioworklet-shadowrealm.js", ".any.js")] 706 wrapper = """%(meta)s 707 TestRunner.prototype.createShadowRealmAndStartTests = async function() { 708 try { 709 const queryPart = import.meta.url.split('?')[1]; 710 const locationSearch = queryPart ? '?' + queryPart : ''; 711 712 const r = new ShadowRealm(); 713 const adaptor = this.fetchOverPortExecutor.bind(this); 714 setupFakeDynamicImportInShadowRealm(r, adaptor); 715 716 await shadowRealmEvalAsync(r, ` 717 await fakeDynamicImport("/resources/testharness-shadowrealm-inner.js"); 718 await fakeDynamicImport("/resources/testharness.js"); 719 `); 720 r.evaluate("setShadowRealmGlobalProperties")(locationSearch, adaptor); 721 722 await shadowRealmEvalAsync(r, ` 723 %(script)s 724 await fakeDynamicImport("%(path)s"); 725 `); 726 const forwardMessage = (msgJSON) => 727 this.port.postMessage(JSON.parse(msgJSON)); 728 r.evaluate("begin_shadow_realm_tests")(forwardMessage); 729 } catch (e) { 730 this.port.postMessage(createSetupErrorResult(e)); 731 } 732 } 733 """ 734 735 def _create_script_import(self, attribute): 736 return 'await fakeDynamicImport("%s");' % attribute 737 738 739 rewrites = [("GET", "/resources/WebIDLParser.js", "/resources/webidl2/lib/webidl2.js")] 740 741 742 class RoutesBuilder: 743 def __init__(self, inject_script = None): 744 self.forbidden_override = [("GET", "/tools/runner/*", handlers.file_handler), 745 ("POST", "/tools/runner/update_manifest.py", 746 handlers.python_script_handler)] 747 748 self.forbidden = [("*", "/_certs/*", handlers.ErrorHandler(404)), 749 ("*", "/tools/*", handlers.ErrorHandler(404)), 750 ("*", "{spec}/tools/*", handlers.ErrorHandler(404)), 751 ("*", "/results/", handlers.ErrorHandler(404))] 752 753 self.extra = [] 754 self.inject_script_data = None 755 if inject_script is not None: 756 with open(inject_script, "rb") as f: 757 self.inject_script_data = f.read() 758 759 self.mountpoint_routes = OrderedDict() 760 761 self.add_mount_point("/", None) 762 763 def get_routes(self): 764 routes = self.forbidden_override + self.forbidden + self.extra 765 # Using reversed here means that mount points that are added later 766 # get higher priority. This makes sense since / is typically added 767 # first. 768 for item in reversed(self.mountpoint_routes.values()): 769 routes.extend(item) 770 return routes 771 772 def add_handler(self, method, route, handler): 773 self.extra.append((str(method), str(route), handler)) 774 775 def add_static(self, path, format_args, content_type, route, headers=None): 776 if headers is None: 777 headers = {} 778 handler = handlers.StaticHandler(path, format_args, content_type, **headers) 779 self.add_handler("GET", str(route), handler) 780 781 def add_mount_point(self, url_base, path): 782 url_base = "/%s/" % url_base.strip("/") if url_base != "/" else "/" 783 784 self.mountpoint_routes[url_base] = [] 785 786 routes = [ 787 ("GET", "*.worker.html", WorkersHandler), 788 ("GET", "*.worker-module.html", WorkerModulesHandler), 789 ("GET", "*.window.html", WindowHandler), 790 ("GET", "*.extension.html", ExtensionHandler), 791 ("GET", "*.any.html", AnyHtmlHandler), 792 ("GET", "*.any.sharedworker.html", SharedWorkersHandler), 793 ("GET", "*.any.sharedworker-module.html", SharedWorkerModulesHandler), 794 ("GET", "*.any.serviceworker.html", ServiceWorkersHandler), 795 ("GET", "*.any.serviceworker-module.html", ServiceWorkerModulesHandler), 796 ("GET", "*.any.shadowrealm-in-window.html", ShadowRealmInWindowHandler), 797 ("GET", "*.any.shadowrealm-in-shadowrealm.html", ShadowRealmInShadowRealmHandler), 798 ("GET", "*.any.shadowrealm-in-dedicatedworker.html", ShadowRealmInDedicatedWorkerHandler), 799 ("GET", "*.any.shadowrealm-in-sharedworker.html", ShadowRealmInSharedWorkerHandler), 800 ("GET", "*.any.shadowrealm-in-serviceworker.html", ShadowRealmInServiceWorkerHandler), 801 ("GET", "*.any.shadowrealm-in-audioworklet.html", ShadowRealmInAudioWorkletHandler), 802 ("GET", "*.any.window-module.html", WindowModulesHandler), 803 ("GET", "*.any.worker.js", ClassicWorkerHandler), 804 ("GET", "*.any.worker-module.js", ModuleWorkerHandler), 805 ("GET", "*.any.serviceworker-shadowrealm.js", ShadowRealmServiceWorkerWrapperHandler), 806 ("GET", "*.any.worker-shadowrealm.js", ShadowRealmWorkerWrapperHandler), 807 ("GET", "*.any.audioworklet-shadowrealm.js", ShadowRealmAudioWorkletWrapperHandler), 808 ("GET", "*.asis", handlers.AsIsHandler), 809 ("*", "/.well-known/attribution-reporting/report-event-attribution", handlers.PythonScriptHandler), 810 ("*", "/.well-known/attribution-reporting/debug/report-event-attribution", handlers.PythonScriptHandler), 811 ("*", "/.well-known/attribution-reporting/report-aggregate-attribution", handlers.PythonScriptHandler), 812 ("*", "/.well-known/attribution-reporting/debug/report-aggregate-attribution", handlers.PythonScriptHandler), 813 ("*", "/.well-known/attribution-reporting/debug/report-aggregate-debug", handlers.PythonScriptHandler), 814 ("*", "/.well-known/attribution-reporting/debug/verbose", handlers.PythonScriptHandler), 815 ("GET", "/.well-known/interest-group/permissions/", handlers.PythonScriptHandler), 816 ("*", "/.well-known/interest-group/real-time-report", handlers.PythonScriptHandler), 817 ("*", "/.well-known/private-aggregation/*", handlers.PythonScriptHandler), 818 ("GET", "/.well-known/shared-storage/trusted-origins", handlers.PythonScriptHandler), 819 ("*", "/.well-known/web-identity", handlers.PythonScriptHandler), 820 ("*", "/.well-known/device-bound-sessions", handlers.PythonScriptHandler), 821 ("*", "*.py", handlers.PythonScriptHandler), 822 ("GET", "*", handlers.FileHandler) 823 ] 824 825 for (method, suffix, handler_cls) in routes: 826 handler = handler_cls(base_path=path, url_base=url_base) 827 if self.inject_script_data is not None: 828 handler = HtmlScriptInjectorHandlerWrapper(inject=self.inject_script_data, wrap=handler) 829 830 self.mountpoint_routes[url_base].append( 831 (method, 832 "%s%s" % (url_base if url_base != "/" else "", suffix), 833 handler)) 834 835 def add_file_mount_point(self, file_url, base_path): 836 assert file_url.startswith("/") 837 url_base = file_url[0:file_url.rfind("/") + 1] 838 self.mountpoint_routes[file_url] = [("GET", file_url, handlers.FileHandler(base_path=base_path, url_base=url_base))] 839 840 841 def get_route_builder(logger, aliases, config): 842 builder = RoutesBuilder(config.inject_script) 843 for alias in aliases: 844 url = alias["url-path"] 845 directory = alias["local-dir"] 846 if not url.startswith("/") or len(directory) == 0: 847 logger.error("\"url-path\" value must start with '/'.") 848 continue 849 if url.endswith("/"): 850 builder.add_mount_point(url, directory) 851 else: 852 builder.add_file_mount_point(url, directory) 853 return builder 854 855 856 class ServerProc: 857 def __init__(self, mp_context, scheme=None): 858 self.proc = None 859 self.daemon = None 860 self.mp_context = mp_context 861 self.stop_flag = mp_context.Event() 862 self.scheme = scheme 863 864 def start(self, init_func, host, port, paths, routes, bind_address, config, log_handlers, **kwargs): 865 self.proc = self.mp_context.Process(target=self.create_daemon, 866 args=(init_func, host, port, paths, routes, bind_address, 867 config, log_handlers, dict(**os.environ)), 868 name="%s on port %s" % (self.scheme, port), 869 kwargs=kwargs) 870 self.proc.daemon = True 871 self.proc.start() 872 873 def create_daemon(self, init_func, host, port, paths, routes, bind_address, 874 config, log_handlers, env, **kwargs): 875 # Ensure that when we start this in a new process we have the global lock 876 # in the logging module unlocked 877 importlib.reload(logging) 878 os.environ = env 879 logger = get_logger(config.logging["level"], log_handlers) 880 881 if sys.platform == "darwin": 882 # on Darwin, NOFILE starts with a very low limit (256), so bump it up a little 883 # by way of comparison, Debian starts with a limit of 1024, Windows 512 884 import resource # local, as it only exists on Unix-like systems 885 maxfilesperproc = int(subprocess.check_output( 886 ["sysctl", "-n", "kern.maxfilesperproc"] 887 ).strip()) 888 soft, hard = resource.getrlimit(resource.RLIMIT_NOFILE) 889 # 2048 is somewhat arbitrary, but gives us some headroom for wptrunner --parallel 890 # note that it's expected that 2048 will be the min here 891 new_soft = min(2048, maxfilesperproc, hard) 892 if soft < new_soft: 893 resource.setrlimit(resource.RLIMIT_NOFILE, (new_soft, hard)) 894 try: 895 self.daemon = init_func(logger, host, port, paths, routes, bind_address, config, **kwargs) 896 except OSError: 897 logger.critical("Socket error on port %s" % port) 898 raise 899 except Exception: 900 logger.critical(traceback.format_exc()) 901 raise 902 903 if self.daemon: 904 try: 905 self.daemon.start() 906 try: 907 self.stop_flag.wait() 908 except KeyboardInterrupt: 909 pass 910 finally: 911 self.daemon.stop() 912 except Exception: 913 logger.critical(traceback.format_exc()) 914 raise 915 916 def request_shutdown(self): 917 if self.is_alive(): 918 self.stop_flag.set() 919 920 def wait(self, timeout=None): 921 self.proc.join(timeout) 922 923 def is_alive(self): 924 return self.proc.is_alive() 925 926 927 def check_subdomains(logger, config, routes, mp_context, log_handlers): 928 paths = config.paths 929 bind_address = config.bind_address 930 931 host = config.server_host 932 port = get_port() 933 logger.debug("Going to use port %d to check subdomains" % port) 934 935 wrapper = ServerProc(mp_context) 936 wrapper.start(start_http_server, host, port, paths, routes, 937 bind_address, config, log_handlers) 938 939 url = f"http://{host}:{port}/" 940 connected = False 941 for i in range(10): 942 try: 943 urllib.request.urlopen(url) 944 connected = True 945 break 946 except urllib.error.URLError: 947 time.sleep(1) 948 949 if not connected: 950 logger.critical("Failed to connect to test server " 951 "on {}. {}".format(url, EDIT_HOSTS_HELP)) 952 sys.exit(1) 953 954 for domain in config.domains_set: 955 if domain == host: 956 continue 957 958 try: 959 urllib.request.urlopen("http://%s:%d/" % (domain, port)) 960 except Exception: 961 logger.critical(f"Failed probing domain {domain}. {EDIT_HOSTS_HELP}") 962 sys.exit(1) 963 964 wrapper.request_shutdown() 965 wrapper.wait() 966 967 968 def make_hosts_file(config, host): 969 rv = ["# Start web-platform-tests hosts"] 970 971 for domain in sorted( 972 config.domains_set, key=lambda x: tuple(reversed(x.split("."))) 973 ): 974 rv.append("%s\t%s" % (host, domain)) 975 976 # Windows interprets the IP address 0.0.0.0 as non-existent, making it an 977 # appropriate alias for non-existent hosts. However, UNIX-like systems 978 # interpret the same address to mean any IP address, which is inappropriate 979 # for this context. These systems do not reserve any value for this 980 # purpose, so the inavailability of the domains must be taken for granted. 981 # 982 # https://github.com/web-platform-tests/wpt/issues/10560 983 if platform.uname()[0] == "Windows": 984 for not_domain in sorted( 985 config.not_domains_set, key=lambda x: tuple(reversed(x.split("."))) 986 ): 987 rv.append("0.0.0.0\t%s" % not_domain) 988 989 rv.append("# End web-platform-tests hosts") 990 rv.append("") 991 992 return "\n".join(rv) 993 994 995 def start_servers(logger, host, ports, paths, routes, bind_address, config, 996 mp_context, log_handlers, **kwargs): 997 servers = defaultdict(list) 998 for scheme, ports in ports.items(): 999 assert len(ports) == {"http": 2, "https": 2}.get(scheme, 1) 1000 1001 # If trying to start HTTP/2.0 server, check compatibility 1002 if scheme == 'h2' and not http2_compatible(): 1003 logger.error('Cannot start HTTP/2.0 server as the environment is not compatible. ' + 1004 'Requires OpenSSL 1.0.2+') 1005 continue 1006 1007 # Skip WebTransport over HTTP/3 server unless if is enabled explicitly. 1008 if scheme == 'webtransport-h3' and not kwargs.get("webtransport_h3"): 1009 continue 1010 1011 for port in ports: 1012 if port is None: 1013 continue 1014 1015 init_func = { 1016 "http": start_http_server, 1017 "http-local": start_http_server, 1018 "http-public": start_http_server, 1019 "https": start_https_server, 1020 "https-local": start_https_server, 1021 "https-public": start_https_server, 1022 "h2": start_http2_server, 1023 "ws": start_ws_server, 1024 "wss": start_wss_server, 1025 "webtransport-h3": start_webtransport_h3_server, 1026 }[scheme] 1027 1028 server_proc = ServerProc(mp_context, scheme=scheme) 1029 server_proc.start(init_func, host, port, paths, routes, bind_address, 1030 config, log_handlers, **kwargs) 1031 servers[scheme].append((port, server_proc)) 1032 1033 return servers 1034 1035 1036 def startup_failed(logger): 1037 logger.critical(EDIT_HOSTS_HELP) 1038 sys.exit(1) 1039 1040 1041 def start_http_server(logger, host, port, paths, routes, bind_address, config, **kwargs): 1042 try: 1043 return wptserve.WebTestHttpd(host=host, 1044 port=port, 1045 doc_root=paths["doc_root"], 1046 routes=routes, 1047 rewrites=rewrites, 1048 bind_address=bind_address, 1049 config=config, 1050 use_ssl=False, 1051 key_file=None, 1052 certificate=None, 1053 latency=kwargs.get("latency")) 1054 except Exception as error: 1055 logger.critical(f"start_http_server: Caught exception from wptserve.WebTestHttpd: {error}") 1056 startup_failed(logger) 1057 1058 1059 def start_https_server(logger, host, port, paths, routes, bind_address, config, **kwargs): 1060 try: 1061 return wptserve.WebTestHttpd(host=host, 1062 port=port, 1063 doc_root=paths["doc_root"], 1064 routes=routes, 1065 rewrites=rewrites, 1066 bind_address=bind_address, 1067 config=config, 1068 use_ssl=True, 1069 key_file=config.ssl_config["key_path"], 1070 certificate=config.ssl_config["cert_path"], 1071 encrypt_after_connect=config.ssl_config["encrypt_after_connect"], 1072 latency=kwargs.get("latency")) 1073 except Exception as error: 1074 logger.critical(f"start_https_server: Caught exception from wptserve.WebTestHttpd: {error}") 1075 startup_failed(logger) 1076 1077 1078 def start_http2_server(logger, host, port, paths, routes, bind_address, config, **kwargs): 1079 try: 1080 return wptserve.WebTestHttpd(host=host, 1081 port=port, 1082 handler_cls=wptserve.Http2WebTestRequestHandler, 1083 doc_root=paths["doc_root"], 1084 ws_doc_root=paths["ws_doc_root"], 1085 routes=routes, 1086 rewrites=rewrites, 1087 bind_address=bind_address, 1088 config=config, 1089 use_ssl=True, 1090 key_file=config.ssl_config["key_path"], 1091 certificate=config.ssl_config["cert_path"], 1092 encrypt_after_connect=config.ssl_config["encrypt_after_connect"], 1093 latency=kwargs.get("latency"), 1094 http2=True) 1095 except Exception as error: 1096 logger.critical(f"start_http2_server: Caught exception from wptserve.WebTestHttpd: {error}") 1097 startup_failed(logger) 1098 1099 1100 class WebSocketDaemon: 1101 def __init__(self, host, port, doc_root, handlers_root, bind_address, ssl_config, 1102 extra_handler_paths=None): 1103 logger = logging.getLogger() 1104 self.host = host 1105 cmd_args = ["-p", port, 1106 "-d", doc_root, 1107 "-w", handlers_root] 1108 1109 if ssl_config is not None: 1110 cmd_args += ["--tls", 1111 "--private-key", ssl_config["key_path"], 1112 "--certificate", ssl_config["cert_path"]] 1113 1114 if (bind_address): 1115 cmd_args = ["-H", host] + cmd_args 1116 opts, args = pywebsocket._parse_args_and_config(cmd_args) 1117 opts.cgi_directories = [] 1118 opts.is_executable_method = None 1119 self.server = pywebsocket.WebSocketServer(opts) 1120 if extra_handler_paths: 1121 for path in extra_handler_paths: 1122 self.server.websocket_server_options.dispatcher._source_handler_files_in_dir(path, path, False, None) 1123 ports = [item[0].getsockname()[1] for item in self.server._sockets] 1124 if not ports: 1125 # TODO: Fix the logging configuration in WebSockets processes 1126 # see https://github.com/web-platform-tests/wpt/issues/22719 1127 logger.critical("Failed to start websocket server on port %s, " 1128 "is something already using that port?" % port) 1129 raise OSError() 1130 assert all(item == ports[0] for item in ports) 1131 self.port = ports[0] 1132 self.started = False 1133 self.server_thread = None 1134 1135 def start(self): 1136 self.started = True 1137 self.server_thread = threading.Thread(target=self.server.serve_forever) 1138 self.server_thread.setDaemon(True) # don't hang on exit 1139 self.server_thread.start() 1140 1141 def stop(self): 1142 """ 1143 Stops the server. 1144 1145 If the server is not running, this method has no effect. 1146 """ 1147 if self.started: 1148 try: 1149 self.server.shutdown() 1150 self.server.server_close() 1151 self.server_thread.join() 1152 self.server_thread = None 1153 except AttributeError: 1154 pass 1155 self.started = False 1156 self.server = None 1157 1158 1159 def start_ws_server(logger, host, port, paths, routes, bind_address, config, **kwargs): 1160 try: 1161 return WebSocketDaemon(host, 1162 str(port), 1163 repo_root, 1164 config.paths["ws_doc_root"], 1165 bind_address, 1166 ssl_config=None, 1167 extra_handler_paths=config.paths["ws_extra"]) 1168 except Exception as error: 1169 logger.critical(f"start_ws_server: Caught exception from WebSocketDomain: {error}") 1170 startup_failed(logger) 1171 1172 1173 def start_wss_server(logger, host, port, paths, routes, bind_address, config, **kwargs): 1174 try: 1175 return WebSocketDaemon(host, 1176 str(port), 1177 repo_root, 1178 config.paths["ws_doc_root"], 1179 bind_address, 1180 config.ssl_config, 1181 extra_handler_paths=config.paths["ws_extra"]) 1182 except Exception as error: 1183 logger.critical(f"start_wss_server: Caught exception from WebSocketDomain: {error}") 1184 startup_failed(logger) 1185 1186 1187 def start_webtransport_h3_server(logger, host, port, paths, routes, bind_address, config, **kwargs): 1188 try: 1189 # TODO(bashi): Move the following import to the beginning of this file 1190 # once WebTransportH3Server is enabled by default. 1191 from webtransport.h3.webtransport_h3_server import WebTransportH3Server # type: ignore 1192 return WebTransportH3Server(host=host, 1193 port=port, 1194 doc_root=paths["doc_root"], 1195 cert_path=config.ssl_config["cert_path"], 1196 key_path=config.ssl_config["key_path"], 1197 logger=logger) 1198 except Exception as error: 1199 logger.critical( 1200 f"Failed to start WebTransport over HTTP/3 server: {error}") 1201 sys.exit(0) 1202 1203 1204 def start(logger, config, routes, mp_context, log_handlers, **kwargs): 1205 host = config["server_host"] 1206 ports = config.ports 1207 paths = config.paths 1208 bind_address = config["bind_address"] 1209 1210 logger.debug("Using ports: %r" % ports) 1211 1212 servers = start_servers(logger, host, ports, paths, routes, bind_address, config, mp_context, 1213 log_handlers, **kwargs) 1214 1215 return servers 1216 1217 1218 def iter_servers(servers): 1219 for servers in servers.values(): 1220 for port, server in servers: 1221 yield server 1222 1223 1224 def _make_subdomains_product(s: Set[str], depth: int = 2) -> Set[str]: 1225 return {".".join(x) for x in chain(*(product(s, repeat=i) for i in range(1, depth+1)))} 1226 1227 1228 _subdomains = {"www", 1229 "www1", 1230 "www2", 1231 "天気の良い日", 1232 "élève"} 1233 1234 _not_subdomains = {"nonexistent"} 1235 1236 _subdomains = _make_subdomains_product(_subdomains) 1237 1238 _not_subdomains = _make_subdomains_product(_not_subdomains) 1239 1240 1241 class ConfigBuilder(config.ConfigBuilder): 1242 """serve config 1243 1244 This subclasses wptserve.config.ConfigBuilder to add serve config options. 1245 """ 1246 1247 _default = { 1248 "browser_host": "web-platform.test", 1249 "alternate_hosts": { 1250 "alt": "not-web-platform.test" 1251 }, 1252 "doc_root": repo_root, 1253 "ws_doc_root": os.path.join(repo_root, "websockets", "handlers"), 1254 "ws_extra": None, 1255 "server_host": None, 1256 "ports": { 1257 "http": [8000, "auto"], 1258 "http-local": ["auto"], 1259 "http-public": ["auto"], 1260 "https": [8443, 8444], 1261 "https-local": ["auto"], 1262 "https-public": ["auto"], 1263 "ws": ["auto"], 1264 "wss": ["auto"], 1265 "webtransport-h3": ["auto"], 1266 }, 1267 "check_subdomains": True, 1268 "bind_address": True, 1269 "ssl": { 1270 "type": "pregenerated", 1271 "encrypt_after_connect": False, 1272 "openssl": { 1273 "openssl_binary": "openssl", 1274 "base_path": "_certs", 1275 "password": "web-platform-tests", 1276 "force_regenerate": False, 1277 "duration": 30, 1278 "base_conf_path": None 1279 }, 1280 "pregenerated": { 1281 "host_key_path": os.path.join(repo_root, "tools", "certs", "web-platform.test.key"), 1282 "host_cert_path": os.path.join(repo_root, "tools", "certs", "web-platform.test.pem") 1283 }, 1284 "none": {} 1285 }, 1286 "aliases": [], 1287 "logging": { 1288 "level": "info", 1289 "suppress_handler_traceback": False 1290 } 1291 } 1292 1293 computed_properties = ["ws_doc_root"] + config.ConfigBuilder.computed_properties 1294 1295 def __init__(self, logger, *args, **kwargs): 1296 if "subdomains" not in kwargs: 1297 kwargs["subdomains"] = _subdomains 1298 if "not_subdomains" not in kwargs: 1299 kwargs["not_subdomains"] = _not_subdomains 1300 super().__init__( 1301 logger, 1302 *args, 1303 **kwargs 1304 ) 1305 with self as c: 1306 browser_host = c.get("browser_host") 1307 alternate_host = c.get("alternate_hosts", {}).get("alt") 1308 1309 if not domains_are_distinct(browser_host, alternate_host): 1310 raise ValueError( 1311 "Alternate host must be distinct from browser host" 1312 ) 1313 1314 def _get_ws_doc_root(self, data): 1315 if data["ws_doc_root"] is not None: 1316 return data["ws_doc_root"] 1317 else: 1318 return os.path.join(data["doc_root"], "websockets", "handlers") 1319 1320 def _get_paths(self, data): 1321 rv = super()._get_paths(data) 1322 rv["ws_doc_root"] = data["ws_doc_root"] 1323 rv["ws_extra"] = data["ws_extra"] 1324 return rv 1325 1326 1327 def build_config(logger, override_path=None, config_cls=ConfigBuilder, **kwargs): 1328 rv = config_cls(logger) 1329 1330 enable_http2 = kwargs.get("h2") 1331 if enable_http2 is None: 1332 enable_http2 = True 1333 if enable_http2: 1334 rv._default["ports"]["h2"] = [9000] 1335 1336 if override_path and os.path.exists(override_path): 1337 with open(override_path) as f: 1338 override_obj = json.load(f) 1339 rv.update(override_obj) 1340 1341 if kwargs.get("config_path"): 1342 other_path = os.path.abspath(os.path.expanduser(kwargs.get("config_path"))) 1343 if os.path.exists(other_path): 1344 with open(other_path) as f: 1345 override_obj = json.load(f) 1346 rv.update(override_obj) 1347 else: 1348 raise ValueError("Config path %s does not exist" % other_path) 1349 1350 if kwargs.get("verbose"): 1351 rv.logging["level"] = "DEBUG" 1352 1353 setattr(rv, "inject_script", kwargs.get("inject_script")) 1354 1355 overriding_path_args = [("doc_root", "Document root"), 1356 ("ws_doc_root", "WebSockets document root")] 1357 for key, title in overriding_path_args: 1358 value = kwargs.get(key) 1359 if value is None: 1360 continue 1361 value = os.path.abspath(os.path.expanduser(value)) 1362 if not os.path.exists(value): 1363 raise ValueError("%s path %s does not exist" % (title, value)) 1364 setattr(rv, key, value) 1365 1366 return rv 1367 1368 1369 def get_parser(): 1370 parser = argparse.ArgumentParser() 1371 parser.add_argument("--latency", type=int, 1372 help="Artificial latency to add before sending http responses, in ms") 1373 parser.add_argument("--config", dest="config_path", 1374 help="Path to external config file") 1375 parser.add_argument("--doc_root", help="Path to document root. Overrides config.") 1376 parser.add_argument("--ws_doc_root", 1377 help="Path to WebSockets document root. Overrides config.") 1378 parser.add_argument("--ws_extra", action="append", default=[], 1379 help="Path to extra directory containing ws handlers. Overrides config.") 1380 parser.add_argument("--inject-script", 1381 help="Path to script file to inject, useful for testing polyfills.") 1382 parser.add_argument("--alias_file", 1383 help="File with entries for aliases/multiple doc roots. In form of `/ALIAS_NAME/, DOC_ROOT\\n`") 1384 parser.add_argument("--h2", action="store_true", default=None, 1385 help=argparse.SUPPRESS) 1386 parser.add_argument("--no-h2", action="store_false", dest="h2", default=None, 1387 help="Disable the HTTP/2.0 server") 1388 parser.add_argument("--webtransport-h3", action="store_true", 1389 help="Enable WebTransport over HTTP/3 server") 1390 parser.add_argument("--exit-after-start", action="store_true", 1391 help="Exit after starting servers") 1392 parser.add_argument("--verbose", action="store_true", help="Enable verbose logging") 1393 parser.set_defaults(report=False) 1394 parser.set_defaults(is_wave=False) 1395 return parser 1396 1397 1398 class MpContext: 1399 def __getattr__(self, name): 1400 return getattr(multiprocessing, name) 1401 1402 1403 def get_logger(log_level, log_handlers): 1404 """Get a logger configured to log at level log_level 1405 1406 If the logger has existing handlers the log_handlers argument is ignored. 1407 Otherwise the handlers in log_handlers are added to the logger. If there are 1408 no log_handlers passed and no configured handlers, a stream handler is added 1409 to the logger. 1410 1411 Typically this is called once per process to set up logging in that process. 1412 1413 :param log_level: - A string representing a log level e.g. "info" 1414 :param log_handlers: - Optional list of Handler objects. 1415 """ 1416 logger = logging.getLogger() 1417 logger.setLevel(getattr(logging, log_level.upper())) 1418 if not logger.hasHandlers(): 1419 if log_handlers is not None: 1420 for handler in log_handlers: 1421 logger.addHandler(handler) 1422 else: 1423 handler = logging.StreamHandler(sys.stdout) 1424 formatter = logging.Formatter("[%(asctime)s %(processName)s] %(levelname)s - %(message)s") 1425 handler.setFormatter(formatter) 1426 logger.addHandler(handler) 1427 return logger 1428 1429 1430 def run(config_cls=ConfigBuilder, route_builder=None, mp_context=None, log_handlers=None, 1431 **kwargs): 1432 logger = get_logger("INFO", log_handlers) 1433 1434 if mp_context is None: 1435 mp_context = multiprocessing.get_context("spawn") 1436 1437 with build_config(logger, 1438 os.path.join(repo_root, "config.json"), 1439 config_cls=config_cls, 1440 **kwargs) as config: 1441 # This sets the right log level 1442 logger = get_logger(config.logging["level"], log_handlers) 1443 1444 bind_address = config["bind_address"] 1445 1446 if kwargs.get("alias_file"): 1447 with open(kwargs["alias_file"]) as alias_file: 1448 for line in alias_file: 1449 alias, doc_root = (x.strip() for x in line.split(",")) 1450 config["aliases"].append({ 1451 "url-path": alias, 1452 "local-dir": doc_root, 1453 }) 1454 1455 if route_builder is None: 1456 route_builder = get_route_builder 1457 routes = route_builder(logger, config.aliases, config).get_routes() 1458 1459 if config["check_subdomains"]: 1460 check_subdomains(logger, config, routes, mp_context, log_handlers) 1461 1462 stash_address = None 1463 if bind_address: 1464 stash_address = (config.server_host, get_port("")) 1465 logger.debug("Going to use port %d for stash" % stash_address[1]) 1466 1467 with stash.StashServer(stash_address, authkey=str(uuid.uuid4())): 1468 servers = start(logger, config, routes, mp_context, log_handlers, **kwargs) 1469 1470 if not kwargs.get("exit_after_start"): 1471 try: 1472 # Periodically check if all the servers are alive 1473 server_process_exited = False 1474 while not server_process_exited: 1475 for server in iter_servers(servers): 1476 server.proc.join(1) 1477 if not server.proc.is_alive(): 1478 server_process_exited = True 1479 break 1480 except KeyboardInterrupt: 1481 pass 1482 1483 failed_subproc = 0 1484 for server in iter_servers(servers): 1485 logger.info('Status of subprocess "%s": running', server.proc.name) 1486 server.request_shutdown() 1487 1488 for server in iter_servers(servers): 1489 server.wait(timeout=1) 1490 if server.proc.exitcode == 0: 1491 logger.info('Status of subprocess "%s": exited correctly', server.proc.name) 1492 elif server.proc.exitcode is None: 1493 logger.warning( 1494 'Status of subprocess "%s": shutdown timed out', 1495 server.proc.name) 1496 failed_subproc += 1 1497 else: 1498 subproc = server.proc 1499 logger.warning('Status of subprocess "%s": failed. Exit with non-zero status: %d', 1500 subproc.name, subproc.exitcode) 1501 failed_subproc += 1 1502 return failed_subproc 1503 1504 1505 def main(): 1506 kwargs = vars(get_parser().parse_args()) 1507 return run(**kwargs)