network_bench.py (27079B)
1 # This Source Code Form is subject to the terms of the Mozilla Public 2 # License, v. 2.0. If a copy of the MPL was not distributed with this 3 # file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 5 import json 6 import os 7 import re 8 import signal 9 import socket 10 import subprocess 11 import tempfile 12 import threading 13 from pathlib import Path 14 from subprocess import PIPE, Popen 15 16 import filters 17 from base_python_support import BasePythonSupport 18 from logger.logger import RaptorLogger 19 20 LOG = RaptorLogger(component="raptor-browsertime") 21 22 23 class NetworkBench(BasePythonSupport): 24 def __init__(self, **kwargs): 25 super().__init__(**kwargs) 26 self._is_chrome = False 27 self.browsertime_node = None 28 self.backend_server = None 29 self.backend_port = None 30 self.caddy_port = None 31 self.caddy_server = None 32 self.caddy_stdout = None 33 self.caddy_stderr = None 34 self.http_version = "h1" 35 self.transfer_type = "download" 36 # loopback 37 self.interface = "lo" 38 self.network_type = "unthrottled" 39 self.packet_loss_rate = None 40 self.cleanup = [] 41 42 def setup_test(self, test, args): 43 from cmdline import CHROME_ANDROID_APPS, CHROMIUM_DISTROS 44 45 LOG.info("setup_test: '%s'" % test) 46 47 self._is_chrome = ( 48 args.app in CHROMIUM_DISTROS or args.app in CHROME_ANDROID_APPS 49 ) 50 51 test_name = test.get("name").split("-", 2) 52 self.http_version = test_name[0] if test_name[0] in ["h3", "h2"] else "unknown" 53 self.transfer_type = ( 54 test_name[1] if test_name[1] in ["download", "upload"] else "unknown" 55 ) 56 LOG.info(f"http_version: '{self.http_version}', type: '{self.transfer_type}'") 57 58 if self.http_version == "unknown" or self.transfer_type == "unknown": 59 raise Exception("Unsupported test") 60 61 def check_caddy_installed(self): 62 try: 63 result = subprocess.run( 64 ["caddy", "version"], 65 check=False, 66 capture_output=True, 67 text=True, 68 ) 69 if result.returncode == 0: 70 LOG.info("Caddy is installed. Version: %s" % result.stdout.strip()) 71 return True 72 else: 73 LOG.error("Caddy is not installed.") 74 except FileNotFoundError: 75 LOG.error("Caddy is not installed.") 76 return False 77 78 def start_backend_server(self, path): 79 if self.browsertime_node is None or not self.browsertime_node.exists(): 80 return None 81 82 LOG.info("node bin: %s" % self.browsertime_node) 83 84 server_path = ( 85 Path(__file__).parent / ".." / ".." / "browsertime" / "utils" / path 86 ) 87 LOG.info("server_path: %s" % server_path) 88 89 if not server_path.exists(): 90 return None 91 92 process = Popen( 93 [self.browsertime_node, server_path], 94 stdin=PIPE, 95 stdout=PIPE, 96 stderr=PIPE, 97 universal_newlines=True, 98 start_new_session=True, 99 ) 100 msg = process.stdout.readline() 101 LOG.info("server msg: %s" % msg) 102 match = re.search(r"Server is running on http://[^:]+:(\d+)", msg) 103 if match: 104 self.backend_port = match.group(1) 105 LOG.info("backend port: %s" % self.backend_port) 106 return process 107 return None 108 109 def find_free_port(self, socket_type): 110 with socket.socket(socket.AF_INET, socket_type) as s: 111 s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) 112 s.bind(("localhost", 0)) 113 return s.getsockname()[1] 114 115 def start_caddy(self, download_html, test_file_path): 116 if not self.check_caddy_installed(): 117 return None 118 if self.caddy_port is None or not (1 <= self.caddy_port <= 65535): 119 return None 120 121 utils_path = Path(__file__).parent / ".." / ".." / "browsertime" / "utils" 122 123 if not utils_path.exists(): 124 return None 125 126 key_path = utils_path / "http2-cert.key" 127 LOG.info("key_path: %s" % key_path) 128 if not key_path.exists(): 129 return None 130 131 pem_path = utils_path / "http2-cert.pem" 132 LOG.info("pem_path: %s" % pem_path) 133 if not pem_path.exists(): 134 return None 135 136 port_str = f":{self.caddy_port}" 137 if self.transfer_type == "upload": 138 upstream = f"localhost:{self.backend_port}" 139 routes = [ 140 { 141 "handle": [ 142 { 143 "handler": "reverse_proxy", 144 "upstreams": [{"dial": upstream}], 145 } 146 ] 147 } 148 ] 149 elif self.transfer_type == "download": 150 routes = [ 151 { 152 "match": [{"path": [f"/{download_html.name}"]}], 153 "handle": [ 154 { 155 "handler": "file_server", 156 "root": str(tempfile.gettempdir()), 157 "browse": {}, 158 } 159 ], 160 }, 161 { 162 "match": [{"path": [f"/{test_file_path.name}"]}], 163 "handle": [ 164 { 165 "handler": "headers", 166 "response": { 167 "set": { 168 "Cache-Control": [ 169 "no-store", 170 "no-cache", 171 "must-revalidate", 172 "proxy-revalidate", 173 "max-age=0", 174 ], 175 "Pragma": ["no-cache"], 176 "Expires": ["0"], 177 } 178 }, 179 }, 180 { 181 "handler": "file_server", 182 "root": str(tempfile.gettempdir()), 183 "browse": {}, 184 }, 185 ], 186 }, 187 ] 188 protocols = ["h3"] if self.http_version == "h3" else ["h1", "h2"] 189 caddyfile_content = { 190 "admin": {"disabled": True}, 191 "apps": { 192 "http": { 193 "servers": { 194 "server1": { 195 "listen": [port_str], 196 "protocols": protocols, 197 "routes": routes, 198 "tls_connection_policies": [ 199 {"certificate_selection": {"any_tag": ["cert1"]}} 200 ], 201 "automatic_https": {"disable": True}, 202 } 203 }, 204 }, 205 "tls": { 206 # Disable 0RTT for now. Can be reverted once 207 # https://github.com/quic-go/quic-go/issues/5001 and 208 # https://github.com/mozilla/neqo/issues/2476 are fixed. 209 "session_tickets": {"disabled": True}, 210 "certificates": { 211 "load_files": [ 212 { 213 "certificate": str(pem_path), 214 "key": str(key_path), 215 "tags": ["cert1"], 216 } 217 ] 218 }, 219 }, 220 }, 221 } 222 223 LOG.info("caddyfile_content: %s" % caddyfile_content) 224 225 with tempfile.NamedTemporaryFile( 226 mode="w", delete=False, suffix=".json" 227 ) as temp_json_file: 228 json.dump(caddyfile_content, temp_json_file, indent=2) 229 temp_json_file_path = temp_json_file.name 230 231 LOG.info("temp_json_file_path: %s" % temp_json_file_path) 232 command = ["caddy", "run", "--config", temp_json_file_path] 233 234 def read_output(pipe, log_func): 235 for line in iter(pipe.readline, ""): 236 log_func(line) 237 238 process = Popen( 239 command, 240 stdin=PIPE, 241 stdout=PIPE, 242 stderr=PIPE, 243 universal_newlines=True, 244 start_new_session=True, 245 ) 246 self.caddy_stdout = threading.Thread( 247 target=read_output, args=(process.stdout, LOG.info) 248 ) 249 self.caddy_stderr = threading.Thread( 250 target=read_output, args=(process.stderr, LOG.info) 251 ) 252 self.caddy_stdout.start() 253 self.caddy_stderr.start() 254 return process 255 256 def check_tc_command(self): 257 try: 258 result = subprocess.run( 259 ["sudo", "tc", "-help"], 260 check=False, 261 capture_output=True, 262 text=True, 263 ) 264 if result.returncode == 0: 265 LOG.info("tc can be executed as root") 266 return True 267 else: 268 LOG.error("tc is not available") 269 except Exception as e: 270 LOG.error(f"Error executing tc: {str(e)}") 271 return False 272 273 def run_command(self, command): 274 try: 275 result = subprocess.run( 276 command, 277 shell=True, 278 check=True, 279 capture_output=True, 280 ) 281 LOG.info(command) 282 LOG.info(f"Output: {result.stdout.decode().strip()}") 283 return True 284 except subprocess.CalledProcessError as e: 285 LOG.info(f"Error executing command: {command}") 286 LOG.info(f"Error: {e.stderr.decode()}") 287 return False 288 289 def network_type_to_bandwidth_rtt(self, network_type): 290 # Define the mapping of network types 291 network_mapping = { 292 "1M_400ms": {"bandwidth": "1Mbit", "rtt_ms": 400}, 293 "300M_40ms": {"bandwidth": "300Mbit", "rtt_ms": 40}, 294 "300M_80ms": {"bandwidth": "300Mbit", "rtt_ms": 80}, 295 "10M_40ms": {"bandwidth": "10Mbit", "rtt_ms": 40}, 296 "100M_40ms": {"bandwidth": "100Mbit", "rtt_ms": 40}, 297 } 298 299 # Look up the network type in the mapping 300 result = network_mapping.get(network_type) 301 302 if result is None: 303 raise Exception(f"Unknown network type: {network_type}") 304 return result["bandwidth"], result["rtt_ms"] 305 306 def apply_network_throttling( 307 self, interface, network_type, loss, protocol_and_port 308 ): 309 def calculate_bdp(bandwidth_mbit, rtt_ms): 310 bandwidth_kbps = bandwidth_mbit * 1_000 311 bdp_bits = bandwidth_kbps * rtt_ms 312 bdp_bytes = bdp_bits / 8 313 bdp_bytes = max(bdp_bytes, 1500) 314 return int(bdp_bytes) 315 316 bandwidth_str, rtt_ms = self.network_type_to_bandwidth_rtt(network_type) 317 # The delay used in netem is applied before sending, 318 # so the delay value should be rtt_ms / 2. 319 delay_ms = rtt_ms / 2 320 bandwidth_mbit = float(bandwidth_str.replace("Mbit", "")) 321 bdp_bytes = calculate_bdp(bandwidth_mbit, rtt_ms) 322 323 LOG.info( 324 f"apply_network_throttling: bandwidth={bandwidth_str} delay={delay_ms}ms loss={loss}" 325 ) 326 327 self.run_command(f"sudo tc qdisc del dev {interface} root") 328 if not self.run_command( 329 f"sudo tc qdisc add dev {interface} root handle 1: htb default 12" 330 ): 331 return False 332 else: 333 LOG.info("Register cleanup function") 334 self.cleanup.append( 335 lambda: self.run_command(f"sudo tc qdisc del dev {self.interface} root") 336 ) 337 338 if not self.run_command( 339 f"sudo tc class add dev {interface} parent 1: classid 1:1 htb rate {bandwidth_str}" 340 ): 341 return False 342 343 delay_str = f"{delay_ms}ms" 344 if not loss or loss == "0": 345 if not self.run_command( 346 f"sudo tc qdisc add dev {interface} parent 1:1 handle 10: netem delay {delay_str} limit {bdp_bytes}" 347 ): 348 return False 349 elif not self.run_command( 350 f"sudo tc qdisc add dev {interface} parent 1:1 handle 10: netem delay {delay_str} loss {loss}% limit {bdp_bytes}" 351 ): 352 return False 353 354 protocol = 6 if protocol_and_port[0] == "tcp" else 17 355 port = protocol_and_port[1] 356 # Add a filter to match TCP/UDP traffic on the specified port for IPv4 357 if not self.run_command( 358 f"sudo tc filter add dev {interface} protocol ip parent 1:0 u32 " 359 f"match ip protocol {protocol} 0xff " 360 f"match ip dport {port} 0xffff " 361 f"flowid 1:1" 362 ): 363 return False 364 if not self.run_command( 365 f"sudo tc filter add dev {interface} protocol ip parent 1:0 u32 " 366 f"match ip protocol {protocol} 0xff " 367 f"match ip sport {port} 0xffff " 368 f"flowid 1:1" 369 ): 370 return False 371 # Add a filter to match TCP/UDP traffic on the specified port for IPv6 372 if not self.run_command( 373 f"sudo tc filter add dev {interface} parent 1:0 protocol ipv6 u32 " 374 f"match ip6 protocol {protocol} 0xff " 375 f"match ip6 dport {port} 0xffff " 376 f"flowid 1:1" 377 ): 378 return False 379 if not self.run_command( 380 f"sudo tc filter add dev {interface} parent 1:0 protocol ipv6 u32 " 381 f"match ip6 protocol {protocol} 0xff " 382 f"match ip6 sport {port} 0xffff " 383 f"flowid 1:1" 384 ): 385 return False 386 return True 387 388 def get_network_conditions(self, cmd): 389 try: 390 i = 0 391 while i < len(cmd): 392 if cmd[i] == "--network_type": 393 self.network_type = cmd[i + 1] 394 i += 2 395 elif cmd[i] == "--pkt_loss_rate": 396 self.packet_loss_rate = cmd[i + 1] 397 i += 2 398 else: 399 i += 1 400 except Exception: 401 raise Exception("failed to get network condition") 402 403 def network_type_to_temp_file(self, network_type): 404 def calculate_file_size(bandwidth_mbps): 405 time_seconds = 60 406 # Calculate transfer size in bits 407 transfer_size_bits = bandwidth_mbps * 1e6 * time_seconds 408 transfer_size_bytes = int(transfer_size_bits / 8) 409 return transfer_size_bytes 410 411 def generate_temp_file(file_size_in_bytes, target_dir): 412 prefix = f"temp_file_{file_size_in_bytes}B_" 413 suffix = ".bin" 414 try: 415 with tempfile.NamedTemporaryFile( 416 mode="wb", 417 prefix=prefix, 418 suffix=suffix, 419 dir=target_dir, 420 delete=False, 421 ) as temp_file: 422 # Write random bytes to the file 423 temp_file.write(os.urandom(file_size_in_bytes)) 424 temp_file_path = Path(temp_file.name) 425 LOG.info(f"Temporary file created in: {temp_file_path}") 426 return temp_file_path 427 except Exception as e: 428 LOG.error(f"Failed to create temporary file: {e}") 429 return None 430 431 if network_type == "unthrottled": 432 bandwidth_str = "1000Mbit" 433 else: 434 bandwidth_str, _ = self.network_type_to_bandwidth_rtt(network_type) 435 bandwidth_mbit = float(bandwidth_str.replace("Mbit", "")) 436 437 file_size = calculate_file_size(bandwidth_mbit) 438 temp_file_path = generate_temp_file(file_size, tempfile.gettempdir()) 439 return temp_file_path, file_size 440 441 def generate_download_test_html(self, temp_path, test_file_name): 442 html_content = f""" 443 <!DOCTYPE html> 444 <html> 445 <head> 446 <title>Download test</title> 447 </head> 448 <body> 449 <section>Download test</section> 450 <button id='downloadBtn'>Download Test</button> 451 <p id='download_status'></p> 452 <p id="progress"> </p> 453 <script> 454 let download_status = ''; 455 456 function set_status(status) {{ 457 download_status = status; 458 console.log('download_status:' + status); 459 document.getElementById('download_status').textContent = status; 460 }} 461 462 set_status('not_started'); 463 464 const handleDownloadTest = () => {{ 465 set_status('started'); 466 const startTime = performance.now(); 467 console.log('start'); 468 fetch('/{test_file_name}') 469 .then((response) => response.blob()) 470 .then((_) => {{ 471 console.log('done'); 472 const endTime = performance.now(); 473 const downloadTime = endTime - startTime; 474 set_status('success time:' + downloadTime); 475 }}) 476 .catch((error) => {{ 477 console.error(error); 478 set_status('error'); 479 }}); 480 }}; 481 document 482 .querySelector('#downloadBtn') 483 .addEventListener('click', handleDownloadTest); 484 </script> 485 </body> 486 </html> 487 """ 488 # Write the HTML content to the file 489 prefix = "download_test_" 490 suffix = ".html" 491 with tempfile.NamedTemporaryFile( 492 mode="w", 493 encoding="utf-8", 494 prefix=prefix, 495 suffix=suffix, 496 dir=temp_path, 497 delete=False, 498 ) as html_file: 499 html_file.write(html_content) 500 return Path(html_file.name) 501 return None 502 503 def start_iperf3(self, port): 504 command = ["iperf3", "--server", "--interval", "10", "--port", str(port)] 505 server_process = Popen( 506 command, 507 stdin=PIPE, 508 stdout=PIPE, 509 stderr=PIPE, 510 universal_newlines=True, 511 start_new_session=True, 512 ) 513 514 def read_output(pipe, log_func): 515 for line in iter(pipe.readline, ""): 516 log_func(line) 517 518 server_stdout = threading.Thread( 519 target=read_output, args=(server_process.stdout, LOG.info) 520 ) 521 server_stderr = threading.Thread( 522 target=read_output, args=(server_process.stderr, LOG.info) 523 ) 524 server_stdout.start() 525 server_stderr.start() 526 527 # Configuring MSS to 1240 and buffer length to 1200 leads to an 528 # IP payload of 1280 bytes for iperf3. 529 # This is consistent with the IP payload size observed in Firefox. 530 command = [ 531 "iperf3", 532 "--client", 533 "127.0.0.1", 534 "-p", 535 str(port), 536 "-M", 537 "1240", # MSS 538 "-l", 539 "1200", # The length of buffer to write 540 "--time", 541 "30", # The time in seconds to transmit for 542 "--interval", 543 "10", # Reporting interval in seconds 544 ] 545 546 client_process = Popen( 547 command, 548 stdin=PIPE, 549 stdout=PIPE, 550 stderr=PIPE, 551 universal_newlines=True, 552 start_new_session=True, 553 ) 554 555 client_process.wait() 556 557 self.shutdown_process("iperf3_client", client_process) 558 self.shutdown_process("iperf3_server", server_process) 559 server_stdout.join() 560 server_stderr.join() 561 562 def iperf3_baseline(self): 563 self.run_command("sysctl net.ipv4.tcp_rmem") 564 self.run_command("sysctl net.ipv4.tcp_wmem") 565 tcp_port = self.find_free_port(socket.SOCK_STREAM) 566 LOG.info(f"iperf3_baseline on port:{tcp_port}") 567 568 if not self.check_tc_command(): 569 raise Exception("tc is not available") 570 571 if not self.apply_network_throttling( 572 self.interface, 573 self.network_type, 574 self.packet_loss_rate, 575 ("tcp", tcp_port), 576 ): 577 raise Exception("apply_network_throttling failed") 578 579 self.start_iperf3(tcp_port) 580 581 def modify_command(self, cmd, test): 582 if not self._is_chrome: 583 cmd += [ 584 "--firefox.acceptInsecureCerts", 585 "true", 586 ] 587 protocol = "tcp" 588 if self.http_version == "h3": 589 protocol = "udp" 590 self.caddy_port = self.find_free_port(socket.SOCK_DGRAM) 591 if not self._is_chrome: 592 cmd += [ 593 "--firefox.preference", 594 f"network.http.http3.alt-svc-mapping-for-testing:localhost;h3=:{self.caddy_port}", 595 "--firefox.preference", 596 "network.http.http3.force-use-alt-svc-mapping-for-testing:true", 597 "--firefox.preference", 598 "network.http.http3.disable_when_third_party_roots_found:false", 599 ] 600 else: 601 spki = "VCIlmPM9NkgFQtrs4Oa5TeFcDu6MWRTKSNdePEhOgD8=" 602 cmd += [ 603 f"--chrome.args=--origin-to-force-quic-on=localhost:{self.caddy_port}", 604 f"--chrome.args=--ignore-certificate-errors-spki-list={spki}", 605 ] 606 elif self.http_version == "h2": 607 self.caddy_port = self.find_free_port(socket.SOCK_STREAM) 608 if self._is_chrome: 609 spki = "VCIlmPM9NkgFQtrs4Oa5TeFcDu6MWRTKSNdePEhOgD8=" 610 cmd += [ 611 f"--chrome.args=--ignore-certificate-errors-spki-list={spki}", 612 ] 613 else: 614 raise Exception("Unsupported HTTP version") 615 616 self.get_network_conditions(cmd) 617 temp_file_path = None 618 download_html = None 619 620 if self.network_type != "unthrottled": 621 self.iperf3_baseline() 622 623 if self.transfer_type == "upload": 624 cmd += [ 625 "--browsertime.server_url", 626 f"https://localhost:{self.caddy_port}", 627 ] 628 elif self.transfer_type == "download": 629 temp_file_path, file_size = self.network_type_to_temp_file( 630 self.network_type 631 ) 632 if not temp_file_path: 633 raise Exception("Failed to generate temporary file") 634 self.cleanup.append(lambda: temp_file_path.unlink()) 635 636 download_html = self.generate_download_test_html( 637 tempfile.gettempdir(), temp_file_path.name 638 ) 639 if not download_html: 640 raise Exception("Failed to generate file for download test") 641 self.cleanup.append(lambda: download_html.unlink()) 642 cmd += [ 643 "--browsertime.server_url", 644 f"https://localhost:{self.caddy_port}/{download_html.name}", 645 "--browsertime.test_file_size", 646 str(file_size), 647 ] 648 649 LOG.info("modify_command: %s" % cmd) 650 651 # We know that cmd[0] is the path to nodejs. 652 self.browsertime_node = Path(cmd[0]) 653 self.backend_server = self.start_backend_server("benchmark_backend_server.js") 654 if self.backend_server: 655 self.caddy_server = self.start_caddy(download_html, temp_file_path) 656 if self.caddy_server is None: 657 raise Exception("Failed to start test servers") 658 659 if self.network_type != "unthrottled": 660 if not self.check_tc_command(): 661 raise Exception("tc is not available") 662 if not self.apply_network_throttling( 663 self.interface, 664 self.network_type, 665 self.packet_loss_rate, 666 (protocol, self.caddy_port), 667 ): 668 raise Exception("apply_network_throttling failed") 669 670 def handle_result(self, gt_result, raw_result, last_result=False, **kwargs): 671 goodput_key = ( 672 "upload-goodput" if self.transfer_type == "upload" else "download-goodput" 673 ) 674 675 def get_goodput(data): 676 try: 677 extras = data.get("extras", []) 678 if extras and isinstance(extras, list): 679 custom_data = extras[0].get("custom_data", {}) 680 if goodput_key in custom_data: 681 return custom_data[goodput_key] 682 return None # Return None if any key or index is missing 683 except Exception: 684 return None 685 686 goodput = get_goodput(raw_result) 687 if not goodput: 688 return 689 690 LOG.info(f"Goodput: {goodput}") 691 for g in goodput: 692 gt_result["measurements"].setdefault(goodput_key, []).append(g) 693 694 def _build_subtest(self, measurement_name, replicates, test): 695 unit = test.get("unit", "Mbit/s") 696 if test.get("subtest_unit"): 697 unit = test.get("subtest_unit") 698 699 return { 700 "name": measurement_name, 701 "lowerIsBetter": test.get("lower_is_better", False), 702 "alertThreshold": float(test.get("alert_threshold", 2.0)), 703 "unit": unit, 704 "replicates": replicates, 705 "shouldAlert": True, 706 "value": round(filters.mean(replicates), 3), 707 } 708 709 def summarize_test(self, test, suite, **kwargs): 710 suite["type"] = "benchmark" 711 if suite["subtests"] == {}: 712 suite["subtests"] = [] 713 for measurement_name, replicates in test["measurements"].items(): 714 if not replicates: 715 continue 716 suite["subtests"].append( 717 self._build_subtest(measurement_name, replicates, test) 718 ) 719 suite["subtests"].sort(key=lambda subtest: subtest["name"]) 720 721 def summarize_suites(self, suites): 722 for index, item in enumerate(suites): 723 if "extraOptions" in item: 724 item["extraOptions"].append(self.network_type) 725 loss_str = ( 726 f"loss-{self.packet_loss_rate}" 727 if self.packet_loss_rate 728 else "loss-0" 729 ) 730 item["extraOptions"].append(loss_str) 731 732 def shutdown_process(self, name, proc): 733 LOG.info("%s server shutting down ..." % name) 734 if proc.poll() is not None: 735 LOG.info("server already dead %s" % proc.poll()) 736 else: 737 LOG.info("server pid is %s" % str(proc.pid)) 738 try: 739 os.killpg(proc.pid, signal.SIGTERM) 740 except Exception as e: 741 LOG.error("Failed during kill: " + str(e)) 742 743 def clean_up(self): 744 if self.caddy_server: 745 self.shutdown_process("Caddy", self.caddy_server) 746 if self.backend_server: 747 self.shutdown_process("Backend", self.backend_server) 748 if self.caddy_stdout: 749 self.caddy_stdout.join() 750 if self.caddy_stderr: 751 self.caddy_stderr.join() 752 while self.cleanup: 753 func = self.cleanup.pop() 754 func()