tests.py (10676B)
1 # Library for JSTest tests. 2 # 3 # This contains classes that represent an individual test, including 4 # metadata, and know how to run the tests and determine failures. 5 6 import os 7 import sys 8 from contextlib import contextmanager 9 10 # When run on tbpl, we run each test multiple times with the following 11 # arguments. 12 JITFLAGS = { 13 "all": [ 14 [], # no flags, normal baseline and ion 15 [ 16 "--ion-eager", 17 "--ion-offthread-compile=off", # implies --baseline-eager 18 "--more-compartments", 19 ], 20 [ 21 "--ion-eager", 22 "--ion-offthread-compile=off", 23 "--ion-check-range-analysis", 24 "--ion-extra-checks", 25 "--no-sse3", 26 "--no-threads", 27 ], 28 ["--baseline-eager", "--write-protect-code=off"], 29 ["--no-blinterp", "--no-baseline", "--no-ion", "--more-compartments"], 30 ["--blinterp-eager"], 31 ], 32 # Like 'all' above but for jstests. This has fewer jit-specific 33 # configurations. 34 "jstests": [ 35 [], # no flags, normal baseline and ion 36 [ 37 "--ion-eager", 38 "--ion-offthread-compile=off", # implies --baseline-eager 39 "--more-compartments", 40 ], 41 ["--baseline-eager", "--write-protect-code=off"], 42 ["--no-blinterp", "--no-baseline", "--no-ion", "--more-compartments"], 43 ], 44 # used by jit_test.py 45 "ion": [ 46 ["--baseline-eager", "--write-protect-code=off"], 47 ["--ion-eager", "--ion-offthread-compile=off", "--more-compartments"], 48 ], 49 # Run reduced variants on debug builds, since they take longer time. 50 "debug": [ 51 [], # no flags, normal baseline and ion 52 [ 53 "--ion-eager", 54 "--ion-offthread-compile=off", # implies --baseline-eager 55 "--more-compartments", 56 ], 57 ["--baseline-eager", "--write-protect-code=off"], 58 ], 59 # Cover cases useful for tsan. Note that we test --ion-eager without 60 # --ion-offthread-compile=off here, because it helps catch races. 61 "tsan": [ 62 [], 63 [ 64 "--ion-eager", 65 "--ion-check-range-analysis", 66 "--ion-extra-checks", 67 "--no-sse3", 68 ], 69 ["--no-blinterp", "--no-baseline", "--no-ion"], 70 ], 71 "baseline": [ 72 ["--no-ion"], 73 ], 74 # Interpreter-only, for tools that cannot handle binary code generation. 75 "interp": [ 76 [ 77 "--no-blinterp", 78 "--no-baseline", 79 "--wasm-compiler=none", 80 "--no-native-regexp", 81 ] 82 ], 83 "none": [[]], # no flags, normal baseline and ion 84 } 85 86 87 def get_jitflags(variant, **kwargs): 88 if variant not in JITFLAGS: 89 print(f'Invalid jitflag: "{variant}"') 90 sys.exit(1) 91 if variant == "none" and "none" in kwargs: 92 return kwargs["none"] 93 return JITFLAGS[variant] 94 95 96 def valid_jitflags(): 97 return JITFLAGS.keys() 98 99 100 def get_environment_overlay(js_shell, gc_zeal): 101 """ 102 Build a dict of additional environment variables that must be set to run 103 tests successfully. 104 """ 105 106 # When updating this also update |buildBrowserEnv| in layout/tools/reftest/runreftest.py. 107 env = { 108 # Force Pacific time zone to avoid failures in Date tests. 109 "TZ": "PST8PDT", 110 # Force date strings to English. 111 "LC_ALL": "en_US.UTF-8", 112 # Tell the shell to disable crash dialogs on windows. 113 "XRE_NO_WINDOWS_CRASH_DIALOG": "1", 114 } 115 116 # Add the binary's directory to the library search path so that we find the 117 # nspr and icu we built, instead of the platform supplied ones (or none at 118 # all on windows). 119 if sys.platform.startswith("linux"): 120 env["LD_LIBRARY_PATH"] = os.path.dirname(js_shell) 121 elif sys.platform.startswith("darwin"): 122 env["DYLD_LIBRARY_PATH"] = os.path.dirname(js_shell) 123 elif sys.platform.startswith("win"): 124 env["PATH"] = os.path.dirname(js_shell) 125 126 if gc_zeal: 127 env["JS_GC_ZEAL"] = gc_zeal 128 129 return env 130 131 132 @contextmanager 133 def change_env(env_overlay): 134 # Apply the overlaid environment and record the current state. 135 prior_env = {} 136 for key, val in env_overlay.items(): 137 prior_env[key] = os.environ.get(key, None) 138 if "PATH" in key and key in os.environ: 139 os.environ[key] = f"{val}{os.pathsep}{os.environ[key]}" 140 else: 141 os.environ[key] = val 142 143 try: 144 # Execute with the new environment. 145 yield 146 147 finally: 148 # Restore the prior environment. 149 for key, val in prior_env.items(): 150 if val is not None: 151 os.environ[key] = val 152 else: 153 del os.environ[key] 154 155 156 def get_cpu_count(): 157 """ 158 Guess at a reasonable parallelism count to set as the default for the 159 current machine and run. 160 """ 161 # Python 2.6+ 162 try: 163 import multiprocessing 164 165 return multiprocessing.cpu_count() 166 except (ImportError, NotImplementedError): 167 pass 168 169 # POSIX 170 try: 171 res = int(os.sysconf("SC_NPROCESSORS_ONLN")) 172 if res > 0: 173 return res 174 except (AttributeError, ValueError): 175 pass 176 177 # Windows 178 try: 179 res = int(os.environ["NUMBER_OF_PROCESSORS"]) 180 if res > 0: 181 return res 182 except (KeyError, ValueError): 183 pass 184 185 return 1 186 187 188 class RefTestCase: 189 """A test case consisting of a test and an expected result.""" 190 191 def __init__(self, root, path, extra_helper_paths=None, wpt=None): 192 # str: path of the tests root dir 193 self.root = root 194 # str: path of JS file relative to tests root dir 195 self.path = path 196 # [str]: Extra options to pass to the shell 197 self.options = [] 198 # [str]: JIT flags to pass to the shell 199 self.jitflags = [] 200 # [str]: flags to never pass to the shell for this test 201 self.ignoredflags = [] 202 # str or None: path to reflect-stringify.js file to test 203 # instead of actually running tests 204 self.test_reflect_stringify = None 205 # bool: True => test is module code 206 self.is_module = False 207 # bool: True => test is asynchronous and runs additional code after completing the first 208 # turn of the event loop. 209 self.is_async = False 210 # bool: True => run test, False => don't run 211 self.enable = True 212 # str?: Optional error type 213 self.error = None 214 # bool: expected result, True => pass 215 self.expect = True 216 # bool: True => ignore output as 'random' 217 self.random = False 218 # bool: True => test may run slowly 219 self.slow = False 220 # bool: True => test will not run alongside any other heavy tests 221 self.heavy = False 222 # bool: True => test is test262 testcase with raw flag, that turns off 223 # running shell.js files inside test262 224 self.is_test262_raw = False 225 226 # Use self-hosted XDR instead of parsing the source stored in the binary. 227 # str?: Path computed when generating the command 228 self.selfhosted_xdr_path = None 229 # str: XDR mode (= "off", "encode", "decode") to use with the 230 # self-hosted code. 231 self.selfhosted_xdr_mode = "off" 232 233 # The terms parsed to produce the above properties. 234 self.terms = None 235 236 # The tag between |...| in the test header. 237 self.tag = None 238 239 # Anything occuring after -- in the test header. 240 self.comment = None 241 242 self.extra_helper_paths = extra_helper_paths or [] 243 self.wpt = wpt 244 245 def prefix_command(self): 246 """Return the '-f' options needed to run a test with the given path.""" 247 path = self.path 248 prefix = [] 249 while path != "": 250 assert path != "/" 251 path = os.path.dirname(path) 252 253 if self.is_test262_raw and path != "": 254 # Skip running shell.js under test262 if the test has raw flag. 255 # Top-level shell.js is still necessary to define reportCompare. 256 continue 257 258 shell_path = os.path.join(self.root, path, "shell.js") 259 if os.path.exists(shell_path): 260 prefix.append(shell_path) 261 prefix.append("-f") 262 prefix.reverse() 263 264 for extra_path in self.extra_helper_paths: 265 prefix.append("-f") 266 prefix.append(extra_path) 267 268 return prefix 269 270 def abs_path(self): 271 return os.path.join(self.root, self.path) 272 273 def get_command(self, prefix, tempdir): 274 cmd = prefix + self.jitflags + self.options + self.prefix_command() 275 # Note: The tempdir provided as argument is managed by the caller 276 # should remain alive as long as the test harness. Therefore, the XDR 277 # content of the self-hosted code would be accessible to all JS Shell 278 # instances. 279 if self.selfhosted_xdr_mode != "off": 280 self.selfhosted_xdr_path = os.path.join(tempdir, "shell.xdr") 281 cmd += [ 282 "--selfhosted-xdr-path", 283 self.selfhosted_xdr_path, 284 "--selfhosted-xdr-mode", 285 self.selfhosted_xdr_mode, 286 ] 287 if self.test_reflect_stringify is not None: 288 cmd += [self.test_reflect_stringify, "--check", self.abs_path()] 289 elif self.is_module: 290 cmd += ["--module", self.abs_path()] 291 else: 292 cmd += ["-f", self.abs_path()] 293 for flag in self.ignoredflags: 294 if flag in cmd: 295 cmd.remove(flag) 296 return cmd 297 298 def __str__(self): 299 ans = self.path 300 if not self.enable: 301 ans += ", skip" 302 if self.error is not None: 303 ans += ", error=" + self.error 304 if not self.expect: 305 ans += ", fails" 306 if self.random: 307 ans += ", random" 308 if self.slow: 309 ans += ", slow" 310 if self.heavy: 311 ans += ", heavy" 312 if "-d" in self.options: 313 ans += ", debugMode" 314 return ans 315 316 @staticmethod 317 def build_js_cmd_prefix(js_path, js_args, debugger_prefix): 318 parts = [] 319 if debugger_prefix: 320 parts += debugger_prefix 321 parts.append(js_path) 322 if js_args: 323 parts += js_args 324 return parts 325 326 def __cmp__(self, other): 327 if self.path == other.path: 328 return 0 329 elif self.path < other.path: 330 return -1 331 return 1 332 333 def __hash__(self): 334 return self.path.__hash__() 335 336 def __repr__(self): 337 return "<lib.tests.RefTestCase %s>" % (self.path,)