generate.py (18914B)
1 #!/usr/bin/env python3 2 3 import argparse 4 import collections 5 import copy 6 import json 7 import os 8 import sys 9 10 import spec_validator 11 import util 12 13 14 def expand_pattern(expansion_pattern, test_expansion_schema): 15 expansion = {} 16 for artifact_key in expansion_pattern: 17 artifact_value = expansion_pattern[artifact_key] 18 if artifact_value == '*': 19 expansion[artifact_key] = test_expansion_schema[artifact_key] 20 elif isinstance(artifact_value, list): 21 expansion[artifact_key] = artifact_value 22 elif isinstance(artifact_value, dict): 23 # Flattened expansion. 24 expansion[artifact_key] = [] 25 values_dict = expand_pattern(artifact_value, 26 test_expansion_schema[artifact_key]) 27 for sub_key in values_dict.keys(): 28 expansion[artifact_key] += values_dict[sub_key] 29 else: 30 expansion[artifact_key] = [artifact_value] 31 32 return expansion 33 34 35 def permute_expansion(expansion, 36 artifact_order, 37 selection={}, 38 artifact_index=0): 39 assert isinstance(artifact_order, list), "artifact_order should be a list" 40 41 if artifact_index >= len(artifact_order): 42 yield selection 43 return 44 45 artifact_key = artifact_order[artifact_index] 46 47 for artifact_value in expansion[artifact_key]: 48 selection[artifact_key] = artifact_value 49 for next_selection in permute_expansion(expansion, artifact_order, 50 selection, artifact_index + 1): 51 yield next_selection 52 53 54 # Dumps the test config `selection` into a serialized JSON string. 55 def dump_test_parameters(selection): 56 return json.dumps( 57 selection, 58 indent=2, 59 separators=(',', ': '), 60 sort_keys=True, 61 cls=util.CustomEncoder) 62 63 64 def get_test_filename(spec_directory, spec_json, selection): 65 '''Returns the filname for the main test HTML file''' 66 67 selection_for_filename = copy.deepcopy(selection) 68 # Use 'unset' rather than 'None' in test filenames. 69 if selection_for_filename['delivery_value'] is None: 70 selection_for_filename['delivery_value'] = 'unset' 71 72 return os.path.join( 73 spec_directory, 74 spec_json['test_file_path_pattern'] % selection_for_filename) 75 76 77 def get_csp_value(value): 78 ''' 79 Returns actual CSP header values (e.g. "worker-src 'self'") for the 80 given string used in PolicyDelivery's value (e.g. "worker-src-self"). 81 ''' 82 83 # script-src 84 # Test-related scripts like testharness.js and inline scripts containing 85 # test bodies. 86 # 'unsafe-inline' is added as a workaround here. This is probably not so 87 # bad, as it shouldn't intefere non-inline-script requests that we want to 88 # test. 89 if value == 'script-src-wildcard': 90 return "script-src * 'unsafe-inline'" 91 if value == 'script-src-self': 92 return "script-src 'self' 'unsafe-inline'" 93 # Workaround for "script-src 'none'" would be more complicated, because 94 # - "script-src 'none' 'unsafe-inline'" is handled somehow differently from 95 # "script-src 'none'", i.e. 96 # https://w3c.github.io/webappsec-csp/#match-url-to-source-list Step 3 97 # handles the latter but not the former. 98 # - We need nonce- or path-based additional values to allow same-origin 99 # test scripts like testharness.js. 100 # Therefore, we disable 'script-src-none' tests for now in 101 # `/content-security-policy/spec.src.json`. 102 if value == 'script-src-none': 103 return "script-src 'none'" 104 105 # worker-src 106 if value == 'worker-src-wildcard': 107 return 'worker-src *' 108 if value == 'worker-src-self': 109 return "worker-src 'self'" 110 if value == 'worker-src-none': 111 return "worker-src 'none'" 112 raise Exception('Invalid delivery_value: %s' % value) 113 114 def handle_deliveries(policy_deliveries): 115 ''' 116 Generate <meta> elements and HTTP headers for the given list of 117 PolicyDelivery. 118 TODO(hiroshige): Merge duplicated code here, scope/document.py, etc. 119 ''' 120 121 meta = '' 122 headers = {} 123 124 for delivery in policy_deliveries: 125 if delivery.value is None: 126 continue 127 if delivery.key == 'referrerPolicy': 128 if delivery.delivery_type == 'meta': 129 meta += \ 130 '<meta name="referrer" content="%s">' % delivery.value 131 elif delivery.delivery_type == 'http-rp': 132 headers['Referrer-Policy'] = delivery.value 133 # TODO(kristijanburnik): Limit to WPT origins. 134 headers['Access-Control-Allow-Origin'] = '*' 135 else: 136 raise Exception( 137 'Invalid delivery_type: %s' % delivery.delivery_type) 138 elif delivery.key == 'mixedContent': 139 assert (delivery.value == 'opt-in') 140 if delivery.delivery_type == 'meta': 141 meta += '<meta http-equiv="Content-Security-Policy" ' + \ 142 'content="block-all-mixed-content">' 143 elif delivery.delivery_type == 'http-rp': 144 headers['Content-Security-Policy'] = 'block-all-mixed-content' 145 else: 146 raise Exception( 147 'Invalid delivery_type: %s' % delivery.delivery_type) 148 elif delivery.key == 'contentSecurityPolicy': 149 csp_value = get_csp_value(delivery.value) 150 if delivery.delivery_type == 'meta': 151 meta += '<meta http-equiv="Content-Security-Policy" ' + \ 152 'content="' + csp_value + '">' 153 elif delivery.delivery_type == 'http-rp': 154 headers['Content-Security-Policy'] = csp_value 155 else: 156 raise Exception( 157 'Invalid delivery_type: %s' % delivery.delivery_type) 158 elif delivery.key == 'upgradeInsecureRequests': 159 # https://w3c.github.io/webappsec-upgrade-insecure-requests/#delivery 160 assert (delivery.value == 'upgrade') 161 if delivery.delivery_type == 'meta': 162 meta += '<meta http-equiv="Content-Security-Policy" ' + \ 163 'content="upgrade-insecure-requests">' 164 elif delivery.delivery_type == 'http-rp': 165 headers[ 166 'Content-Security-Policy'] = 'upgrade-insecure-requests' 167 else: 168 raise Exception( 169 'Invalid delivery_type: %s' % delivery.delivery_type) 170 else: 171 raise Exception('Invalid delivery_key: %s' % delivery.key) 172 return {"meta": meta, "headers": headers} 173 174 175 def generate_selection(spec_json, selection): 176 ''' 177 Returns a scenario object (with a top-level source_context_list entry, 178 which will be removed in generate_test_file() later). 179 ''' 180 181 target_policy_delivery = util.PolicyDelivery(selection['delivery_type'], 182 selection['delivery_key'], 183 selection['delivery_value']) 184 del selection['delivery_type'] 185 del selection['delivery_key'] 186 del selection['delivery_value'] 187 188 # Parse source context list and policy deliveries of source contexts. 189 # `util.ShouldSkip()` exceptions are raised if e.g. unsuppported 190 # combinations of source contexts and policy deliveries are used. 191 source_context_list_scheme = spec_json['source_context_list_schema'][ 192 selection['source_context_list']] 193 selection['source_context_list'] = [ 194 util.SourceContext.from_json(source_context, target_policy_delivery, 195 spec_json['source_context_schema']) 196 for source_context in source_context_list_scheme['sourceContextList'] 197 ] 198 199 # Check if the subresource is supported by the innermost source context. 200 innermost_source_context = selection['source_context_list'][-1] 201 supported_subresource = spec_json['source_context_schema'][ 202 'supported_subresource'][innermost_source_context.source_context_type] 203 if supported_subresource != '*': 204 if selection['subresource'] not in supported_subresource: 205 raise util.ShouldSkip() 206 207 # Parse subresource policy deliveries. 208 selection[ 209 'subresource_policy_deliveries'] = util.PolicyDelivery.list_from_json( 210 source_context_list_scheme['subresourcePolicyDeliveries'], 211 target_policy_delivery, spec_json['subresource_schema'] 212 ['supported_delivery_type'][selection['subresource']]) 213 214 # Generate per-scenario test description. 215 selection['test_description'] = spec_json[ 216 'test_description_template'] % selection 217 218 return selection 219 220 221 def generate_test_file(spec_directory, test_helper_filenames, 222 test_html_template_basename, test_filename, scenarios): 223 ''' 224 Generates a test HTML file (and possibly its associated .headers file) 225 from `scenarios`. 226 ''' 227 228 # Scenarios for the same file should have the same `source_context_list`, 229 # including the top-level one. 230 # Note: currently, non-top-level source contexts aren't necessarily required 231 # to be the same, but we set this requirement as it will be useful e.g. when 232 # we e.g. reuse a worker among multiple scenarios. 233 for scenario in scenarios: 234 assert (scenario['source_context_list'] == scenarios[0] 235 ['source_context_list']) 236 237 # We process the top source context below, and do not include it in 238 # the JSON objects (i.e. `scenarios`) in generated HTML files. 239 top_source_context = scenarios[0]['source_context_list'].pop(0) 240 assert (top_source_context.source_context_type == 'top') 241 for scenario in scenarios[1:]: 242 assert (scenario['source_context_list'].pop(0) == top_source_context) 243 244 parameters = {} 245 246 # Sort scenarios, to avoid unnecessary diffs due to different orders in 247 # `scenarios`. 248 serialized_scenarios = sorted( 249 [dump_test_parameters(scenario) for scenario in scenarios]) 250 251 parameters['scenarios'] = ",\n".join(serialized_scenarios).replace( 252 "\n", "\n" + " " * 10) 253 254 test_directory = os.path.dirname(test_filename) 255 256 parameters['helper_js'] = "" 257 for test_helper_filename in test_helper_filenames: 258 parameters['helper_js'] += ' <script src="%s"></script>\n' % ( 259 os.path.relpath(test_helper_filename, test_directory)) 260 parameters['sanity_checker_js'] = os.path.relpath( 261 os.path.join(spec_directory, 'generic', 'sanity-checker.js'), 262 test_directory) 263 parameters['spec_json_js'] = os.path.relpath( 264 os.path.join(spec_directory, 'generic', 'spec_json.js'), 265 test_directory) 266 267 test_headers_filename = test_filename + ".headers" 268 269 test_html_template = util.get_template(test_html_template_basename) 270 disclaimer_template = util.get_template('disclaimer.template') 271 272 html_template_filename = os.path.join(util.template_directory, 273 test_html_template_basename) 274 generated_disclaimer = disclaimer_template \ 275 % {'generating_script_filename': os.path.relpath(sys.argv[0], 276 util.test_root_directory), 277 'spec_directory': os.path.relpath(spec_directory, 278 util.test_root_directory)} 279 280 # Adjust the template for the test invoking JS. Indent it to look nice. 281 parameters['generated_disclaimer'] = generated_disclaimer.rstrip() 282 283 # Directory for the test files. 284 try: 285 os.makedirs(test_directory) 286 except: 287 pass 288 289 delivery = handle_deliveries(top_source_context.policy_deliveries) 290 291 if len(delivery['headers']) > 0: 292 with open(test_headers_filename, "w") as f: 293 for header in delivery['headers']: 294 f.write('%s: %s\n' % (header, delivery['headers'][header])) 295 296 parameters['meta_delivery_method'] = delivery['meta'] 297 # Obey the lint and pretty format. 298 if len(parameters['meta_delivery_method']) > 0: 299 parameters['meta_delivery_method'] = "\n " + \ 300 parameters['meta_delivery_method'] 301 302 # Write out the generated HTML file. 303 util.write_file(test_filename, test_html_template % parameters) 304 305 306 def generate_test_source_files(spec_directory, test_helper_filenames, 307 spec_json, target): 308 test_expansion_schema = spec_json['test_expansion_schema'] 309 specification = spec_json['specification'] 310 311 if target == "debug": 312 spec_json_js_template = util.get_template('spec_json.js.template') 313 util.write_file( 314 os.path.join(spec_directory, "generic", "spec_json.js"), 315 spec_json_js_template % {'spec_json': json.dumps(spec_json)}) 316 util.write_file( 317 os.path.join(spec_directory, "generic", 318 "debug-output.spec.src.json"), 319 json.dumps(spec_json, indent=2, separators=(',', ': '))) 320 321 # Choose a debug/release template depending on the target. 322 html_template = "test.%s.html.template" % target 323 324 artifact_order = list(test_expansion_schema.keys()) 325 artifact_order.remove('expansion') 326 327 excluded_selection_pattern = '' 328 for key in artifact_order: 329 excluded_selection_pattern += '%(' + key + ')s/' 330 331 # Create list of excluded tests. 332 exclusion_dict = set() 333 for excluded_pattern in spec_json['excluded_tests']: 334 excluded_expansion = \ 335 expand_pattern(excluded_pattern, test_expansion_schema) 336 for excluded_selection in permute_expansion(excluded_expansion, 337 artifact_order): 338 excluded_selection['delivery_key'] = spec_json['delivery_key'] 339 exclusion_dict.add(excluded_selection_pattern % excluded_selection) 340 341 # `scenarios[filename]` represents the list of scenario objects to be 342 # generated into `filename`. 343 scenarios = {} 344 345 for spec in specification: 346 # Used to make entries with expansion="override" override preceding 347 # entries with the same |selection_path|. 348 output_dict = {} 349 350 for expansion_pattern in spec['test_expansion']: 351 expansion = expand_pattern(expansion_pattern, 352 test_expansion_schema) 353 for selection in permute_expansion(expansion, artifact_order): 354 selection['delivery_key'] = spec_json['delivery_key'] 355 selection_path = spec_json['selection_pattern'] % selection 356 if selection_path in output_dict: 357 if expansion_pattern['expansion'] != 'override': 358 print("Error: expansion is default in:") 359 print(dump_test_parameters(selection)) 360 print("but overrides:") 361 print(dump_test_parameters( 362 output_dict[selection_path])) 363 sys.exit(1) 364 output_dict[selection_path] = copy.deepcopy(selection) 365 366 for selection_path in output_dict: 367 selection = output_dict[selection_path] 368 if (excluded_selection_pattern % selection) in exclusion_dict: 369 print('Excluding selection:', selection_path) 370 continue 371 try: 372 test_filename = get_test_filename(spec_directory, spec_json, 373 selection) 374 scenario = generate_selection(spec_json, selection) 375 scenarios[test_filename] = scenarios.get(test_filename, 376 []) + [scenario] 377 except util.ShouldSkip: 378 continue 379 380 for filename in scenarios: 381 generate_test_file(spec_directory, test_helper_filenames, 382 html_template, filename, scenarios[filename]) 383 384 385 def merge_json(base, child): 386 for key in child: 387 if key not in base: 388 base[key] = child[key] 389 continue 390 # `base[key]` and `child[key]` both exists. 391 if isinstance(base[key], list) and isinstance(child[key], list): 392 base[key].extend(child[key]) 393 elif isinstance(base[key], dict) and isinstance(child[key], dict): 394 merge_json(base[key], child[key]) 395 else: 396 base[key] = child[key] 397 398 399 def main(): 400 parser = argparse.ArgumentParser( 401 description='Test suite generator utility') 402 parser.add_argument( 403 '-t', 404 '--target', 405 type=str, 406 choices=("release", "debug"), 407 default="release", 408 help='Sets the appropriate template for generating tests') 409 parser.add_argument( 410 '-s', 411 '--spec', 412 type=str, 413 default=os.getcwd(), 414 help='Specify a file used for describing and generating the tests') 415 # TODO(kristijanburnik): Add option for the spec_json file. 416 args = parser.parse_args() 417 418 spec_directory = os.path.abspath(args.spec) 419 420 # Read `spec.src.json` files, starting from `spec_directory`, and 421 # continuing to parent directories as long as `spec.src.json` exists. 422 spec_filenames = [] 423 test_helper_filenames = [] 424 spec_src_directory = spec_directory 425 while len(spec_src_directory) >= len(util.test_root_directory): 426 spec_filename = os.path.join(spec_src_directory, "spec.src.json") 427 if not os.path.exists(spec_filename): 428 break 429 spec_filenames.append(spec_filename) 430 test_filename = os.path.join(spec_src_directory, 'generic', 431 'test-case.sub.js') 432 assert (os.path.exists(test_filename)) 433 test_helper_filenames.append(test_filename) 434 spec_src_directory = os.path.abspath( 435 os.path.join(spec_src_directory, "..")) 436 437 spec_filenames = list(reversed(spec_filenames)) 438 test_helper_filenames = list(reversed(test_helper_filenames)) 439 440 if len(spec_filenames) == 0: 441 print('Error: No spec.src.json is found at %s.' % spec_directory) 442 return 443 444 # Load the default spec JSON file, ... 445 default_spec_filename = os.path.join(util.script_directory, 446 'spec.src.json') 447 spec_json = collections.OrderedDict() 448 if os.path.exists(default_spec_filename): 449 spec_json = util.load_spec_json(default_spec_filename) 450 451 # ... and then make spec JSON files in subdirectories override the default. 452 for spec_filename in spec_filenames: 453 child_spec_json = util.load_spec_json(spec_filename) 454 merge_json(spec_json, child_spec_json) 455 456 spec_validator.assert_valid_spec_json(spec_json) 457 generate_test_source_files(spec_directory, test_helper_filenames, 458 spec_json, args.target) 459 460 461 if __name__ == '__main__': 462 main()