spec_validator.py (10049B)
1 #!/usr/bin/env python3 2 3 import json, sys 4 5 6 def assert_non_empty_string(obj, field): 7 assert field in obj, 'Missing field "%s"' % field 8 assert isinstance(obj[field], str), \ 9 'Field "%s" must be a string' % field 10 assert len(obj[field]) > 0, 'Field "%s" must not be empty' % field 11 12 13 def assert_non_empty_list(obj, field): 14 assert isinstance(obj[field], list), \ 15 '%s must be a list' % field 16 assert len(obj[field]) > 0, \ 17 '%s list must not be empty' % field 18 19 20 def assert_non_empty_dict(obj, field): 21 assert isinstance(obj[field], dict), \ 22 '%s must be a dict' % field 23 assert len(obj[field]) > 0, \ 24 '%s dict must not be empty' % field 25 26 27 def assert_contains(obj, field): 28 assert field in obj, 'Must contain field "%s"' % field 29 30 31 def assert_value_from(obj, field, items): 32 assert obj[field] in items, \ 33 'Field "%s" must be from: %s' % (field, str(items)) 34 35 36 def assert_atom_or_list_items_from(obj, field, items): 37 if isinstance(obj[field], str) or isinstance( 38 obj[field], int) or obj[field] is None: 39 assert_value_from(obj, field, items) 40 return 41 42 assert isinstance(obj[field], list), '%s must be a list' % field 43 for allowed_value in obj[field]: 44 assert allowed_value != '*', "Wildcard is not supported for lists!" 45 assert allowed_value in items, \ 46 'Field "%s" must be from: %s' % (field, str(items)) 47 48 49 def assert_contains_only_fields(obj, expected_fields): 50 for expected_field in expected_fields: 51 assert_contains(obj, expected_field) 52 53 for actual_field in obj: 54 assert actual_field in expected_fields, \ 55 'Unexpected field "%s".' % actual_field 56 57 58 def leaf_values(schema): 59 if isinstance(schema, list): 60 return schema 61 ret = [] 62 for _, sub_schema in schema.iteritems(): 63 ret += leaf_values(sub_schema) 64 return ret 65 66 67 def assert_value_unique_in(value, used_values): 68 assert value not in used_values, 'Duplicate value "%s"!' % str(value) 69 used_values[value] = True 70 71 72 def assert_valid_artifact(exp_pattern, artifact_key, schema): 73 if isinstance(schema, list): 74 assert_atom_or_list_items_from(exp_pattern, artifact_key, 75 ["*"] + schema) 76 return 77 78 for sub_artifact_key, sub_schema in schema.iteritems(): 79 assert_valid_artifact(exp_pattern[artifact_key], sub_artifact_key, 80 sub_schema) 81 82 83 def validate(spec_json, details): 84 """ Validates the json specification for generating tests. """ 85 86 details['object'] = spec_json 87 assert_contains_only_fields(spec_json, [ 88 "selection_pattern", "test_file_path_pattern", 89 "test_description_template", "test_page_title_template", 90 "specification", "delivery_key", "subresource_schema", 91 "source_context_schema", "source_context_list_schema", 92 "test_expansion_schema", "excluded_tests" 93 ]) 94 assert_non_empty_list(spec_json, "specification") 95 assert_non_empty_dict(spec_json, "test_expansion_schema") 96 assert_non_empty_list(spec_json, "excluded_tests") 97 98 specification = spec_json['specification'] 99 test_expansion_schema = spec_json['test_expansion_schema'] 100 excluded_tests = spec_json['excluded_tests'] 101 102 valid_test_expansion_fields = test_expansion_schema.keys() 103 104 # Should be consistent with `sourceContextMap` in 105 # `/common/security-features/resources/common.sub.js`. 106 valid_source_context_names = [ 107 "top", "iframe", "iframe-blank", "srcdoc", "worker-classic", 108 "worker-module", "worker-classic-data", "worker-module-data", 109 "sharedworker-classic", "sharedworker-module", 110 "sharedworker-classic-data", "sharedworker-module-data" 111 ] 112 113 valid_subresource_names = [ 114 "a-tag", "area-tag", "audio-tag", "form-tag", "iframe-tag", "img-tag", 115 "link-css-tag", "link-prefetch-tag", "object-tag", "picture-tag", 116 "script-tag", "script-tag-dynamic-import", "svg-a-tag", "video-tag" 117 ] + ["beacon", "fetch", "xhr", "websocket"] + [ 118 "worker-classic", "worker-module", "worker-import", 119 "worker-import-data", "sharedworker-classic", "sharedworker-module", 120 "sharedworker-import", "sharedworker-import-data", 121 "serviceworker-classic", "serviceworker-module", 122 "serviceworker-import", "serviceworker-import-data" 123 ] + [ 124 "worklet-animation", "worklet-audio", "worklet-layout", 125 "worklet-paint", "worklet-animation-import", "worklet-audio-import", 126 "worklet-layout-import", "worklet-paint-import", 127 "worklet-animation-import-data", "worklet-audio-import-data", 128 "worklet-layout-import-data", "worklet-paint-import-data" 129 ] 130 131 # Validate each single spec. 132 for spec in specification: 133 details['object'] = spec 134 135 # Validate required fields for a single spec. 136 assert_contains_only_fields(spec, [ 137 'title', 'description', 'specification_url', 'test_expansion' 138 ]) 139 assert_non_empty_string(spec, 'title') 140 assert_non_empty_string(spec, 'description') 141 assert_non_empty_string(spec, 'specification_url') 142 assert_non_empty_list(spec, 'test_expansion') 143 144 for spec_exp in spec['test_expansion']: 145 details['object'] = spec_exp 146 assert_contains_only_fields(spec_exp, valid_test_expansion_fields) 147 148 for artifact in test_expansion_schema: 149 details['test_expansion_field'] = artifact 150 assert_valid_artifact(spec_exp, artifact, 151 test_expansion_schema[artifact]) 152 del details['test_expansion_field'] 153 154 # Validate source_context_schema. 155 details['object'] = spec_json['source_context_schema'] 156 assert_contains_only_fields( 157 spec_json['source_context_schema'], 158 ['supported_delivery_type', 'supported_subresource']) 159 assert_contains_only_fields( 160 spec_json['source_context_schema']['supported_delivery_type'], 161 valid_source_context_names) 162 for source_context in spec_json['source_context_schema'][ 163 'supported_delivery_type']: 164 assert_valid_artifact( 165 spec_json['source_context_schema']['supported_delivery_type'], 166 source_context, test_expansion_schema['delivery_type']) 167 assert_contains_only_fields( 168 spec_json['source_context_schema']['supported_subresource'], 169 valid_source_context_names) 170 for source_context in spec_json['source_context_schema'][ 171 'supported_subresource']: 172 assert_valid_artifact( 173 spec_json['source_context_schema']['supported_subresource'], 174 source_context, leaf_values(test_expansion_schema['subresource'])) 175 176 # Validate subresource_schema. 177 details['object'] = spec_json['subresource_schema'] 178 assert_contains_only_fields(spec_json['subresource_schema'], 179 ['supported_delivery_type']) 180 assert_contains_only_fields( 181 spec_json['subresource_schema']['supported_delivery_type'], 182 leaf_values(test_expansion_schema['subresource'])) 183 for subresource in spec_json['subresource_schema'][ 184 'supported_delivery_type']: 185 assert_valid_artifact( 186 spec_json['subresource_schema']['supported_delivery_type'], 187 subresource, test_expansion_schema['delivery_type']) 188 189 # Validate the test_expansion schema members. 190 details['object'] = test_expansion_schema 191 assert_contains_only_fields(test_expansion_schema, [ 192 'expansion', 'source_scheme', 'source_context_list', 'delivery_type', 193 'delivery_value', 'redirection', 'subresource', 'origin', 'expectation' 194 ]) 195 assert_atom_or_list_items_from(test_expansion_schema, 'expansion', 196 ['default', 'override']) 197 assert_atom_or_list_items_from(test_expansion_schema, 'source_scheme', 198 ['http', 'https']) 199 assert_atom_or_list_items_from( 200 test_expansion_schema, 'source_context_list', 201 spec_json['source_context_list_schema'].keys()) 202 203 # Should be consistent with `preprocess_redirection` in 204 # `/common/security-features/subresource/subresource.py`. 205 assert_atom_or_list_items_from(test_expansion_schema, 'redirection', [ 206 'no-redirect', 'keep-origin', 'swap-origin', 'keep-scheme', 207 'swap-scheme', 'downgrade' 208 ]) 209 for subresource in leaf_values(test_expansion_schema['subresource']): 210 assert subresource in valid_subresource_names, "Invalid subresource %s" % subresource 211 # Should be consistent with getSubresourceOrigin() in 212 # `/common/security-features/resources/common.sub.js`. 213 assert_atom_or_list_items_from(test_expansion_schema, 'origin', [ 214 'same-http', 'same-https', 'same-ws', 'same-wss', 'cross-http', 215 'cross-https', 'cross-ws', 'cross-wss', 'same-http-downgrade', 216 'cross-http-downgrade', 'same-ws-downgrade', 'cross-ws-downgrade' 217 ]) 218 219 # Validate excluded tests. 220 details['object'] = excluded_tests 221 for excluded_test_expansion in excluded_tests: 222 assert_contains_only_fields(excluded_test_expansion, 223 valid_test_expansion_fields) 224 details['object'] = excluded_test_expansion 225 for artifact in test_expansion_schema: 226 details['test_expansion_field'] = artifact 227 assert_valid_artifact(excluded_test_expansion, artifact, 228 test_expansion_schema[artifact]) 229 del details['test_expansion_field'] 230 231 del details['object'] 232 233 234 def assert_valid_spec_json(spec_json): 235 error_details = {} 236 try: 237 validate(spec_json, error_details) 238 except AssertionError as err: 239 print('ERROR:', err) 240 print(json.dumps(error_details, indent=4)) 241 sys.exit(1) 242 243 244 def main(): 245 spec_json = load_spec_json() 246 assert_valid_spec_json(spec_json) 247 print("Spec JSON is valid.") 248 249 250 if __name__ == '__main__': 251 main()