generate.py (6329B)
1 #!/usr/bin/env python3 2 3 import itertools 4 import os 5 6 import jinja2 7 import yaml 8 9 HERE = os.path.abspath(os.path.dirname(__file__)) 10 PROJECT_ROOT = os.path.join(HERE, '..', '..', '..') 11 12 def find_templates(starting_directory): 13 for directory, subdirectories, file_names in os.walk(starting_directory): 14 for file_name in file_names: 15 if file_name.startswith('.'): 16 continue 17 yield file_name, os.path.join(directory, file_name) 18 19 def test_name(directory, template_name, subtest_flags): 20 ''' 21 Create a test name based on a template and the WPT file name flags [1] 22 required for a given subtest. This name is used to determine how subtests 23 may be grouped together. In order to promote grouping, the combination uses 24 a few aspects of how file name flags are interpreted: 25 26 - repeated flags have no effect, so duplicates are removed 27 - flag sequence does not matter, so flags are consistently sorted 28 29 directory | template_name | subtest_flags | result 30 ----------|------------------|-----------------|------- 31 cors | image.html | [] | cors/image.html 32 cors | image.https.html | [] | cors/image.https.html 33 cors | image.html | [https] | cors/image.https.html 34 cors | image.https.html | [https] | cors/image.https.html 35 cors | image.https.html | [https] | cors/image.https.html 36 cors | image.sub.html | [https] | cors/image.https.sub.html 37 cors | image.https.html | [sub] | cors/image.https.sub.html 38 39 [1] docs/writing-tests/file-names.md 40 ''' 41 template_name_parts = template_name.split('.') 42 flags = set(subtest_flags) | set(template_name_parts[1:-1]) 43 test_name_parts = ( 44 [template_name_parts[0]] + 45 sorted(flags) + 46 [template_name_parts[-1]] 47 ) 48 return os.path.join(directory, '.'.join(test_name_parts)) 49 50 def merge(a, b): 51 if type(a) != type(b): 52 raise Exception('Cannot merge disparate types') 53 if type(a) == list: 54 return a + b 55 if type(a) == dict: 56 merged = {} 57 58 for key in a: 59 if key in b: 60 merged[key] = merge(a[key], b[key]) 61 else: 62 merged[key] = a[key] 63 64 for key in b: 65 if not key in a: 66 merged[key] = b[key] 67 68 return merged 69 70 raise Exception('Cannot merge {} type'.format(type(a).__name__)) 71 72 def product(a, b): 73 ''' 74 Given two lists of objects, compute their Cartesian product by merging the 75 elements together. For example, 76 77 product( 78 [{'a': 1}, {'b': 2}], 79 [{'c': 3}, {'d': 4}, {'e': 5}] 80 ) 81 82 returns the following list: 83 84 [ 85 {'a': 1, 'c': 3}, 86 {'a': 1, 'd': 4}, 87 {'a': 1, 'e': 5}, 88 {'b': 2, 'c': 3}, 89 {'b': 2, 'd': 4}, 90 {'b': 2, 'e': 5} 91 ] 92 ''' 93 result = [] 94 95 for a_object in a: 96 for b_object in b: 97 result.append(merge(a_object, b_object)) 98 99 return result 100 101 def make_provenance(project_root, cases, template): 102 return '\n'.join([ 103 'This test was procedurally generated. Please do not modify it directly.', 104 'Sources:', 105 '- {}'.format(os.path.relpath(cases, project_root)), 106 '- {}'.format(os.path.relpath(template, project_root)) 107 ]) 108 109 def collection_filter(obj, title): 110 if not obj: 111 return 'no {}'.format(title) 112 113 members = [] 114 for name, value in obj.items(): 115 if value == '': 116 members.append(name) 117 else: 118 members.append('{}={}'.format(name, value)) 119 120 return '{}: {}'.format(title, ', '.join(members)) 121 122 def pad_filter(value, side, padding): 123 if not value: 124 return '' 125 if side == 'start': 126 return padding + value 127 128 return value + padding 129 130 def main(config_file): 131 with open(config_file, 'r') as handle: 132 config = yaml.safe_load(handle.read()) 133 134 templates_directory = os.path.normpath( 135 os.path.join(os.path.dirname(config_file), config['templates']) 136 ) 137 138 environment = jinja2.Environment( 139 variable_start_string='[%', 140 variable_end_string='%]' 141 ) 142 environment.filters['collection'] = collection_filter 143 environment.filters['pad'] = pad_filter 144 templates = {} 145 subtests = {} 146 147 for template_name, path in find_templates(templates_directory): 148 subtests[template_name] = [] 149 with open(path, 'r') as handle: 150 templates[template_name] = environment.from_string(handle.read()) 151 152 for case in config['cases']: 153 unused_templates = set(templates) - set(case['template_axes']) 154 155 # This warning is intended to help authors avoid mistakenly omitting 156 # templates. It can be silenced by extending the`template_axes` 157 # dictionary with an empty list for templates which are intentionally 158 # unused. 159 if unused_templates: 160 print( 161 'Warning: case does not reference the following templates:' 162 ) 163 print('\n'.join('- {}'.format(name) for name in unused_templates)) 164 165 common_axis = product( 166 case['common_axis'], [case.get('all_subtests', {})] 167 ) 168 169 for template_name, template_axis in case['template_axes'].items(): 170 subtests[template_name].extend(product(common_axis, template_axis)) 171 172 for template_name, template in templates.items(): 173 provenance = make_provenance( 174 PROJECT_ROOT, 175 config_file, 176 os.path.join(templates_directory, template_name) 177 ) 178 get_filename = lambda subtest: test_name( 179 config['output_directory'], 180 template_name, 181 subtest['filename_flags'] 182 ) 183 subtests_by_filename = itertools.groupby( 184 sorted(subtests[template_name], key=get_filename), 185 key=get_filename 186 ) 187 for filename, some_subtests in subtests_by_filename: 188 with open(filename, 'w') as handle: 189 handle.write(templates[template_name].render( 190 subtests=list(some_subtests), 191 provenance=provenance 192 ) + '\n') 193 194 if __name__ == '__main__': 195 main('fetch-metadata.conf.yml')