__main__.py (12451B)
1 # Copyright (C) 2018 and later: Unicode, Inc. and others. 2 # License & terms of use: http://www.unicode.org/copyright.html 3 4 # Python 2/3 Compatibility (ICU-20299) 5 # TODO(ICU-20301): Remove this. 6 from __future__ import print_function 7 8 import argparse 9 import glob as pyglob 10 import io as pyio 11 import json 12 import os 13 import sys 14 15 from . import * 16 from .comment_stripper import CommentStripper 17 from .request_types import CopyRequest 18 from .renderers import makefile, common_exec 19 from . import filtration, utils 20 21 flag_parser = argparse.ArgumentParser( 22 description = """Generates rules for building ICU binary data files from text 23 and other input files in source control. 24 25 Use the --mode option to declare how to execute those rules, either exporting 26 the rules to a Makefile or spawning child processes to run them immediately: 27 28 --mode=gnumake prints a Makefile to standard out. 29 --mode=unix-exec spawns child processes in a Unix-like environment. 30 --mode=windows-exec spawns child processes in a Windows-like environment. 31 32 Tips for --mode=unix-exec 33 ========================= 34 35 Create two empty directories for out_dir and tmp_dir. They will get filled 36 with a lot of intermediate files. 37 38 Set LD_LIBRARY_PATH to include the lib directory. e.g., from icu4c/source: 39 40 $ LD_LIBRARY_PATH=lib PYTHONPATH=python python3 -m icutools.databuilder ... 41 42 Once icutools.databuilder finishes, you have compiled the data, but you have 43 not packaged it into a .dat or .so file. This is done by the separate pkgdata 44 tool in bin. Read the docs of pkgdata: 45 46 $ LD_LIBRARY_PATH=lib ./bin/pkgdata --help 47 48 Example command line to call pkgdata: 49 50 $ LD_LIBRARY_PATH=lib ./bin/pkgdata -m common -p icudt63l -c \\ 51 -O data/icupkg.inc -s $OUTDIR -d $TMPDIR $TMPDIR/icudata.lst 52 53 where $OUTDIR and $TMPDIR are your out and tmp directories, respectively. 54 The above command will create icudt63l.dat in the tmpdir. 55 56 Command-Line Arguments 57 ====================== 58 """, 59 formatter_class = argparse.RawDescriptionHelpFormatter 60 ) 61 62 arg_group_required = flag_parser.add_argument_group("required arguments") 63 arg_group_required.add_argument( 64 "--mode", 65 help = "What to do with the generated rules.", 66 choices = ["gnumake", "unix-exec", "windows-exec", "bazel-exec"], 67 required = True 68 ) 69 70 flag_parser.add_argument( 71 "--src_dir", 72 help = "Path to data source folder (icu4c/source/data).", 73 default = "." 74 ) 75 flag_parser.add_argument( 76 "--filter_file", 77 metavar = "PATH", 78 help = "Path to an ICU data filter JSON file.", 79 default = None 80 ) 81 flag_parser.add_argument( 82 "--include_uni_core_data", 83 help = "Include the full Unicode core data in the dat file.", 84 default = False, 85 action = "store_true" 86 ) 87 flag_parser.add_argument( 88 "--seqmode", 89 help = "Whether to optimize rules to be run sequentially (fewer threads) or in parallel (many threads). Defaults to 'sequential', which is better for unix-exec and windows-exec modes. 'parallel' is often better for massively parallel build systems.", 90 choices = ["sequential", "parallel"], 91 default = "sequential" 92 ) 93 flag_parser.add_argument( 94 "--verbose", 95 help = "Print more verbose output (default false).", 96 default = False, 97 action = "store_true" 98 ) 99 100 arg_group_exec = flag_parser.add_argument_group("arguments for unix-exec and windows-exec modes") 101 arg_group_exec.add_argument( 102 "--out_dir", 103 help = "Path to where to save output data files.", 104 default = "icudata" 105 ) 106 arg_group_exec.add_argument( 107 "--tmp_dir", 108 help = "Path to where to save temporary files.", 109 default = "icutmp" 110 ) 111 arg_group_exec.add_argument( 112 "--tool_dir", 113 help = "Path to where to find binary tools (genrb, etc).", 114 default = "../bin" 115 ) 116 arg_group_exec.add_argument( 117 "--tool_cfg", 118 help = "The build configuration of the tools. Used in 'windows-exec' mode only.", 119 default = "x86/Debug" 120 ) 121 122 123 class Config(object): 124 125 def __init__(self, args): 126 # Process arguments 127 self.max_parallel = (args.seqmode == "parallel") 128 129 # Boolean: Whether to include core Unicode data files in the .dat file 130 self.include_uni_core_data = args.include_uni_core_data 131 132 # Default fields before processing filter file 133 self.filters_json_data = {} 134 self.filter_dir = "ERROR_NO_FILTER_FILE" 135 136 # Process filter file 137 if args.filter_file: 138 try: 139 with open(args.filter_file, "r") as f: 140 print("Note: Applying filters from %s." % args.filter_file, file=sys.stderr) 141 self._parse_filter_file(f) 142 except IOError: 143 print("Error: Could not read filter file %s." % args.filter_file, file=sys.stderr) 144 exit(1) 145 self.filter_dir = os.path.abspath(os.path.dirname(args.filter_file)) 146 147 # Either "unihan" or "implicithan" 148 self.coll_han_type = "unihan" 149 if "collationUCAData" in self.filters_json_data: 150 self.coll_han_type = self.filters_json_data["collationUCAData"] 151 152 # Either "additive" or "subtractive" 153 self.strategy = "subtractive" 154 if "strategy" in self.filters_json_data: 155 self.strategy = self.filters_json_data["strategy"] 156 157 # True or False (could be extended later to support enum/list) 158 self.use_pool_bundle = True 159 if "usePoolBundle" in self.filters_json_data: 160 self.use_pool_bundle = self.filters_json_data["usePoolBundle"] 161 162 def _parse_filter_file(self, f): 163 # Use the Hjson parser if it is available; otherwise, use vanilla JSON. 164 try: 165 import hjson 166 self.filters_json_data = hjson.load(f) 167 except ImportError: 168 self.filters_json_data = json.load(CommentStripper(f)) 169 170 # Optionally pre-validate the JSON schema before further processing. 171 # Some schema errors will be caught later, but this step ensures 172 # maximal validity. 173 try: 174 import jsonschema 175 schema_path = os.path.join(os.path.dirname(__file__), "filtration_schema.json") 176 with open(schema_path) as schema_f: 177 schema = json.load(CommentStripper(schema_f)) 178 validator = jsonschema.Draft4Validator(schema) 179 for error in validator.iter_errors(self.filters_json_data, schema): 180 print("WARNING: ICU data filter JSON file:", error.message, 181 "at", "".join( 182 "[%d]" % part if isinstance(part, int) else ".%s" % part 183 for part in error.absolute_path 184 ), 185 file=sys.stderr) 186 except ImportError: 187 print("Tip: to validate your filter file, install the Pip package 'jsonschema'", file=sys.stderr) 188 pass 189 190 191 def add_copy_input_requests(requests, config, common_vars): 192 files_to_copy = set() 193 for request in requests: 194 request_files = request.all_input_files() 195 # Also add known dependency txt files as possible inputs. 196 # This is required for translit rule files. 197 if hasattr(request, "dep_targets"): 198 request_files += [ 199 f for f in request.dep_targets if isinstance(f, InFile) 200 ] 201 for f in request_files: 202 if isinstance(f, InFile): 203 files_to_copy.add(f) 204 205 result = [] 206 id = 0 207 208 json_data = config.filters_json_data["fileReplacements"] 209 dirname = json_data["directory"] 210 for directive in json_data["replacements"]: 211 if type(directive) == str: 212 input_file = LocalFile(dirname, directive) 213 output_file = InFile(directive) 214 else: 215 input_file = LocalFile(dirname, directive["src"]) 216 output_file = InFile(directive["dest"]) 217 result += [ 218 CopyRequest( 219 name = "input_copy_%d" % id, 220 input_file = input_file, 221 output_file = output_file 222 ) 223 ] 224 files_to_copy.remove(output_file) 225 id += 1 226 227 for f in files_to_copy: 228 result += [ 229 CopyRequest( 230 name = "input_copy_%d" % id, 231 input_file = SrcFile(f.filename), 232 output_file = f 233 ) 234 ] 235 id += 1 236 237 result += requests 238 return result 239 240 241 class IO(object): 242 """I/O operations required when computing the build actions""" 243 244 def __init__(self, src_dir): 245 self.src_dir = src_dir 246 247 def glob(self, pattern): 248 absolute_paths = pyglob.glob(os.path.join(self.src_dir, pattern)) 249 # Strip off the absolute path suffix so we are left with a relative path. 250 relative_paths = [v[len(self.src_dir)+1:] for v in sorted(absolute_paths)] 251 # For the purposes of icutools.databuilder, force Unix-style directory separators. 252 # Within the Python code, including BUILDRULES.py and user-provided config files, 253 # directory separators are normalized to '/', including on Windows platforms. 254 return [v.replace("\\", "/") for v in relative_paths] 255 256 def read_locale_deps(self, tree): 257 return self._read_json("%s/LOCALE_DEPS.json" % tree) 258 259 def _read_json(self, filename): 260 with pyio.open(os.path.join(self.src_dir, filename), "r", encoding="utf-8-sig") as f: 261 return json.load(CommentStripper(f)) 262 263 264 def main(argv): 265 args = flag_parser.parse_args(argv) 266 config = Config(args) 267 268 if args.mode == "gnumake": 269 makefile_vars = { 270 "SRC_DIR": "$(srcdir)", 271 "IN_DIR": "$(srcdir)", 272 "INDEX_NAME": "res_index" 273 } 274 makefile_env = ["ICUDATA_CHAR", "OUT_DIR", "TMP_DIR", "LIBRARY_DATA_DIR"] 275 common = { 276 key: "$(%s)" % key 277 for key in list(makefile_vars.keys()) + makefile_env 278 } 279 common["FILTERS_DIR"] = config.filter_dir 280 common["CWD_DIR"] = os.getcwd() 281 else: 282 makefile_vars = None 283 common = { 284 "SRC_DIR": args.src_dir, 285 "IN_DIR": args.src_dir, 286 "OUT_DIR": args.out_dir, 287 "TMP_DIR": args.tmp_dir, 288 "FILTERS_DIR": config.filter_dir, 289 "CWD_DIR": os.getcwd(), 290 "INDEX_NAME": "res_index", 291 # TODO: Pull this from configure script: 292 "ICUDATA_CHAR": "l", 293 "LIBRARY_DATA_DIR": os.path.join(args.out_dir, "build"), 294 } 295 296 # Automatically load BUILDRULES from the src_dir 297 sys.path.append(args.src_dir) 298 try: 299 import BUILDRULES 300 except ImportError: 301 print("Cannot find BUILDRULES! Did you set your --src_dir?", file=sys.stderr) 302 sys.exit(1) 303 304 io = IO(args.src_dir) 305 requests = BUILDRULES.generate(config, io, common) 306 307 if "fileReplacements" in config.filters_json_data: 308 tmp_in_dir = "{TMP_DIR}/in".format(**common) 309 if makefile_vars: 310 makefile_vars["IN_DIR"] = tmp_in_dir 311 else: 312 common["IN_DIR"] = tmp_in_dir 313 requests = add_copy_input_requests(requests, config, common) 314 315 requests = filtration.apply_filters(requests, config, io) 316 requests = utils.flatten_requests(requests, config, common) 317 318 build_dirs = utils.compute_directories(requests) 319 320 if args.mode == "gnumake": 321 print(makefile.get_gnumake_rules( 322 build_dirs, 323 requests, 324 makefile_vars, 325 common_vars = common 326 )) 327 elif args.mode == "windows-exec": 328 return common_exec.run( 329 platform = "windows", 330 build_dirs = build_dirs, 331 requests = requests, 332 common_vars = common, 333 tool_dir = args.tool_dir, 334 tool_cfg = args.tool_cfg, 335 verbose = args.verbose, 336 ) 337 elif args.mode == "unix-exec": 338 return common_exec.run( 339 platform = "unix", 340 build_dirs = build_dirs, 341 requests = requests, 342 common_vars = common, 343 tool_dir = args.tool_dir, 344 verbose = args.verbose, 345 ) 346 elif args.mode == "bazel-exec": 347 return common_exec.run( 348 platform = "bazel", 349 build_dirs = build_dirs, 350 requests = requests, 351 common_vars = common, 352 tool_dir = args.tool_dir, 353 verbose = args.verbose, 354 ) 355 else: 356 print("Mode not supported: %s" % args.mode) 357 return 1 358 return 0 359 360 if __name__ == "__main__": 361 exit(main(sys.argv[1:]))