manifests.py (16764B)
1 # This Source Code Form is subject to the terms of the Mozilla Public 2 # License, v. 2.0. If a copy of the MPL was not distributed with this 3 # file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 5 import json 6 from contextlib import contextmanager 7 8 import mozpack.path as mozpath 9 10 from .files import ( 11 AbsoluteSymlinkFile, 12 ExistingFile, 13 File, 14 FileFinder, 15 GeneratedFile, 16 HardlinkFile, 17 PreprocessedFile, 18 ) 19 20 21 # This probably belongs in a more generic module. Where? 22 @contextmanager 23 def _auto_fileobj(path, fileobj, mode="r"): 24 if path and fileobj: 25 raise AssertionError("Only 1 of path or fileobj may be defined.") 26 27 if not path and not fileobj: 28 raise AssertionError("Must specified 1 of path or fileobj.") 29 30 if path: 31 fileobj = open(path, mode) 32 33 try: 34 yield fileobj 35 finally: 36 if path: 37 fileobj.close() 38 39 40 class UnreadableInstallManifest(Exception): 41 """Raised when an invalid install manifest is parsed.""" 42 43 44 class InstallManifest: 45 """Describes actions to be used with a copier.FileCopier instance. 46 47 This class facilitates serialization and deserialization of data used to 48 construct a copier.FileCopier and to perform copy operations. 49 50 The manifest defines source paths, destination paths, and a mechanism by 51 which the destination file should come into existence. 52 53 Entries in the manifest correspond to the following types: 54 55 copy -- The file specified as the source path will be copied to the 56 destination path. 57 58 link -- The destination path will be a symlink or hardlink to the source 59 path. If symlinks are not supported, a copy will be performed. 60 61 exists -- The destination path is accounted for and won't be deleted by 62 the FileCopier. If the destination path doesn't exist, an error is 63 raised. 64 65 optional -- The destination path is accounted for and won't be deleted by 66 the FileCopier. No error is raised if the destination path does not 67 exist. 68 69 patternlink -- Paths matched by the expression in the source path 70 will be symlinked or hardlinked to the destination directory. 71 72 patterncopy -- Similar to patternlink except files are copied, not 73 symlinked/hardlinked. 74 75 preprocess -- The file specified at the source path will be run through 76 the preprocessor, and the output will be written to the destination 77 path. 78 79 content -- The destination file will be created with the given content. 80 81 Version 1 of the manifest was the initial version. 82 Version 2 added optional path support 83 Version 3 added support for pattern entries. 84 Version 4 added preprocessed file support. 85 Version 5 added content support. 86 """ 87 88 CURRENT_VERSION = 5 89 90 FIELD_SEPARATOR = "\x1f" 91 92 # Negative values are reserved for non-actionable items, that is, metadata 93 # that doesn't describe files in the destination. 94 LINK = 1 95 COPY = 2 96 REQUIRED_EXISTS = 3 97 OPTIONAL_EXISTS = 4 98 PATTERN_LINK = 5 99 PATTERN_COPY = 6 100 PREPROCESS = 7 101 CONTENT = 8 102 103 def __init__(self, path=None, fileobj=None): 104 """Create a new InstallManifest entry. 105 106 If path is defined, the manifest will be populated with data from the 107 file path. 108 109 If fileobj is defined, the manifest will be populated with data read 110 from the specified file object. 111 112 Both path and fileobj cannot be defined. 113 """ 114 self._dests = {} 115 self._source_files = set() 116 117 if path or fileobj: 118 with _auto_fileobj(path, fileobj, "r") as fh: 119 self._source_files.add(fh.name) 120 self._load_from_fileobj(fh) 121 122 def _load_from_fileobj(self, fileobj): 123 version = fileobj.readline().rstrip() 124 if version not in ("1", "2", "3", "4", "5"): 125 raise UnreadableInstallManifest("Unknown manifest version: %s" % version) 126 127 for line in fileobj: 128 # Explicitly strip on \n so we don't strip out the FIELD_SEPARATOR 129 # as well. 130 line = line.rstrip("\n") 131 132 fields = line.split(self.FIELD_SEPARATOR) 133 134 record_type = int(fields[0]) 135 136 if record_type == self.LINK: 137 dest, source = fields[1:] 138 self.add_link(source, dest) 139 continue 140 141 if record_type == self.COPY: 142 dest, source = fields[1:] 143 self.add_copy(source, dest) 144 continue 145 146 if record_type == self.REQUIRED_EXISTS: 147 _, path = fields 148 self.add_required_exists(path) 149 continue 150 151 if record_type == self.OPTIONAL_EXISTS: 152 _, path = fields 153 self.add_optional_exists(path) 154 continue 155 156 if record_type == self.PATTERN_LINK: 157 _, base, pattern, dest = fields[1:] 158 self.add_pattern_link(base, pattern, dest) 159 continue 160 161 if record_type == self.PATTERN_COPY: 162 _, base, pattern, dest = fields[1:] 163 self.add_pattern_copy(base, pattern, dest) 164 continue 165 166 if record_type == self.PREPROCESS: 167 dest, source, deps, marker, defines, warnings = fields[1:] 168 169 self.add_preprocess( 170 source, 171 dest, 172 deps, 173 marker, 174 self._decode_field_entry(defines), 175 silence_missing_directive_warnings=bool(int(warnings)), 176 ) 177 continue 178 179 if record_type == self.CONTENT: 180 dest, content = fields[1:] 181 182 deserialized_content = self._decode_field_entry(content) 183 if not isinstance(deserialized_content, str): 184 raise TypeError( 185 "not expecting type '%s'" % type(deserialized_content) 186 ) 187 self.add_content(deserialized_content, dest) 188 continue 189 190 # Don't fail for non-actionable items, allowing 191 # forward-compatibility with those we will add in the future. 192 if record_type >= 0: 193 raise UnreadableInstallManifest("Unknown record type: %d" % record_type) 194 195 def __len__(self): 196 return len(self._dests) 197 198 def __contains__(self, item): 199 return item in self._dests 200 201 def __eq__(self, other): 202 return isinstance(other, InstallManifest) and self._dests == other._dests 203 204 def __neq__(self, other): 205 return not self.__eq__(other) 206 207 def __ior__(self, other): 208 if not isinstance(other, InstallManifest): 209 raise ValueError("Can only | with another instance of InstallManifest.") 210 211 self.add_entries_from(other) 212 213 return self 214 215 def _encode_field_entry(self, data): 216 """Converts an object into a format that can be stored in the manifest file. 217 218 Complex data types, such as ``dict``, need to be converted into a text 219 representation before they can be written to a file. 220 """ 221 return json.dumps(data, sort_keys=True) 222 223 def _decode_field_entry(self, data): 224 """Restores an object from a format that can be stored in the manifest file. 225 226 Complex data types, such as ``dict``, need to be converted into a text 227 representation before they can be written to a file. 228 """ 229 return json.loads(data) 230 231 def write(self, path=None, fileobj=None, expand_pattern=False): 232 """Serialize this manifest to a file or file object. 233 234 If path is specified, that file will be written to. If fileobj is specified, 235 the serialized content will be written to that file object. 236 237 It is an error if both are specified. 238 """ 239 with _auto_fileobj(path, fileobj, "wt") as fh: 240 fh.write("%d\n" % self.CURRENT_VERSION) 241 242 for dest in sorted(self._dests): 243 entry = self._dests[dest] 244 245 if expand_pattern and entry[0] in ( 246 self.PATTERN_LINK, 247 self.PATTERN_COPY, 248 ): 249 type, base, pattern, dest = entry 250 type = self.LINK if type == self.PATTERN_LINK else self.COPY 251 finder = FileFinder(base) 252 paths = [f[0] for f in finder.find(pattern)] 253 for file_path in paths: 254 source = mozpath.join(base, file_path) 255 parts = ["%d" % type, mozpath.join(dest, file_path), source] 256 fh.write("%s\n" % self.FIELD_SEPARATOR.join(parts)) 257 else: 258 parts = ["%d" % entry[0], dest] 259 parts.extend(entry[1:]) 260 fh.write("%s\n" % self.FIELD_SEPARATOR.join(parts)) 261 262 def add_link(self, source, dest): 263 """Add a link to this manifest. 264 265 dest will be either a symlink or hardlink to source. 266 """ 267 self._add_entry(dest, (self.LINK, source)) 268 269 def add_copy(self, source, dest): 270 """Add a copy to this manifest. 271 272 source will be copied to dest. 273 """ 274 self._add_entry(dest, (self.COPY, source)) 275 276 def add_required_exists(self, dest): 277 """Record that a destination file must exist. 278 279 This effectively prevents the listed file from being deleted. 280 """ 281 self._add_entry(dest, (self.REQUIRED_EXISTS,)) 282 283 def add_optional_exists(self, dest): 284 """Record that a destination file may exist. 285 286 This effectively prevents the listed file from being deleted. Unlike a 287 "required exists" file, files of this type do not raise errors if the 288 destination file does not exist. 289 """ 290 self._add_entry(dest, (self.OPTIONAL_EXISTS,)) 291 292 def add_pattern_link(self, base, pattern, dest): 293 """Add a pattern match that results in links being created. 294 295 A ``FileFinder`` will be created with its base set to ``base`` 296 and ``FileFinder.find()`` will be called with ``pattern`` to discover 297 source files. Each source file will be either symlinked or hardlinked 298 under ``dest``. 299 300 Filenames under ``dest`` are constructed by taking the path fragment 301 after ``base`` and concatenating it with ``dest``. e.g. 302 303 <base>/foo/bar.h -> <dest>/foo/bar.h 304 """ 305 self._add_entry( 306 mozpath.join(dest, pattern), (self.PATTERN_LINK, base, pattern, dest) 307 ) 308 309 def add_pattern_copy(self, base, pattern, dest): 310 """Add a pattern match that results in copies. 311 312 See ``add_pattern_link()`` for usage. 313 """ 314 self._add_entry( 315 mozpath.join(dest, pattern), (self.PATTERN_COPY, base, pattern, dest) 316 ) 317 318 def add_preprocess( 319 self, 320 source, 321 dest, 322 deps, 323 marker="#", 324 defines={}, 325 silence_missing_directive_warnings=False, 326 ): 327 """Add a preprocessed file to this manifest. 328 329 ``source`` will be passed through preprocessor.py, and the output will be 330 written to ``dest``. 331 """ 332 self._add_entry( 333 dest, 334 ( 335 self.PREPROCESS, 336 source, 337 deps, 338 marker, 339 self._encode_field_entry(defines), 340 "1" if silence_missing_directive_warnings else "0", 341 ), 342 ) 343 344 def add_content(self, content, dest): 345 """Add a file with the given content.""" 346 self._add_entry( 347 dest, 348 ( 349 self.CONTENT, 350 self._encode_field_entry(content), 351 ), 352 ) 353 354 def _add_entry(self, dest, entry): 355 if dest in self._dests: 356 raise ValueError("Item already in manifest: %s" % dest) 357 358 self._dests[dest] = entry 359 360 def add_entries_from(self, other, base=""): 361 """ 362 Copy data from another mozpack.copier.InstallManifest 363 instance, adding an optional base prefix to the destination. 364 365 This allows to merge two manifests into a single manifest, or 366 two take the tagged union of two manifests. 367 """ 368 # We must copy source files to ourselves so extra dependencies from 369 # the preprocessor are taken into account. Ideally, we would track 370 # which source file each entry came from. However, this is more 371 # complicated and not yet implemented. The current implementation 372 # will result in over invalidation, possibly leading to performance 373 # loss. 374 self._source_files |= other._source_files 375 376 for dest in sorted(other._dests): 377 new_dest = mozpath.join(base, dest) if base else dest 378 entry = other._dests[dest] 379 if entry[0] in (self.PATTERN_LINK, self.PATTERN_COPY): 380 entry_type, entry_base, entry_pattern, entry_dest = entry 381 new_entry_dest = mozpath.join(base, entry_dest) if base else entry_dest 382 new_entry = (entry_type, entry_base, entry_pattern, new_entry_dest) 383 else: 384 new_entry = tuple(entry) 385 386 self._add_entry(new_dest, new_entry) 387 388 def populate_registry(self, registry, defines_override={}, link_policy="symlink"): 389 """Populate a mozpack.copier.FileRegistry instance with data from us. 390 391 The caller supplied a FileRegistry instance (or at least something that 392 conforms to its interface) and that instance is populated with data 393 from this manifest. 394 395 Defines can be given to override the ones in the manifest for 396 preprocessing. 397 398 The caller can set a link policy. This determines whether symlinks, 399 hardlinks, or copies are used for LINK and PATTERN_LINK. 400 """ 401 assert link_policy in ("symlink", "hardlink", "copy") 402 for dest in sorted(self._dests): 403 entry = self._dests[dest] 404 install_type = entry[0] 405 406 if install_type == self.LINK: 407 if link_policy == "symlink": 408 cls = AbsoluteSymlinkFile 409 elif link_policy == "hardlink": 410 cls = HardlinkFile 411 else: 412 cls = File 413 registry.add(dest, cls(entry[1])) 414 continue 415 416 if install_type == self.COPY: 417 registry.add(dest, File(entry[1])) 418 continue 419 420 if install_type == self.REQUIRED_EXISTS: 421 registry.add(dest, ExistingFile(required=True)) 422 continue 423 424 if install_type == self.OPTIONAL_EXISTS: 425 registry.add(dest, ExistingFile(required=False)) 426 continue 427 428 if install_type in (self.PATTERN_LINK, self.PATTERN_COPY): 429 _, base, pattern, dest = entry 430 finder = FileFinder(base) 431 paths = [f[0] for f in finder.find(pattern)] 432 433 if install_type == self.PATTERN_LINK: 434 if link_policy == "symlink": 435 cls = AbsoluteSymlinkFile 436 elif link_policy == "hardlink": 437 cls = HardlinkFile 438 else: 439 cls = File 440 else: 441 cls = File 442 443 for path in paths: 444 source = mozpath.join(base, path) 445 registry.add(mozpath.join(dest, path), cls(source)) 446 447 continue 448 449 if install_type == self.PREPROCESS: 450 defines = self._decode_field_entry(entry[4]) 451 if defines_override: 452 defines.update(defines_override) 453 registry.add( 454 dest, 455 PreprocessedFile( 456 entry[1], 457 depfile_path=entry[2], 458 marker=entry[3], 459 defines=defines, 460 extra_depends=self._source_files, 461 silence_missing_directive_warnings=bool(int(entry[5])), 462 ), 463 ) 464 465 continue 466 467 if install_type == self.CONTENT: 468 # GeneratedFile expect the buffer interface, which the unicode 469 # type doesn't have, so encode to a str. 470 content = self._decode_field_entry(entry[1]).encode("utf-8") 471 registry.add(dest, GeneratedFile(content)) 472 continue 473 474 raise Exception( 475 "Unknown install type defined in manifest: %d" % install_type 476 )