__init__.py (15219B)
1 # This Source Code Form is subject to the terms of the Mozilla Public 2 # License, v. 2.0. If a copy of the MPL was not distributed with this 3 # file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 5 import codecs 6 import json 7 import os 8 import re 9 from collections import deque 10 11 import mozpack.path as mozpath 12 from mozbuild.preprocessor import Preprocessor 13 from mozpack.chrome.manifest import ( 14 Manifest, 15 ManifestBinaryComponent, 16 ManifestChrome, 17 ManifestInterfaces, 18 is_manifest, 19 parse_manifest, 20 ) 21 from mozpack.errors import errors 22 23 24 class Component: 25 """ 26 Class that represents a component in a package manifest. 27 """ 28 29 def __init__(self, name, destdir=""): 30 if name.find(" ") > 0: 31 errors.fatal('Malformed manifest: space in component name "%s"' % name) 32 self._name = name 33 self._destdir = destdir 34 35 def __repr__(self): 36 s = self.name 37 if self.destdir: 38 s += ' destdir="%s"' % self.destdir 39 return s 40 41 @property 42 def name(self): 43 return self._name 44 45 @property 46 def destdir(self): 47 return self._destdir 48 49 @staticmethod 50 def _triples(lst): 51 """ 52 Split [1, 2, 3, 4, 5, 6, 7] into [(1, 2, 3), (4, 5, 6)]. 53 """ 54 return zip(*[iter(lst)] * 3) 55 56 KEY_VALUE_RE = re.compile( 57 r""" 58 \s* # optional whitespace. 59 ([a-zA-Z0-9_]+) # key. 60 \s*=\s* # optional space around =. 61 "([^"]*)" # value without surrounding quotes. 62 (?:\s+|$) 63 """, 64 re.VERBOSE, 65 ) 66 67 @staticmethod 68 def _split_options(string): 69 """ 70 Split 'key1="value1" key2="value2"' into 71 {'key1':'value1', 'key2':'value2'}. 72 73 Returned keys and values are all strings. 74 75 Throws ValueError if the input is malformed. 76 """ 77 options = {} 78 splits = Component.KEY_VALUE_RE.split(string) 79 if len(splits) % 3 != 1: 80 # This should never happen -- we expect to always split 81 # into ['', ('key', 'val', '')*]. 82 raise ValueError("Bad input") 83 if splits[0]: 84 raise ValueError("Unrecognized input " + splits[0]) 85 for key, val, no_match in Component._triples(splits[1:]): 86 if no_match: 87 raise ValueError("Unrecognized input " + no_match) 88 options[key] = val 89 return options 90 91 @staticmethod 92 def _split_component_and_options(string): 93 """ 94 Split 'name key1="value1" key2="value2"' into 95 ('name', {'key1':'value1', 'key2':'value2'}). 96 97 Returned name, keys and values are all strings. 98 99 Raises ValueError if the input is malformed. 100 """ 101 splits = string.strip().split(None, 1) 102 if not splits: 103 raise ValueError("No component found") 104 component = splits[0].strip() 105 if not component: 106 raise ValueError("No component found") 107 if not re.match("[a-zA-Z0-9_-]+$", component): 108 raise ValueError("Bad component name " + component) 109 options = Component._split_options(splits[1]) if len(splits) > 1 else {} 110 return component, options 111 112 @staticmethod 113 def from_string(string): 114 """ 115 Create a component from a string. 116 """ 117 try: 118 name, options = Component._split_component_and_options(string) 119 except ValueError as e: 120 errors.fatal("Malformed manifest: %s" % e) 121 return 122 destdir = options.pop("destdir", "") 123 if options: 124 errors.fatal( 125 "Malformed manifest: options %s not recognized" % options.keys() 126 ) 127 return Component(name, destdir=destdir) 128 129 130 class PackageManifestParser: 131 """ 132 Class for parsing of a package manifest, after preprocessing. 133 134 A package manifest is a list of file paths, with some syntaxic sugar: 135 [] designates a toplevel component. Example: [xpcom] 136 - in front of a file specifies it to be removed 137 * wildcard support 138 ** expands to all files and zero or more directories 139 ; file comment 140 141 The parser takes input from the preprocessor line by line, and pushes 142 parsed information to a sink object. 143 144 The add and remove methods of the sink object are called with the 145 current Component instance and a path. 146 """ 147 148 def __init__(self, sink): 149 """ 150 Initialize the package manifest parser with the given sink. 151 """ 152 self._component = Component("") 153 self._sink = sink 154 155 def handle_line(self, str): 156 """ 157 Handle a line of input and push the parsed information to the sink 158 object. 159 """ 160 # Remove comments. 161 str = str.strip() 162 if not str or str.startswith(";"): 163 return 164 if str.startswith("[") and str.endswith("]"): 165 self._component = Component.from_string(str[1:-1]) 166 elif str.startswith("-"): 167 str = str[1:] 168 self._sink.remove(self._component, str) 169 elif "," in str: 170 errors.fatal("Incompatible syntax") 171 else: 172 self._sink.add(self._component, str) 173 174 175 class PreprocessorOutputWrapper: 176 """ 177 File-like helper to handle the preprocessor output and send it to a parser. 178 The parser's handle_line method is called in the relevant errors.context. 179 """ 180 181 def __init__(self, preprocessor, parser): 182 self._parser = parser 183 self._pp = preprocessor 184 185 def write(self, str): 186 with errors.context(self._pp.context["FILE"], self._pp.context["LINE"]): 187 self._parser.handle_line(str) 188 189 190 def preprocess(input, parser, defines={}): 191 """ 192 Preprocess the file-like input with the given defines, and send the 193 preprocessed output line by line to the given parser. 194 """ 195 pp = Preprocessor() 196 pp.context.update(defines) 197 pp.do_filter("substitution") 198 pp.out = PreprocessorOutputWrapper(pp, parser) 199 pp.do_include(input) 200 201 202 def preprocess_manifest(sink, manifest, defines={}): 203 """ 204 Preprocess the given file-like manifest with the given defines, and push 205 the parsed information to a sink. See PackageManifestParser documentation 206 for more details on the sink. 207 """ 208 preprocess(manifest, PackageManifestParser(sink), defines) 209 210 211 class CallDeque(deque): 212 """ 213 Queue of function calls to make. 214 """ 215 216 def append(self, function, *args): 217 deque.append(self, (errors.get_context(), function, args)) 218 219 def execute(self): 220 while True: 221 try: 222 context, function, args = self.popleft() 223 except IndexError: 224 return 225 if context: 226 with errors.context(context[0], context[1]): 227 function(*args) 228 else: 229 function(*args) 230 231 232 class SimplePackager: 233 """ 234 Helper used to translate and buffer instructions from the 235 SimpleManifestSink to a formatter. Formatters expect some information to be 236 given first that the simple manifest contents can't guarantee before the 237 end of the input. 238 """ 239 240 def __init__(self, formatter): 241 self.formatter = formatter 242 # Queue for formatter.add_interfaces()/add_manifest() calls. 243 self._queue = CallDeque() 244 # Queue for formatter.add_manifest() calls for ManifestChrome. 245 self._chrome_queue = CallDeque() 246 # Queue for formatter.add() calls. 247 self._file_queue = CallDeque() 248 # All paths containing addons. (key is path, value is whether it 249 # should be packed or unpacked) 250 self._addons = {} 251 # All manifest paths imported. 252 self._manifests = set() 253 # All manifest paths included from some other manifest. 254 self._included_manifests = {} 255 self._closed = False 256 257 # Parsing RDF is complex, and would require an external library to do 258 # properly. Just go with some hackish but probably sufficient regexp 259 UNPACK_ADDON_RE = re.compile( 260 r"""(?: 261 <em:unpack>true</em:unpack> 262 |em:unpack=(?P<quote>["']?)true(?P=quote) 263 )""", 264 re.VERBOSE, 265 ) 266 267 def add(self, path, file): 268 """ 269 Add the given BaseFile instance with the given path. 270 """ 271 assert not self._closed 272 if is_manifest(path): 273 self._add_manifest_file(path, file) 274 elif path.endswith(".xpt"): 275 self._queue.append(self.formatter.add_interfaces, path, file) 276 else: 277 self._file_queue.append(self.formatter.add, path, file) 278 if mozpath.basename(path) == "install.rdf": 279 addon = True 280 install_rdf = file.open().read().decode() 281 if self.UNPACK_ADDON_RE.search(install_rdf): 282 addon = "unpacked" 283 self._add_addon(mozpath.dirname(path), addon) 284 elif mozpath.basename(path) == "manifest.json": 285 manifest = file.open().read() 286 try: 287 parsed = json.loads(manifest) 288 except ValueError: 289 pass 290 if isinstance(parsed, dict) and "manifest_version" in parsed: 291 self._add_addon(mozpath.dirname(path), True) 292 293 def _add_addon(self, path, addon_type): 294 """ 295 Add the given BaseFile to the collection of addons if a parent 296 directory is not already in the collection. 297 """ 298 if mozpath.basedir(path, self._addons) is not None: 299 return 300 301 for dir in self._addons: 302 if mozpath.basedir(dir, [path]) is not None: 303 del self._addons[dir] 304 break 305 306 self._addons[path] = addon_type 307 308 def _add_manifest_file(self, path, file): 309 """ 310 Add the given BaseFile with manifest file contents with the given path. 311 """ 312 self._manifests.add(path) 313 base = "" 314 if hasattr(file, "path"): 315 # Find the directory the given path is relative to. 316 b = mozpath.normsep(file.path) 317 if b.endswith("/" + path) or b == path: 318 base = os.path.normpath(b[: -len(path)]) 319 for e in parse_manifest(base, path, codecs.getreader("utf-8")(file.open())): 320 # ManifestResources need to be given after ManifestChrome, so just 321 # put all ManifestChrome in a separate queue to make them first. 322 if isinstance(e, ManifestChrome): 323 # e.move(e.base) just returns a clone of the entry. 324 self._chrome_queue.append(self.formatter.add_manifest, e.move(e.base)) 325 elif not isinstance(e, (Manifest, ManifestInterfaces)): 326 self._queue.append(self.formatter.add_manifest, e.move(e.base)) 327 # If a binary component is added to an addon, prevent the addon 328 # from being packed. 329 if isinstance(e, ManifestBinaryComponent): 330 addon = mozpath.basedir(e.base, self._addons) 331 if addon: 332 self._addons[addon] = "unpacked" 333 if isinstance(e, Manifest): 334 if e.flags: 335 errors.fatal("Flags are not supported on " + '"manifest" entries') 336 self._included_manifests[e.path] = path 337 338 def get_bases(self, addons=True): 339 """ 340 Return all paths under which root manifests have been found. Root 341 manifests are manifests that are included in no other manifest. 342 `addons` indicates whether to include addon bases as well. 343 """ 344 all_bases = set( 345 mozpath.dirname(m) for m in self._manifests - set(self._included_manifests) 346 ) 347 if not addons: 348 all_bases -= set(self._addons) 349 else: 350 # If for some reason some detected addon doesn't have a 351 # non-included manifest. 352 all_bases |= set(self._addons) 353 return all_bases 354 355 def close(self): 356 """ 357 Push all instructions to the formatter. 358 """ 359 self._closed = True 360 361 bases = self.get_bases() 362 broken_bases = sorted( 363 m 364 for m, includer in self._included_manifests.items() 365 if mozpath.basedir(m, bases) != mozpath.basedir(includer, bases) 366 ) 367 for m in broken_bases: 368 errors.fatal( 369 '"%s" is included from "%s", which is outside "%s"' 370 % (m, self._included_manifests[m], mozpath.basedir(m, bases)) 371 ) 372 for base in sorted(bases): 373 self.formatter.add_base(base, self._addons.get(base, False)) 374 self._chrome_queue.execute() 375 self._queue.execute() 376 self._file_queue.execute() 377 378 379 class SimpleManifestSink: 380 """ 381 Parser sink for "simple" package manifests. Simple package manifests use 382 the format described in the PackageManifestParser documentation, but don't 383 support file removals, and require manifests, interfaces and chrome data to 384 be explicitely listed. 385 Entries starting with bin/ are searched under bin/ in the FileFinder, but 386 are packaged without the bin/ prefix. 387 """ 388 389 def __init__(self, finder, formatter): 390 """ 391 Initialize the SimpleManifestSink. The given FileFinder is used to 392 get files matching the patterns given in the manifest. The given 393 formatter does the packaging job. 394 """ 395 self._finder = finder 396 self.packager = SimplePackager(formatter) 397 self._closed = False 398 self._manifests = set() 399 400 @staticmethod 401 def normalize_path(path): 402 """ 403 Remove any bin/ prefix. 404 """ 405 if mozpath.basedir(path, ["bin"]) == "bin": 406 return mozpath.relpath(path, "bin") 407 return path 408 409 def add(self, component, pattern): 410 """ 411 Add files with the given pattern in the given component. 412 """ 413 assert not self._closed 414 added = False 415 for p, f in self._finder.find(pattern): 416 added = True 417 if is_manifest(p): 418 self._manifests.add(p) 419 dest = mozpath.join(component.destdir, SimpleManifestSink.normalize_path(p)) 420 self.packager.add(dest, f) 421 if not added: 422 errors.error("Missing file(s): %s" % pattern) 423 424 def remove(self, component, pattern): 425 """ 426 Remove files with the given pattern in the given component. 427 """ 428 assert not self._closed 429 errors.fatal("Removal is unsupported") 430 431 def close(self, auto_root_manifest=True): 432 """ 433 Add possibly missing bits and push all instructions to the formatter. 434 """ 435 if auto_root_manifest: 436 # Simple package manifests don't contain the root manifests, so 437 # find and add them. 438 paths = [mozpath.dirname(m) for m in self._manifests] 439 path = mozpath.dirname(mozpath.commonprefix(paths)) 440 for p, f in self._finder.find(mozpath.join(path, "chrome.manifest")): 441 if p not in self._manifests: 442 self.packager.add(SimpleManifestSink.normalize_path(p), f) 443 self.packager.close()