formats.py (13531B)
1 # This Source Code Form is subject to the terms of the Mozilla Public 2 # License, v. 2.0. If a copy of the MPL was not distributed with this 3 # file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 5 from urllib.parse import urlparse 6 7 import mozpack.path as mozpath 8 from mozpack.chrome.manifest import ( 9 Manifest, 10 ManifestBinaryComponent, 11 ManifestChrome, 12 ManifestInterfaces, 13 ManifestMultiContent, 14 ManifestResource, 15 ) 16 from mozpack.copier import FileRegistry, FileRegistrySubtree, Jarrer 17 from mozpack.errors import errors 18 from mozpack.files import ManifestFile 19 20 """ 21 Formatters are classes receiving packaging instructions and creating the 22 appropriate package layout. 23 24 There are three distinct formatters, each handling one of the different chrome 25 formats: 26 - flat: essentially, copies files from the source with the same file system 27 layout. Manifests entries are grouped in a single manifest per directory, 28 as well as XPT interfaces. 29 - jar: chrome content is packaged in jar files. 30 - omni: chrome content, modules, non-binary components, and many other 31 elements are packaged in an omnijar file for each base directory. 32 33 The base interface provides the following methods: 34 - add_base(path [, addon]) 35 Register a base directory for an application or GRE, or an addon. 36 Base directories usually contain a root manifest (manifests not 37 included in any other manifest) named chrome.manifest. 38 The optional addon argument tells whether the base directory 39 is that of a packed addon (True), unpacked addon ('unpacked') or 40 otherwise (False). 41 The method may only be called in sorted order of `path` (alphanumeric 42 order, parents before children). 43 - add(path, content) 44 Add the given content (BaseFile instance) at the given virtual path 45 - add_interfaces(path, content) 46 Add the given content (BaseFile instance) as an interface. Equivalent 47 to add(path, content) with the right add_manifest(). 48 - add_manifest(entry) 49 Add a ManifestEntry. 50 - contains(path) 51 Returns whether the given virtual path is known of the formatter. 52 53 The virtual paths mentioned above are paths as they would be with a flat 54 chrome. 55 56 Formatters all take a FileCopier instance they will fill with the packaged 57 data. 58 """ 59 60 61 class PiecemealFormatter: 62 """ 63 Generic formatter that dispatches across different sub-formatters 64 according to paths. 65 """ 66 67 def __init__(self, copier): 68 assert isinstance(copier, (FileRegistry, FileRegistrySubtree)) 69 self.copier = copier 70 self._sub_formatter = {} 71 self._frozen_bases = False 72 73 def add_base(self, base, addon=False): 74 # Only allow to add a base directory before calls to _get_base() 75 assert not self._frozen_bases 76 assert base not in self._sub_formatter 77 assert all(base > b for b in self._sub_formatter) 78 self._add_base(base, addon) 79 80 def _get_base(self, path): 81 """ 82 Return the deepest base directory containing the given path. 83 """ 84 self._frozen_bases = True 85 base = mozpath.basedir(path, self._sub_formatter.keys()) 86 relpath = mozpath.relpath(path, base) if base else path 87 return base, relpath 88 89 def add(self, path, content): 90 base, relpath = self._get_base(path) 91 if base is None: 92 return self.copier.add(relpath, content) 93 return self._sub_formatter[base].add(relpath, content) 94 95 def add_manifest(self, entry): 96 base, relpath = self._get_base(entry.base) 97 assert base is not None 98 return self._sub_formatter[base].add_manifest(entry.move(relpath)) 99 100 def add_interfaces(self, path, content): 101 base, relpath = self._get_base(path) 102 assert base is not None 103 return self._sub_formatter[base].add_interfaces(relpath, content) 104 105 def contains(self, path): 106 assert "*" not in path 107 base, relpath = self._get_base(path) 108 if base is None: 109 return self.copier.contains(relpath) 110 return self._sub_formatter[base].contains(relpath) 111 112 113 class FlatFormatter(PiecemealFormatter): 114 """ 115 Formatter for the flat package format. 116 """ 117 118 def _add_base(self, base, addon=False): 119 self._sub_formatter[base] = FlatSubFormatter( 120 FileRegistrySubtree(base, self.copier) 121 ) 122 123 124 class FlatSubFormatter: 125 """ 126 Sub-formatter for the flat package format. 127 """ 128 129 def __init__(self, copier): 130 assert isinstance(copier, (FileRegistry, FileRegistrySubtree)) 131 self.copier = copier 132 self._chrome_db = {} 133 134 def add(self, path, content): 135 self.copier.add(path, content) 136 137 def add_manifest(self, entry): 138 # Store manifest entries in a single manifest per directory, named 139 # after their parent directory, except for root manifests, all named 140 # chrome.manifest. 141 if entry.base: 142 name = mozpath.basename(entry.base) 143 else: 144 name = "chrome" 145 path = mozpath.normpath(mozpath.join(entry.base, "%s.manifest" % name)) 146 if not self.copier.contains(path): 147 # Add a reference to the manifest file in the parent manifest, if 148 # the manifest file is not a root manifest. 149 if entry.base: 150 parent = mozpath.dirname(entry.base) 151 relbase = mozpath.basename(entry.base) 152 relpath = mozpath.join(relbase, mozpath.basename(path)) 153 self.add_manifest(Manifest(parent, relpath)) 154 self.copier.add(path, ManifestFile(entry.base)) 155 156 if isinstance(entry, ManifestChrome): 157 data = self._chrome_db.setdefault(entry.name, {}) 158 if isinstance(entry, ManifestMultiContent): 159 entries = data.setdefault(entry.type, {}).setdefault(entry.id, []) 160 else: 161 entries = data.setdefault(entry.type, []) 162 for e in entries: 163 # Ideally, we'd actually check whether entry.flags are more 164 # specific than e.flags, but in practice the following test 165 # is enough for now. 166 if entry == e: 167 errors.warn('"%s" is duplicated. Skipping.' % entry) 168 return 169 if not entry.flags or e.flags and entry.flags == e.flags: 170 errors.fatal('"%s" overrides "%s"' % (entry, e)) 171 entries.append(entry) 172 173 self.copier[path].add(entry) 174 175 def add_interfaces(self, path, content): 176 self.copier.add(path, content) 177 self.add_manifest( 178 ManifestInterfaces(mozpath.dirname(path), mozpath.basename(path)) 179 ) 180 181 def contains(self, path): 182 assert "*" not in path 183 return self.copier.contains(path) 184 185 186 class JarFormatter(PiecemealFormatter): 187 """ 188 Formatter for the jar package format. Assumes manifest entries related to 189 chrome are registered before the chrome data files are added. Also assumes 190 manifest entries for resources are registered after chrome manifest 191 entries. 192 """ 193 194 def __init__(self, copier, compress=True): 195 PiecemealFormatter.__init__(self, copier) 196 self._compress = compress 197 198 def _add_base(self, base, addon=False): 199 if addon is True: 200 jarrer = Jarrer(self._compress) 201 self.copier.add(base + ".xpi", jarrer) 202 self._sub_formatter[base] = FlatSubFormatter(jarrer) 203 else: 204 self._sub_formatter[base] = JarSubFormatter( 205 FileRegistrySubtree(base, self.copier), self._compress 206 ) 207 208 209 class JarSubFormatter(PiecemealFormatter): 210 """ 211 Sub-formatter for the jar package format. It is a PiecemealFormatter that 212 dispatches between further sub-formatter for each of the jar files it 213 dispatches the chrome data to, and a FlatSubFormatter for the non-chrome 214 files. 215 """ 216 217 def __init__(self, copier, compress=True): 218 PiecemealFormatter.__init__(self, copier) 219 self._frozen_chrome = False 220 self._compress = compress 221 self._sub_formatter[""] = FlatSubFormatter(copier) 222 223 def _jarize(self, entry, relpath): 224 """ 225 Transform a manifest entry in one pointing to chrome data in a jar. 226 Return the corresponding chrome path and the new entry. 227 """ 228 base = entry.base 229 basepath = mozpath.split(relpath)[0] 230 chromepath = mozpath.join(base, basepath) 231 entry = ( 232 entry.rebase(chromepath) 233 .move(mozpath.join(base, "jar:%s.jar!" % basepath)) 234 .rebase(base) 235 ) 236 return chromepath, entry 237 238 def add_manifest(self, entry): 239 if isinstance(entry, ManifestChrome) and not urlparse(entry.relpath).scheme: 240 chromepath, entry = self._jarize(entry, entry.relpath) 241 assert not self._frozen_chrome 242 if chromepath not in self._sub_formatter: 243 jarrer = Jarrer(self._compress) 244 self.copier.add(chromepath + ".jar", jarrer) 245 self._sub_formatter[chromepath] = FlatSubFormatter(jarrer) 246 elif isinstance(entry, ManifestResource) and not urlparse(entry.target).scheme: 247 chromepath, new_entry = self._jarize(entry, entry.target) 248 if chromepath in self._sub_formatter: 249 entry = new_entry 250 PiecemealFormatter.add_manifest(self, entry) 251 252 253 class OmniJarFormatter(JarFormatter): 254 """ 255 Formatter for the omnijar package format. 256 """ 257 258 def __init__(self, copier, omnijar_name, compress=True, non_resources=()): 259 JarFormatter.__init__(self, copier, compress) 260 self._omnijar_name = omnijar_name 261 self._non_resources = non_resources 262 263 def _add_base(self, base, addon=False): 264 if addon: 265 # Because add_base is always called with parents before children, 266 # all the possible ancestry of `base` is already present in 267 # `_sub_formatter`. 268 parent_base = mozpath.basedir(base, self._sub_formatter.keys()) 269 rel_base = mozpath.relpath(base, parent_base) 270 # If the addon is under a resource directory, package it in the 271 # omnijar. 272 parent_sub_formatter = self._sub_formatter[parent_base] 273 if parent_sub_formatter.is_resource(rel_base): 274 omnijar_sub_formatter = parent_sub_formatter._sub_formatter[ 275 self._omnijar_name 276 ] 277 self._sub_formatter[base] = FlatSubFormatter( 278 FileRegistrySubtree(rel_base, omnijar_sub_formatter.copier) 279 ) 280 return 281 JarFormatter._add_base(self, base, addon) 282 else: 283 self._sub_formatter[base] = OmniJarSubFormatter( 284 FileRegistrySubtree(base, self.copier), 285 self._omnijar_name, 286 self._compress, 287 self._non_resources, 288 ) 289 290 291 class OmniJarSubFormatter(PiecemealFormatter): 292 """ 293 Sub-formatter for the omnijar package format. It is a PiecemealFormatter 294 that dispatches between a FlatSubFormatter for the resources data and 295 another FlatSubFormatter for the other files. 296 """ 297 298 def __init__(self, copier, omnijar_name, compress=True, non_resources=()): 299 PiecemealFormatter.__init__(self, copier) 300 self._omnijar_name = omnijar_name 301 self._compress = compress 302 self._non_resources = non_resources 303 self._sub_formatter[""] = FlatSubFormatter(copier) 304 jarrer = Jarrer(self._compress) 305 self._sub_formatter[omnijar_name] = FlatSubFormatter(jarrer) 306 307 def _get_base(self, path): 308 base = self._omnijar_name if self.is_resource(path) else "" 309 # Only add the omnijar file if something ends up in it. 310 if base and not self.copier.contains(base): 311 self.copier.add(base, self._sub_formatter[base].copier) 312 return base, path 313 314 def add_manifest(self, entry): 315 base = "" 316 if not isinstance(entry, ManifestBinaryComponent): 317 base = self._omnijar_name 318 formatter = self._sub_formatter[base] 319 return formatter.add_manifest(entry) 320 321 def is_resource(self, path): 322 """ 323 Return whether the given path corresponds to a resource to be put in an 324 omnijar archive. 325 """ 326 if any(mozpath.match(path, p.replace("*", "**")) for p in self._non_resources): 327 return False 328 path = mozpath.split(path) 329 if path[0] == "chrome": 330 return len(path) == 1 or path[1] != "icons" 331 if path[0] == "components": 332 return path[-1].endswith((".js", ".xpt")) 333 if path[0] == "res": 334 return len(path) == 1 or ( 335 path[1] != "cursors" 336 and path[1] != "touchbar" 337 and path[1] != "MainMenu.nib" 338 ) 339 if path[0] == "defaults": 340 return len(path) != 3 or not ( 341 path[2] == "channel-prefs.js" and path[1] in ["pref", "preferences"] 342 ) 343 if len(path) <= 2 and path[-1] == "greprefs.js": 344 # Accommodate `greprefs.js` and `$ANDROID_CPU_ARCH/greprefs.js`. 345 return True 346 return path[0] in [ 347 "modules", 348 "moz-src", 349 "actors", 350 "dictionaries", 351 "hyphenation", 352 "localization", 353 "default.locale", 354 "contentaccessible", 355 ]