l10n.py (11852B)
1 # This Source Code Form is subject to the terms of the Mozilla Public 2 # License, v. 2.0. If a copy of the MPL was not distributed with this 3 # file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 5 """ 6 Replace localized parts of a packaged directory with data from a langpack 7 directory. 8 """ 9 10 import json 11 import os 12 13 from createprecomplete import generate_precomplete 14 15 import mozpack.path as mozpath 16 from mozpack.chrome.manifest import ( 17 Manifest, 18 ManifestChrome, 19 ManifestEntryWithRelPath, 20 ManifestLocale, 21 is_manifest, 22 ) 23 from mozpack.copier import FileCopier, Jarrer 24 from mozpack.errors import errors 25 from mozpack.files import ComposedFinder, GeneratedFile, ManifestFile 26 from mozpack.mozjar import JAR_DEFLATED 27 from mozpack.packager import Component, SimpleManifestSink, SimplePackager 28 from mozpack.packager.formats import FlatFormatter, JarFormatter, OmniJarFormatter 29 from mozpack.packager.unpack import UnpackFinder 30 31 32 class LocaleManifestFinder: 33 def __init__(self, finder): 34 entries = self.entries = [] 35 bases = self.bases = [] 36 37 class MockFormatter: 38 def add_interfaces(self, path, content): 39 pass 40 41 def add(self, path, content): 42 pass 43 44 def add_manifest(self, entry): 45 if entry.localized: 46 entries.append(entry) 47 48 def add_base(self, base, addon=False): 49 bases.append(base) 50 51 # SimplePackager rejects "manifest foo.manifest" entries with 52 # additional flags (such as "manifest foo.manifest application=bar"). 53 # Those type of entries are used by language packs to work as addons, 54 # but are not necessary for the purpose of l10n repacking. So we wrap 55 # the finder in order to remove those entries. 56 class WrapFinder: 57 def __init__(self, finder): 58 self._finder = finder 59 60 def find(self, pattern): 61 for p, f in self._finder.find(pattern): 62 if isinstance(f, ManifestFile): 63 unwanted = [ 64 e for e in f._entries if isinstance(e, Manifest) and e.flags 65 ] 66 if unwanted: 67 f = ManifestFile( 68 f._base, [e for e in f._entries if e not in unwanted] 69 ) 70 yield p, f 71 72 sink = SimpleManifestSink(WrapFinder(finder), MockFormatter()) 73 sink.add(Component(""), "*") 74 sink.close(False) 75 76 # Find unique locales used in these manifest entries. 77 self.locales = list( 78 set(e.id for e in self.entries if isinstance(e, ManifestLocale)) 79 ) 80 81 82 class L10NRepackFormatterMixin: 83 def __init__(self, *args, **kwargs): 84 super().__init__(*args, **kwargs) 85 self._dictionaries = {} 86 87 def add(self, path, file): 88 base, relpath = self._get_base(path) 89 if path.endswith(".dic"): 90 if relpath.startswith("dictionaries/"): 91 root, ext = mozpath.splitext(mozpath.basename(path)) 92 self._dictionaries[root] = path 93 elif path.endswith("/built_in_addons.json"): 94 data = json.loads(file.open().read()) 95 data["dictionaries"] = self._dictionaries 96 # The GeneratedFile content is only really generated after 97 # all calls to formatter.add. 98 file = GeneratedFile(lambda: json.dumps(data)) 99 elif relpath.startswith("META-INF/"): 100 # Ignore signatures inside omnijars. We drop these items: if we 101 # don't treat them as omnijar resources, they will be included in 102 # the top-level package, and that's not how omnijars are signed (Bug 103 # 1750676). If we treat them as omnijar resources, they will stay 104 # in the omnijar, as expected -- but the signatures won't be valid 105 # after repacking. Therefore, drop them. 106 return 107 super().add(path, file) 108 109 110 def L10NRepackFormatter(klass): 111 class L10NRepackFormatter(L10NRepackFormatterMixin, klass): 112 pass 113 114 return L10NRepackFormatter 115 116 117 FlatFormatter = L10NRepackFormatter(FlatFormatter) 118 JarFormatter = L10NRepackFormatter(JarFormatter) 119 OmniJarFormatter = L10NRepackFormatter(OmniJarFormatter) 120 121 122 def _repack(app_finder, l10n_finder, copier, formatter, non_chrome=set()): 123 app = LocaleManifestFinder(app_finder) 124 l10n = LocaleManifestFinder(l10n_finder) 125 126 # The code further below assumes there's only one locale replaced with 127 # another one. 128 if len(app.locales) > 1: 129 errors.fatal("Multiple app locales aren't supported: " + ",".join(app.locales)) 130 if len(l10n.locales) > 1: 131 errors.fatal( 132 "Multiple l10n locales aren't supported: " + ",".join(l10n.locales) 133 ) 134 locale = app.locales[0] 135 l10n_locale = l10n.locales[0] 136 137 # For each base directory, store what path a locale chrome package name 138 # corresponds to. 139 # e.g., for the following entry under app/chrome: 140 # locale foo en-US path/to/files 141 # keep track that the locale path for foo in app is 142 # app/chrome/path/to/files. 143 # As there may be multiple locale entries with the same base, but with 144 # different flags, that tracking takes the flags into account when there 145 # are some. Example: 146 # locale foo en-US path/to/files/win os=Win 147 # locale foo en-US path/to/files/mac os=Darwin 148 def key(entry): 149 if entry.flags: 150 return "%s %s" % (entry.name, entry.flags) 151 return entry.name 152 153 l10n_paths = {} 154 for e in l10n.entries: 155 if isinstance(e, ManifestChrome): 156 base = mozpath.basedir(e.path, app.bases) 157 l10n_paths.setdefault(base, {}) 158 l10n_paths[base][key(e)] = e.path 159 160 # For chrome and non chrome files or directories, store what langpack path 161 # corresponds to a package path. 162 paths = {} 163 for e in app.entries: 164 if isinstance(e, ManifestEntryWithRelPath): 165 base = mozpath.basedir(e.path, app.bases) 166 if base not in l10n_paths: 167 errors.fatal("Locale doesn't contain %s/" % base) 168 # Allow errors to accumulate 169 continue 170 if key(e) not in l10n_paths[base]: 171 errors.fatal("Locale doesn't have a manifest entry for '%s'" % e.name) 172 # Allow errors to accumulate 173 continue 174 paths[e.path] = l10n_paths[base][key(e)] 175 176 for pattern in non_chrome: 177 for base in app.bases: 178 path = mozpath.join(base, pattern) 179 left = set(p for p, f in app_finder.find(path)) 180 right = set(p for p, f in l10n_finder.find(path)) 181 for p in right: 182 paths[p] = p 183 for p in left - right: 184 paths[p] = None 185 186 # Create a new package, with non localized bits coming from the original 187 # package, and localized bits coming from the langpack. 188 packager = SimplePackager(formatter) 189 for p, f in app_finder: 190 if is_manifest(p): 191 # Remove localized manifest entries. 192 for e in [e for e in f if e.localized]: 193 f.remove(e) 194 # If the path is one that needs a locale replacement, use the 195 # corresponding file from the langpack. 196 path = None 197 if p in paths: 198 path = paths[p] 199 if not path: 200 continue 201 else: 202 base = mozpath.basedir(p, paths.keys()) 203 if base: 204 subpath = mozpath.relpath(p, base) 205 path = mozpath.normpath(mozpath.join(paths[base], subpath)) 206 207 if path: 208 files = [f for p, f in l10n_finder.find(path)] 209 if not files: 210 if base not in non_chrome: 211 finderBase = "" 212 if hasattr(l10n_finder, "base"): 213 finderBase = l10n_finder.base 214 errors.error("Missing file: %s" % os.path.join(finderBase, path)) 215 else: 216 packager.add(path, files[0]) 217 else: 218 packager.add(p, f) 219 220 # Add localized manifest entries from the langpack. 221 l10n_manifests = [] 222 for base in set(e.base for e in l10n.entries): 223 m = ManifestFile(base, [e for e in l10n.entries if e.base == base]) 224 path = mozpath.join(base, "chrome.%s.manifest" % l10n_locale) 225 l10n_manifests.append((path, m)) 226 bases = packager.get_bases() 227 for path, m in l10n_manifests: 228 base = mozpath.basedir(path, bases) 229 packager.add(path, m) 230 # Add a "manifest $path" entry in the top manifest under that base. 231 m = ManifestFile(base) 232 m.add(Manifest(base, mozpath.relpath(path, base))) 233 packager.add(mozpath.join(base, "chrome.manifest"), m) 234 235 packager.close() 236 237 # Add any remaining non chrome files. 238 for pattern in non_chrome: 239 for base in bases: 240 for p, f in l10n_finder.find(mozpath.join(base, pattern)): 241 if not formatter.contains(p): 242 formatter.add(p, f) 243 244 # Resources in `localization` directories are packaged from the source and then 245 # if localized versions are present in the l10n dir, we package them as well 246 # keeping the source dir resources as a runtime fallback. 247 for p, f in l10n_finder.find("**/localization"): 248 if not formatter.contains(p): 249 formatter.add(p, f) 250 251 # Transplant jar preloading information. 252 for path, log in app_finder.jarlogs.items(): 253 assert isinstance(copier[path], Jarrer) 254 copier[path].preload([l.replace(locale, l10n_locale) for l in log]) 255 256 257 def repack( 258 source, l10n, extra_l10n={}, non_resources=[], non_chrome=set(), minify=False 259 ): 260 """ 261 Replace localized data from the `source` directory with localized data 262 from `l10n` and `extra_l10n`. 263 264 The `source` argument points to a directory containing a packaged 265 application (in omnijar, jar or flat form). 266 The `l10n` argument points to a directory containing the main localized 267 data (usually in the form of a language pack addon) to use to replace 268 in the packaged application. 269 The `extra_l10n` argument contains a dict associating relative paths in 270 the source to separate directories containing localized data for them. 271 This can be used to point at different language pack addons for different 272 parts of the package application. 273 The `non_resources` argument gives a list of relative paths in the source 274 that should not be added in an omnijar in case the packaged application 275 is in that format. 276 The `non_chrome` argument gives a list of file/directory patterns for 277 localized files that are not listed in a chrome.manifest. 278 If `minify`, `.properties` files are minified. 279 """ 280 app_finder = UnpackFinder(source, minify=minify) 281 l10n_finder = UnpackFinder(l10n, minify=minify) 282 if extra_l10n: 283 finders = { 284 "": l10n_finder, 285 } 286 for base, path in extra_l10n.items(): 287 finders[base] = UnpackFinder(path, minify=minify) 288 l10n_finder = ComposedFinder(finders) 289 copier = FileCopier() 290 compress = min(app_finder.compressed, JAR_DEFLATED) 291 if app_finder.kind == "flat": 292 formatter = FlatFormatter(copier) 293 elif app_finder.kind == "jar": 294 formatter = JarFormatter(copier, compress=compress) 295 elif app_finder.kind == "omni": 296 formatter = OmniJarFormatter( 297 copier, app_finder.omnijar, compress=compress, non_resources=non_resources 298 ) 299 300 with errors.accumulate(): 301 _repack(app_finder, l10n_finder, copier, formatter, non_chrome) 302 copier.copy(source, skip_if_older=False) 303 generate_precomplete(source)