protoresources.py (10689B)
1 # Copyright 2020 The Chromium Authors 2 # Use of this source code is governed by a BSD-style license that can be 3 # found in the LICENSE file. 4 """Functions that modify resources in protobuf format. 5 6 Format reference: 7 https://cs.android.com/search?q=f:aapt2.*Resources.proto 8 """ 9 10 import logging 11 import os 12 import struct 13 import sys 14 import zipfile 15 16 from util import build_utils 17 from util import resource_utils 18 19 sys.path[1:1] = [ 20 # `Resources_pb2` module imports `descriptor`, which imports `six`. 21 os.path.join(build_utils.DIR_SOURCE_ROOT, 'third_party', 'six', 'src'), 22 # Make sure the pb2 files are able to import google.protobuf 23 os.path.join(build_utils.DIR_SOURCE_ROOT, 'third_party', 'protobuf', 24 'python'), 25 ] 26 27 from proto import Resources_pb2 28 29 # First bytes in an .flat.arsc file. 30 # uint32: Magic ("ARSC"), version (1), num_entries (1), type (0) 31 _FLAT_ARSC_HEADER = b'AAPT\x01\x00\x00\x00\x01\x00\x00\x00\x00\x00\x00\x00' 32 33 # The package ID hardcoded for shared libraries. See 34 # _HardcodeSharedLibraryDynamicAttributes() for more details. If this value 35 # changes make sure to change REQUIRED_PACKAGE_IDENTIFIER in WebLayerImpl.java. 36 SHARED_LIBRARY_HARDCODED_ID = 36 37 38 39 def _ProcessZip(zip_path, process_func): 40 """Filters a .zip file via: new_bytes = process_func(filename, data).""" 41 has_changes = False 42 zip_entries = [] 43 with zipfile.ZipFile(zip_path) as src_zip: 44 for info in src_zip.infolist(): 45 data = src_zip.read(info) 46 new_data = process_func(info.filename, data) 47 if new_data is not data: 48 has_changes = True 49 data = new_data 50 zip_entries.append((info, data)) 51 52 # Overwrite the original zip file. 53 if has_changes: 54 with zipfile.ZipFile(zip_path, 'w') as f: 55 for info, data in zip_entries: 56 f.writestr(info, data) 57 58 59 def _ProcessProtoItem(item): 60 if not item.HasField('ref'): 61 return 62 63 # If this is a dynamic attribute (type ATTRIBUTE, package ID 0), hardcode 64 # the package to SHARED_LIBRARY_HARDCODED_ID. 65 if item.ref.type == Resources_pb2.Reference.ATTRIBUTE and not (item.ref.id 66 & 0xff000000): 67 item.ref.id |= (0x01000000 * SHARED_LIBRARY_HARDCODED_ID) 68 item.ref.ClearField('is_dynamic') 69 70 71 def _ProcessProtoValue(value): 72 if value.HasField('item'): 73 _ProcessProtoItem(value.item) 74 return 75 76 compound_value = value.compound_value 77 if compound_value.HasField('style'): 78 for entry in compound_value.style.entry: 79 _ProcessProtoItem(entry.item) 80 elif compound_value.HasField('array'): 81 for element in compound_value.array.element: 82 _ProcessProtoItem(element.item) 83 elif compound_value.HasField('plural'): 84 for entry in compound_value.plural.entry: 85 _ProcessProtoItem(entry.item) 86 87 88 def _ProcessProtoXmlNode(xml_node): 89 if not xml_node.HasField('element'): 90 return 91 92 for attribute in xml_node.element.attribute: 93 _ProcessProtoItem(attribute.compiled_item) 94 95 for child in xml_node.element.child: 96 _ProcessProtoXmlNode(child) 97 98 99 def _SplitLocaleResourceType(_type, allowed_resource_names): 100 """Splits locale specific resources out of |_type| and returns them. 101 102 Any locale specific resources will be removed from |_type|, and a new 103 Resources_pb2.Type value will be returned which contains those resources. 104 105 Args: 106 _type: A Resources_pb2.Type value 107 allowed_resource_names: Names of locale resources that should be kept in the 108 main type. 109 """ 110 locale_entries = [] 111 for entry in _type.entry: 112 if entry.name in allowed_resource_names: 113 continue 114 115 # First collect all resources values with a locale set. 116 config_values_with_locale = [] 117 for config_value in entry.config_value: 118 if config_value.config.locale: 119 config_values_with_locale.append(config_value) 120 121 if config_values_with_locale: 122 # Remove the locale resources from the original entry 123 for value in config_values_with_locale: 124 entry.config_value.remove(value) 125 126 # Add locale resources to a new Entry, and save for later. 127 locale_entry = Resources_pb2.Entry() 128 locale_entry.CopyFrom(entry) 129 del locale_entry.config_value[:] 130 locale_entry.config_value.extend(config_values_with_locale) 131 locale_entries.append(locale_entry) 132 133 if not locale_entries: 134 return None 135 136 # Copy the original type and replace the entries with |locale_entries|. 137 locale_type = Resources_pb2.Type() 138 locale_type.CopyFrom(_type) 139 del locale_type.entry[:] 140 locale_type.entry.extend(locale_entries) 141 return locale_type 142 143 144 def _HardcodeInTable(table, is_bundle_module, shared_resources_allowlist): 145 translations_package = None 146 if is_bundle_module: 147 # A separate top level package will be added to the resources, which 148 # contains only locale specific resources. The package ID of the locale 149 # resources is hardcoded to SHARED_LIBRARY_HARDCODED_ID. This causes 150 # resources in locale splits to all get assigned 151 # SHARED_LIBRARY_HARDCODED_ID as their package ID, which prevents a bug 152 # in shared library bundles where each split APK gets a separate dynamic 153 # ID, and cannot be accessed by the main APK. 154 translations_package = Resources_pb2.Package() 155 translations_package.package_id.id = SHARED_LIBRARY_HARDCODED_ID 156 translations_package.package_name = (table.package[0].package_name + 157 '_translations') 158 159 # These resources are allowed in the base resources, since they are needed 160 # by WebView. 161 allowed_resource_names = set() 162 if shared_resources_allowlist: 163 allowed_resource_names = set( 164 resource_utils.GetRTxtStringResourceNames(shared_resources_allowlist)) 165 166 for package in table.package: 167 for _type in package.type: 168 for entry in _type.entry: 169 for config_value in entry.config_value: 170 _ProcessProtoValue(config_value.value) 171 172 if translations_package is not None: 173 locale_type = _SplitLocaleResourceType(_type, allowed_resource_names) 174 if locale_type: 175 translations_package.type.add().CopyFrom(locale_type) 176 177 if translations_package is not None: 178 table.package.add().CopyFrom(translations_package) 179 180 181 def HardcodeSharedLibraryDynamicAttributes(zip_path, 182 is_bundle_module, 183 shared_resources_allowlist=None): 184 """Hardcodes the package IDs of dynamic attributes and locale resources. 185 186 Hardcoding dynamic attribute package IDs is a workaround for b/147674078, 187 which affects Android versions pre-N. Hardcoding locale resource package IDs 188 is a workaround for b/155437035, which affects resources built with 189 --shared-lib on all Android versions 190 191 Args: 192 zip_path: Path to proto APK file. 193 is_bundle_module: True for bundle modules. 194 shared_resources_allowlist: Set of resource names to not extract out of the 195 main package. 196 """ 197 198 def process_func(filename, data): 199 if filename == 'resources.pb': 200 table = Resources_pb2.ResourceTable() 201 table.ParseFromString(data) 202 _HardcodeInTable(table, is_bundle_module, shared_resources_allowlist) 203 data = table.SerializeToString() 204 elif filename.endswith('.xml') and not filename.startswith('res/raw'): 205 xml_node = Resources_pb2.XmlNode() 206 xml_node.ParseFromString(data) 207 _ProcessProtoXmlNode(xml_node) 208 data = xml_node.SerializeToString() 209 return data 210 211 _ProcessZip(zip_path, process_func) 212 213 214 class _ResourceStripper: 215 def __init__(self, partial_path, keep_predicate): 216 self.partial_path = partial_path 217 self.keep_predicate = keep_predicate 218 self._has_changes = False 219 220 @staticmethod 221 def _IterStyles(entry): 222 for config_value in entry.config_value: 223 value = config_value.value 224 if value.HasField('compound_value'): 225 compound_value = value.compound_value 226 if compound_value.HasField('style'): 227 yield compound_value.style 228 229 def _StripStyles(self, entry, type_and_name): 230 # Strip style entries that refer to attributes that have been stripped. 231 for style in self._IterStyles(entry): 232 entries = style.entry 233 new_entries = [] 234 for e in entries: 235 full_name = '{}/{}'.format(type_and_name, e.key.name) 236 if not self.keep_predicate(full_name): 237 logging.debug('Stripped %s/%s', self.partial_path, full_name) 238 else: 239 new_entries.append(e) 240 241 if len(new_entries) != len(entries): 242 self._has_changes = True 243 del entries[:] 244 entries.extend(new_entries) 245 246 def _StripEntries(self, entries, type_name): 247 new_entries = [] 248 for entry in entries: 249 type_and_name = '{}/{}'.format(type_name, entry.name) 250 if not self.keep_predicate(type_and_name): 251 logging.debug('Stripped %s/%s', self.partial_path, type_and_name) 252 else: 253 new_entries.append(entry) 254 self._StripStyles(entry, type_and_name) 255 256 if len(new_entries) != len(entries): 257 self._has_changes = True 258 del entries[:] 259 entries.extend(new_entries) 260 261 def StripTable(self, table): 262 self._has_changes = False 263 for package in table.package: 264 for _type in package.type: 265 self._StripEntries(_type.entry, _type.name) 266 return self._has_changes 267 268 269 def _TableFromFlatBytes(data): 270 # https://cs.android.com/search?q=f:aapt2.*Container.cpp 271 size_idx = len(_FLAT_ARSC_HEADER) 272 proto_idx = size_idx + 8 273 if data[:size_idx] != _FLAT_ARSC_HEADER: 274 raise Exception('Error parsing {} in {}'.format(info.filename, zip_path)) 275 # Size is stored as uint64. 276 size = struct.unpack('<Q', data[size_idx:proto_idx])[0] 277 table = Resources_pb2.ResourceTable() 278 proto_bytes = data[proto_idx:proto_idx + size] 279 table.ParseFromString(proto_bytes) 280 return table 281 282 283 def _FlatBytesFromTable(table): 284 proto_bytes = table.SerializeToString() 285 size = struct.pack('<Q', len(proto_bytes)) 286 overage = len(proto_bytes) % 4 287 padding = b'\0' * (4 - overage) if overage else b'' 288 return b''.join((_FLAT_ARSC_HEADER, size, proto_bytes, padding)) 289 290 291 def StripUnwantedResources(partial_path, keep_predicate): 292 """Removes resources from .arsc.flat files inside of a .zip. 293 294 Args: 295 partial_path: Path to a .zip containing .arsc.flat entries 296 keep_predicate: Given "$partial_path/$res_type/$res_name", returns 297 whether to keep the resource. 298 """ 299 stripper = _ResourceStripper(partial_path, keep_predicate) 300 301 def process_file(filename, data): 302 if filename.endswith('.arsc.flat'): 303 table = _TableFromFlatBytes(data) 304 if stripper.StripTable(table): 305 data = _FlatBytesFromTable(table) 306 return data 307 308 _ProcessZip(partial_path, process_file)