generate_def_files.py (7686B)
1 #!/usr/bin/env python3 2 3 """Script to generate Chromium's Abseil .def files at roll time. 4 5 This script generates //third_party/abseil-app/absl/symbols_*.def at Abseil 6 roll time. 7 8 Since Abseil doesn't export symbols, Chromium is forced to consider all 9 Abseil's symbols as publicly visible. On POSIX it is possible to use 10 -fvisibility=default but on Windows a .def file with all the symbols 11 is needed. 12 13 Unless you are on a Windows machine, you need to set up your Chromium 14 checkout for cross-compilation by following the instructions at 15 https://chromium.googlesource.com/chromium/src.git/+/main/docs/win_cross.md. 16 If you are on Windows, you may need to tweak this script to run, e.g. by 17 changing "gn" to "gn.bat", changing "llvm-nm" to the name of your copy of 18 llvm-nm, etc. 19 """ 20 21 import fnmatch 22 import logging 23 import os 24 import re 25 import subprocess 26 import sys 27 import tempfile 28 import time 29 30 assert sys.platform != 'win32', \ 31 "This doesn't work on Windows due to https://crbug.com/3790230222" 32 33 # Matches mangled symbols containing 'absl' or starting with 'Absl'. This is 34 # a good enough heuristic to select Abseil symbols to list in the .def file. 35 # See https://learn.microsoft.com/en-us/cpp/build/reference/decorated-names, 36 # which describes decorations under different calling conventions. We mostly 37 # just attempt to handle any leading underscore for C names (as in __cdecl). 38 ABSL_SYM_RE = r'0* [BT] (?P<symbol>[?]+[^?].*absl.*|_?Absl.*)' 39 if sys.platform == 'win32': 40 # Typical dumpbin /symbol lines look like this: 41 # 04B 0000000C SECT14 notype Static | ?$S1@?1??SetCurrent 42 # ThreadIdentity@base_internal@absl@@YAXPAUThreadIdentity@12@P6AXPAX@Z@Z@4IA 43 # (unsigned int `void __cdecl absl::base_internal::SetCurrentThreadIdentity... 44 # We need to start on "| ?" and end on the first " (" (stopping on space would 45 # also work). 46 # This regex is identical inside the () characters except for the ? after .*, 47 # which is needed to prevent greedily grabbing the undecorated version of the 48 # symbols. 49 ABSL_SYM_RE = r'.*External \| (?P<symbol>[?]+[^?].*?absl.*?|_?Absl.*?)($| \(.*)' 50 # Typical exported symbols in dumpbin /directives look like: 51 # /EXPORT:?kHexChar@numbers_internal@absl@@3QBDB,DATA 52 ABSL_EXPORTED_RE = r'.*/EXPORT:(.*),.*' 53 54 55 def _DebugOrRelease(is_debug): 56 return 'dbg' if is_debug else 'rel' 57 58 59 def _GenerateDefFile(cpu, is_debug, extra_gn_args=[], suffix=None): 60 """Generates a .def file for the absl component build on the specified CPU.""" 61 if extra_gn_args: 62 assert suffix != None, 'suffix is needed when extra_gn_args is used' 63 64 flavor = _DebugOrRelease(is_debug) 65 gn_args = [ 66 'ffmpeg_branding = "Chrome"', 67 'is_component_build = true', 68 'is_debug = {}'.format(str(is_debug).lower()), 69 'proprietary_codecs = true', 70 'symbol_level = 0', 71 'target_cpu = "{}"'.format(cpu), 72 'target_os = "win"', 73 ] 74 gn_args.extend(extra_gn_args) 75 76 gn = 'gn' 77 autoninja = 'autoninja' 78 symbol_dumper = ['third_party/llvm-build/Release+Asserts/bin/llvm-nm'] 79 if sys.platform == 'win32': 80 gn = 'gn.bat' 81 autoninja = 'autoninja.bat' 82 symbol_dumper = ['dumpbin', '/symbols'] 83 import shutil 84 if not shutil.which('dumpbin'): 85 logging.error('dumpbin not found. Run tools\\win\\setenv.bat.') 86 exit(1) 87 cwd = os.getcwd() 88 with tempfile.TemporaryDirectory(dir=cwd) as out_dir: 89 logging.info('[%s - %s] Creating tmp out dir in %s', cpu, flavor, out_dir) 90 subprocess.check_call([gn, 'gen', out_dir, '--args=' + ' '.join(gn_args)], 91 cwd=cwd) 92 logging.info('[%s - %s] gn gen completed', cpu, flavor) 93 subprocess.check_call( 94 [autoninja, '-C', out_dir, 'third_party/abseil-cpp:absl_component_deps'], 95 cwd=os.getcwd()) 96 logging.info('[%s - %s] autoninja completed', cpu, flavor) 97 98 obj_files = [] 99 for root, _dirnames, filenames in os.walk( 100 os.path.join(out_dir, 'obj', 'third_party', 'abseil-cpp')): 101 matched_files = fnmatch.filter(filenames, '*.obj') 102 obj_files.extend((os.path.join(root, f) for f in matched_files)) 103 104 logging.info('[%s - %s] Found %d object files.', cpu, flavor, len(obj_files)) 105 106 absl_symbols = set() 107 dll_exports = set() 108 if sys.platform == 'win32': 109 for f in obj_files: 110 # Track all of the functions exported with __declspec(dllexport) and 111 # don't list them in the .def file - double-exports are not allowed. The 112 # error is "lld-link: error: duplicate /export option". 113 exports_out = subprocess.check_output(['dumpbin', '/directives', f], cwd=os.getcwd()) 114 for line in exports_out.splitlines(): 115 line = line.decode('utf-8') 116 match = re.match(ABSL_EXPORTED_RE, line) 117 if match: 118 dll_exports.add(match.groups()[0]) 119 for f in obj_files: 120 stdout = subprocess.check_output(symbol_dumper + [f], cwd=os.getcwd()) 121 for line in stdout.splitlines(): 122 try: 123 line = line.decode('utf-8') 124 except UnicodeDecodeError: 125 # Due to a dumpbin bug there are sometimes invalid utf-8 characters in 126 # the output. This only happens on an unimportant line so it can 127 # safely and silently be skipped. 128 # https://developercommunity.visualstudio.com/content/problem/1091330/dumpbin-symbols-produces-randomly-wrong-output-on.html 129 continue 130 match = re.match(ABSL_SYM_RE, line) 131 if match: 132 symbol = match.group('symbol') 133 assert symbol.count(' ') == 0, ('Regex matched too much, probably got ' 134 'undecorated name as well') 135 # Avoid getting names exported with dllexport, to avoid 136 # "lld-link: error: duplicate /export option" on symbols such as: 137 # ?kHexChar@numbers_internal@absl@@3QBDB 138 if symbol in dll_exports: 139 continue 140 # Avoid to export deleting dtors since they trigger 141 # "lld-link: error: export of deleting dtor" linker errors, see 142 # crbug.com/1201277. 143 if symbol.startswith('??_G'): 144 continue 145 # Strip any leading underscore for C names (as in __cdecl). It's only 146 # there on x86, but the x86 toolchain falls over when you include it! 147 if cpu == 'x86' and symbol.startswith('_'): 148 symbol = symbol[1:] 149 absl_symbols.add(symbol) 150 151 logging.info('[%s - %s] Found %d absl symbols.', cpu, flavor, len(absl_symbols)) 152 153 if extra_gn_args: 154 def_file = os.path.join('third_party', 'abseil-cpp', 155 'symbols_{}_{}_{}.def'.format(cpu, flavor, suffix)) 156 else: 157 def_file = os.path.join('third_party', 'abseil-cpp', 158 'symbols_{}_{}.def'.format(cpu, flavor)) 159 160 with open(def_file, 'w', newline='') as f: 161 f.write('EXPORTS\n') 162 for s in sorted(absl_symbols): 163 f.write(' {}\n'.format(s)) 164 165 # Hack, it looks like there is a race in the directory cleanup. 166 time.sleep(10) 167 168 logging.info('[%s - %s] .def file successfully generated.', cpu, flavor) 169 170 171 if __name__ == '__main__': 172 logging.getLogger().setLevel(logging.INFO) 173 174 if sys.version_info.major == 2: 175 logging.error('This script requires Python 3.') 176 exit(1) 177 178 if not os.getcwd().endswith('src') or not os.path.exists('chrome/browser'): 179 logging.error('Run this script from a chromium/src/ directory.') 180 exit(1) 181 182 _GenerateDefFile('x86', True) 183 _GenerateDefFile('x86', False) 184 _GenerateDefFile('x64', True) 185 _GenerateDefFile('x64', False) 186 _GenerateDefFile('x64', False, ['is_asan = true'], 'asan') 187 _GenerateDefFile('arm64', True) 188 _GenerateDefFile('arm64', False)