setup_toolchain.py (13041B)
1 # Copyright 2013 The Chromium Authors 2 # Use of this source code is governed by a BSD-style license that can be 3 # found in the LICENSE file. 4 # 5 # Copies the given "win tool" (which the toolchain uses to wrap compiler 6 # invocations) and the environment blocks for the 32-bit and 64-bit builds on 7 # Windows to the build directory. 8 # 9 # The arguments are the visual studio install location and the location of the 10 # win tool. The script assumes that the root build directory is the current dir 11 # and the files will be written to the current directory. 12 13 14 import errno 15 import json 16 import os 17 import re 18 import subprocess 19 import sys 20 21 sys.path.append(os.path.join(os.path.dirname(__file__), os.pardir, os.pardir)) 22 import gn_helpers 23 24 SCRIPT_DIR = os.path.dirname(__file__) 25 SDK_VERSION = '10.0.26100.0' 26 27 28 def _ExtractImportantEnvironment(output_of_set): 29 """Extracts environment variables required for the toolchain to run from 30 a textual dump output by the cmd.exe 'set' command.""" 31 envvars_to_save = ( 32 'cipd_cache_dir', # needed by vpython 33 'homedrive', # needed by vpython 34 'homepath', # needed by vpython 35 'include', 36 'lib', 37 'libpath', 38 'luci_context', # needed by vpython 39 'path', 40 'pathext', 41 'systemroot', 42 'temp', 43 'tmp', 44 'userprofile', # needed by vpython 45 'vpython_virtualenv_root' # needed by vpython 46 ) 47 env = {} 48 # This occasionally happens and leads to misleading SYSTEMROOT error messages 49 # if not caught here. 50 if output_of_set.count('=') == 0: 51 raise Exception('Invalid output_of_set. Value is:\n%s' % output_of_set) 52 for line in output_of_set.splitlines(): 53 for envvar in envvars_to_save: 54 if re.match(envvar + '=', line.lower()): 55 var, setting = line.split('=', 1) 56 if envvar == 'path': 57 # Our own rules and actions in Chromium rely on python being in the 58 # path. Add the path to this python here so that if it's not in the 59 # path when ninja is run later, python will still be found. 60 setting = os.path.dirname(sys.executable) + os.pathsep + setting 61 if envvar in ['include', 'lib']: 62 # Make sure that the include and lib paths point to directories that 63 # exist. This ensures a (relatively) clear error message if the 64 # required SDK is not installed. 65 for part in setting.split(';'): 66 if not os.path.exists(part) and len(part) != 0: 67 raise Exception( 68 'Path "%s" from environment variable "%s" does not exist. ' 69 'Make sure the necessary SDK is installed.' % (part, envvar)) 70 env[var.upper()] = setting 71 break 72 if sys.platform in ('win32', 'cygwin'): 73 for required in ('SYSTEMROOT', 'TEMP', 'TMP'): 74 if required not in env: 75 raise Exception('Environment variable "%s" ' 76 'required to be set to valid path' % required) 77 return env 78 79 80 def _DetectVisualStudioPath(): 81 """Return path to the installed Visual Studio. 82 """ 83 84 # Use the code in build/vs_toolchain.py to avoid duplicating code. 85 chromium_dir = os.path.abspath(os.path.join(SCRIPT_DIR, '..', '..', '..')) 86 sys.path.append(os.path.join(chromium_dir, 'build')) 87 import vs_toolchain 88 return vs_toolchain.DetectVisualStudioPath() 89 90 91 def _LoadEnvFromBat(args): 92 """Given a bat command, runs it and returns env vars set by it.""" 93 args = args[:] 94 args.extend(('&&', 'set')) 95 popen = subprocess.Popen( 96 args, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) 97 variables, _ = popen.communicate() 98 if popen.returncode != 0: 99 raise Exception('"%s" failed with error %d' % (args, popen.returncode)) 100 return variables.decode(errors='ignore') 101 102 103 def _LoadToolchainEnv(cpu, toolchain_root, sdk_dir, target_store): 104 """Returns a dictionary with environment variables that must be set while 105 running binaries from the toolchain (e.g. INCLUDE and PATH for cl.exe).""" 106 # Check if we are running in the SDK command line environment and use 107 # the setup script from the SDK if so. |cpu| should be either 108 # 'x86' or 'x64' or 'arm' or 'arm64'. 109 assert cpu in ('x86', 'x64', 'arm', 'arm64') 110 if bool(int(os.environ.get('DEPOT_TOOLS_WIN_TOOLCHAIN', 1))) and sdk_dir: 111 # Load environment from json file. 112 env = os.path.normpath(os.path.join(sdk_dir, 'bin/SetEnv.%s.json' % cpu)) 113 env = json.load(open(env))['env'] 114 if env['VSINSTALLDIR'] == [["..", "..\\"]]: 115 # Old-style paths were relative to the win_sdk\bin directory. 116 json_relative_dir = os.path.join(sdk_dir, 'bin') 117 else: 118 # New-style paths are relative to the toolchain directory. 119 json_relative_dir = toolchain_root 120 for k in env: 121 entries = [os.path.join(*([json_relative_dir] + e)) for e in env[k]] 122 # clang-cl wants INCLUDE to be ;-separated even on non-Windows, 123 # lld-link wants LIB to be ;-separated even on non-Windows. Path gets :. 124 # The separator for INCLUDE here must match the one used in main() below. 125 sep = os.pathsep if k == 'PATH' else ';' 126 env[k] = sep.join(entries) 127 # PATH is a bit of a special case, it's in addition to the current PATH. 128 env['PATH'] = env['PATH'] + os.pathsep + os.environ['PATH'] 129 # Augment with the current env to pick up TEMP and friends. 130 for k in os.environ: 131 if k not in env: 132 env[k] = os.environ[k] 133 134 varlines = [] 135 for k in sorted(env.keys()): 136 varlines.append('%s=%s' % (str(k), str(env[k]))) 137 variables = '\n'.join(varlines) 138 139 # Check that the json file contained the same environment as the .cmd file. 140 if sys.platform in ('win32', 'cygwin'): 141 script = os.path.normpath(os.path.join(sdk_dir, 'Bin/SetEnv.cmd')) 142 arg = '/' + cpu 143 json_env = _ExtractImportantEnvironment(variables) 144 cmd_env = _ExtractImportantEnvironment(_LoadEnvFromBat([script, arg])) 145 assert _LowercaseDict(json_env) == _LowercaseDict(cmd_env) 146 else: 147 if 'GYP_MSVS_OVERRIDE_PATH' not in os.environ: 148 os.environ['GYP_MSVS_OVERRIDE_PATH'] = _DetectVisualStudioPath() 149 # We only support x64-hosted tools. 150 script_path = os.path.normpath(os.path.join( 151 os.environ['GYP_MSVS_OVERRIDE_PATH'], 152 'VC/vcvarsall.bat')) 153 if not os.path.exists(script_path): 154 # vcvarsall.bat for VS 2017 fails if run after running vcvarsall.bat from 155 # VS 2013 or VS 2015. Fix this by clearing the vsinstalldir environment 156 # variable. Since vcvarsall.bat appends to the INCLUDE, LIB, and LIBPATH 157 # environment variables we need to clear those to avoid getting double 158 # entries when vcvarsall.bat has been run before gn gen. vcvarsall.bat 159 # also adds to PATH, but there is no clean way of clearing that and it 160 # doesn't seem to cause problems. 161 if 'VSINSTALLDIR' in os.environ: 162 del os.environ['VSINSTALLDIR'] 163 if 'INCLUDE' in os.environ: 164 del os.environ['INCLUDE'] 165 if 'LIB' in os.environ: 166 del os.environ['LIB'] 167 if 'LIBPATH' in os.environ: 168 del os.environ['LIBPATH'] 169 other_path = os.path.normpath(os.path.join( 170 os.environ['GYP_MSVS_OVERRIDE_PATH'], 171 'VC/Auxiliary/Build/vcvarsall.bat')) 172 if not os.path.exists(other_path): 173 raise Exception('%s is missing - make sure VC++ tools are installed.' % 174 script_path) 175 script_path = other_path 176 cpu_arg = "amd64" 177 if (cpu != 'x64'): 178 # x64 is default target CPU thus any other CPU requires a target set 179 cpu_arg += '_' + cpu 180 args = [script_path, cpu_arg, ] 181 # Store target must come before any SDK version declaration 182 if (target_store): 183 args.append('store') 184 # Explicitly specifying the SDK version to build with to avoid accidentally 185 # building with a new and untested SDK. This should stay in sync with the 186 # packaged toolchain in build/vs_toolchain.py. 187 args.append(SDK_VERSION) 188 variables = _LoadEnvFromBat(args) 189 return _ExtractImportantEnvironment(variables) 190 191 192 def _FormatAsEnvironmentBlock(envvar_dict): 193 """Format as an 'environment block' directly suitable for CreateProcess. 194 Briefly this is a list of key=value\0, terminated by an additional \0. See 195 CreateProcess documentation for more details.""" 196 block = '' 197 nul = '\0' 198 for key, value in envvar_dict.items(): 199 block += key + '=' + value + nul 200 block += nul 201 return block 202 203 204 def _LowercaseDict(d): 205 """Returns a copy of `d` with both key and values lowercased. 206 207 Args: 208 d: dict to lowercase (e.g. {'A': 'BcD'}). 209 210 Returns: 211 A dict with both keys and values lowercased (e.g.: {'a': 'bcd'}). 212 """ 213 return {k.lower(): d[k].lower() for k in d} 214 215 216 def FindFileInEnvList(env, env_name, separator, file_name, optional=False): 217 parts = env[env_name].split(separator) 218 for path in parts: 219 if os.path.exists(os.path.join(path, file_name)): 220 return os.path.realpath(path) 221 assert optional, "%s is not found in %s:\n%s\nCheck if it is installed." % ( 222 file_name, env_name, '\n'.join(parts)) 223 return '' 224 225 226 def main(): 227 if len(sys.argv) != 7: 228 print('Usage setup_toolchain.py ' 229 '<visual studio path> <win sdk path> ' 230 '<runtime dirs> <target_os> <target_cpu> ' 231 '<environment block name|none>') 232 sys.exit(2) 233 # toolchain_root and win_sdk_path are only read if the hermetic Windows 234 # toolchain is set, that is if DEPOT_TOOLS_WIN_TOOLCHAIN is not set to 0. 235 # With the hermetic Windows toolchain, the visual studio path in argv[1] 236 # is the root of the Windows toolchain directory. 237 toolchain_root = sys.argv[1] 238 win_sdk_path = sys.argv[2] 239 240 runtime_dirs = sys.argv[3] 241 target_os = sys.argv[4] 242 target_cpu = sys.argv[5] 243 environment_block_name = sys.argv[6] 244 if (environment_block_name == 'none'): 245 environment_block_name = '' 246 247 if (target_os == 'winuwp'): 248 target_store = True 249 else: 250 target_store = False 251 252 cpus = ('x86', 'x64', 'arm', 'arm64') 253 assert target_cpu in cpus 254 vc_bin_dir = 'fake_path/cl.exe' 255 include = '' 256 lib = '' 257 258 def relflag(s): # Make s relative to builddir when cwd and sdk on same drive. 259 try: 260 return os.path.relpath(s).replace('\\', '/') 261 except ValueError: 262 return s 263 264 def q(s): # Quote s if it contains spaces or other weird characters. 265 return s if re.match(r'^[a-zA-Z0-9._/\\:-]*$', s) else '"' + s + '"' 266 267 # for cpu in cpus: 268 # if cpu == target_cpu: 269 # # Extract environment variables for subprocesses. 270 # env = _LoadToolchainEnv(cpu, toolchain_root, win_sdk_path, target_store) 271 # env['PATH'] = runtime_dirs + os.pathsep + env['PATH'] 272 # 273 # vc_bin_dir = FindFileInEnvList(env, 'PATH', os.pathsep, 'cl.exe') 274 # 275 # # The separator for INCLUDE here must match the one used in 276 # # _LoadToolchainEnv() above. 277 # include = [p.replace('"', r'\"') for p in env['INCLUDE'].split(';') if p] 278 # include = list(map(relflag, include)) 279 # 280 # lib = [p.replace('"', r'\"') for p in env['LIB'].split(';') if p] 281 # lib = list(map(relflag, lib)) 282 # 283 # include_I = ['/I' + i for i in include] 284 # include_imsvc = ['-imsvc' + i for i in include] 285 # libpath_flags = ['-libpath:' + i for i in lib] 286 # 287 # if (environment_block_name != ''): 288 # env_block = _FormatAsEnvironmentBlock(env) 289 # with open(environment_block_name, 'w', encoding='utf8') as f: 290 # f.write(env_block) 291 292 def ListToArgString(x): 293 return gn_helpers.ToGNString(' '.join(q(i) for i in x)) 294 295 def ListToArgList(x): 296 return f'[{", ".join(gn_helpers.ToGNString(i) for i in x)}]' 297 298 env = {} 299 env['PATH'] = '' 300 include_I = include 301 include_imsvc = include 302 libpath_flags = '' 303 print('vc_bin_dir = ' + gn_helpers.ToGNString(vc_bin_dir)) 304 print(f'include_flags_I = {ListToArgString(include_I)}') 305 print(f'include_flags_I_list = {ListToArgList(include_I)}') 306 if bool(int(os.environ.get('DEPOT_TOOLS_WIN_TOOLCHAIN', 1))) and win_sdk_path: 307 flags = ['/winsysroot' + relflag(toolchain_root)] 308 print(f'include_flags_imsvc = {ListToArgString(flags)}') 309 print(f'include_flags_imsvc_list = {ListToArgList(flags)}') 310 else: 311 print(f'include_flags_imsvc = {ListToArgString(include_imsvc)}') 312 print(f'include_flags_imsvc_list = {ListToArgList(include_imsvc)}') 313 print('paths = ' + gn_helpers.ToGNString(env['PATH'])) 314 print(f'libpath_flags = {ListToArgString(libpath_flags)}') 315 print(f'libpath_flags_list = {ListToArgList(libpath_flags)}') 316 if bool(int(os.environ.get('DEPOT_TOOLS_WIN_TOOLCHAIN', 1))) and win_sdk_path: 317 flags = ['/winsysroot:' + relflag(toolchain_root)] 318 print(f'libpath_lldlink_flags = {ListToArgString(flags)}') 319 print(f'libpath_lldlink_flags_list = {ListToArgList(flags)}') 320 else: 321 print(f'libpath_lldlink_flags = {ListToArgString(libpath_flags)}') 322 print(f'libpath_lldlink_flags_list = {ListToArgList(libpath_flags)}') 323 324 325 if __name__ == '__main__': 326 main()