rustc_wrapper.py (8658B)
1 #!/usr/bin/env python3 2 3 # Copyright 2021 The Chromium Authors 4 # Use of this source code is governed by a BSD-style license that can be 5 # found in the LICENSE file. 6 7 import argparse 8 import pathlib 9 import subprocess 10 import shlex 11 import os 12 import sys 13 import re 14 15 # Set up path to be able to import action_helpers. 16 sys.path.append( 17 os.path.join(os.path.dirname(os.path.abspath(__file__)), os.pardir, 18 os.pardir, 'build')) 19 import action_helpers 20 21 # This script wraps rustc for (currently) these reasons: 22 # * To work around some ldflags escaping performed by ninja/gn 23 # * To remove dependencies on some environment variables from the .d file. 24 # * To enable use of .rsp files. 25 # * To work around two gn bugs on Windows 26 # 27 # LDFLAGS ESCAPING 28 # 29 # This script performs a simple function to work around some of the 30 # parameter escaping performed by ninja/gn. 31 # 32 # rustc invocations are given access to {{rustflags}} and {{ldflags}}. 33 # We want to pass {{ldflags}} into rustc, using -Clink-args="{{ldflags}}". 34 # Unfortunately, ninja assumes that each item in {{ldflags}} is an 35 # independent command-line argument and will have escaped them appropriately 36 # for use on a bare command line, instead of in a string. 37 # 38 # This script converts such {{ldflags}} into individual -Clink-arg=X 39 # arguments to rustc. 40 # 41 # RUSTENV dependency stripping 42 # 43 # When Rust code depends on an environment variable at build-time 44 # (using the env! macro), rustc spots that and adds it to the .d file. 45 # Ninja then parses that .d file and determines that the environment 46 # dependency means that the target always needs to be rebuilt. 47 # 48 # That's all correct, but _we_ know that some of these environment 49 # variables (typically, all of them) are set by .gn files which ninja 50 # tracks independently. So we remove them from the .d file. 51 # 52 # RSP files: 53 # 54 # We want to put the ninja/gn variables {{rustdeps}} and {{externs}} 55 # in an RSP file. Unfortunately, they are space-separated variables 56 # but Rust requires a newline-separated input. This script duly makes 57 # the adjustment. This works around a gn issue: 58 # TODO(https://bugs.chromium.org/p/gn/issues/detail?id=249): fix this 59 # 60 # WORKAROUND WINDOWS BUGS: 61 # 62 # On Windows platforms, this temporarily works around some issues in gn. 63 # See comments inline, linking to the relevant gn fixes. 64 # 65 # Usage: 66 # rustc_wrapper.py --rustc <path to rustc> --depfile <path to .d file> 67 # -- <normal rustc args> LDFLAGS {{ldflags}} RUSTENV {{rustenv}} 68 # The LDFLAGS token is discarded, and everything after that is converted 69 # to being a series of -Clink-arg=X arguments, until or unless RUSTENV 70 # is encountered, after which those are interpreted as environment 71 # variables to pass to rustc (and which will be removed from the .d file). 72 # 73 # Both LDFLAGS and RUSTENV **MUST** be specified, in that order, even if 74 # the list following them is empty. 75 # 76 # TODO(https://github.com/rust-lang/rust/issues/73632): avoid using rustc 77 # for linking in the first place. Most of our binaries are linked using 78 # clang directly, but there are some types of Rust build product which 79 # must currently be created by rustc (e.g. unit test executables). As 80 # part of support for using non-rustc linkers, we should arrange to extract 81 # such functionality from rustc so that we can make all types of binary 82 # using our clang toolchain. That will remove the need for most of this 83 # script. 84 85 FILE_RE = re.compile("[^:]+: (.+)") 86 87 88 # Equivalent of python3.9 built-in 89 def remove_lib_suffix_from_l_args(text): 90 if text.startswith("-l") and text.endswith(".lib"): 91 return text[:-len(".lib")] 92 return text 93 94 95 def verify_inputs(depline, sources, abs_build_root): 96 """Verify everything used by rustc (found in `depline`) was specified in the 97 GN build rule (found in `sources` or `inputs`). 98 99 TODO(danakj): This allows things in `sources` that were not actually used by 100 rustc since third-party packages sources need to be a union of all build 101 configs/platforms for simplicity in generating build rules. For first-party 102 code we could be more strict and reject things in `sources` that were not 103 consumed. 104 """ 105 106 # str.removeprefix() does not exist before python 3.9. 107 def remove_prefix(text, prefix): 108 if text.startswith(prefix): 109 return text[len(prefix):] 110 return text 111 112 def normalize_path(p): 113 return os.path.relpath(os.path.normpath(remove_prefix( 114 p, abs_build_root))).replace('\\', '/') 115 116 # Collect the files that rustc says are needed. 117 found_files = {} 118 m = FILE_RE.match(depline) 119 if m: 120 files = m.group(1) 121 found_files = {normalize_path(f): f for f in files.split()} 122 # Get which ones are not listed in GN. 123 missing_files = found_files.keys() - sources 124 125 if not missing_files: 126 return True 127 128 # The matching did a bunch of path manipulation to get paths relative to the 129 # build dir such that they would match GN. In errors, we will print out the 130 # exact path that rustc produces for easier debugging and writing of stdlib 131 # config rules. 132 for file_files_key in missing_files: 133 gn_type = "sources" if file_files_key.endswith(".rs") else "inputs" 134 print(f'ERROR: file not in GN {gn_type}: {found_files[file_files_key]}', 135 file=sys.stderr) 136 return False 137 138 139 def main(): 140 parser = argparse.ArgumentParser() 141 parser.add_argument('--rustc', required=True, type=pathlib.Path) 142 parser.add_argument('--depfile', required=True, type=pathlib.Path) 143 parser.add_argument('--rsp', type=pathlib.Path, required=True) 144 parser.add_argument('--target-windows', action='store_true') 145 parser.add_argument('-v', action='store_true') 146 parser.add_argument('args', metavar='ARG', nargs='+') 147 148 args = parser.parse_args() 149 150 remaining_args = args.args 151 152 ldflags_separator = remaining_args.index("LDFLAGS") 153 rustenv_separator = remaining_args.index("RUSTENV", ldflags_separator) 154 # Sometimes we duplicate the SOURCES list into the command line for debugging 155 # issues on the bots. 156 try: 157 sources_separator = remaining_args.index("SOURCES", rustenv_separator) 158 except: 159 sources_separator = None 160 rustc_args = remaining_args[:ldflags_separator] 161 ldflags = remaining_args[ldflags_separator + 1:rustenv_separator] 162 rustenv = remaining_args[rustenv_separator + 1:sources_separator] 163 164 abs_build_root = os.getcwd().replace('\\', '/') + '/' 165 is_windows = sys.platform == 'win32' or args.target_windows 166 167 rustc_args.extend(["-Clink-arg=%s" % arg for arg in ldflags]) 168 169 with open(args.rsp) as rspfile: 170 rsp_args = [l.rstrip() for l in rspfile.read().split(' ') if l.rstrip()] 171 172 sources_separator = rsp_args.index("SOURCES") 173 sources = set(rsp_args[sources_separator + 1:]) 174 rsp_args = rsp_args[:sources_separator] 175 176 if is_windows: 177 # Work around for "-l<foo>.lib", where ".lib" suffix is undesirable. 178 # Full fix will come from https://gn-review.googlesource.com/c/gn/+/12480 179 rsp_args = [remove_lib_suffix_from_l_args(arg) for arg in rsp_args] 180 rustc_args = [remove_lib_suffix_from_l_args(arg) for arg in rustc_args] 181 out_rsp = str(args.rsp) + ".rsp" 182 with open(out_rsp, 'w') as rspfile: 183 # rustc needs the rsp file to be separated by newlines. Note that GN 184 # generates the file separated by spaces: 185 # https://bugs.chromium.org/p/gn/issues/detail?id=249, 186 rspfile.write("\n".join(rsp_args)) 187 rustc_args.append(f'@{out_rsp}') 188 189 env = os.environ.copy() 190 fixed_env_vars = [] 191 for item in rustenv: 192 (k, v) = item.split("=", 1) 193 env[k] = v 194 fixed_env_vars.append(k) 195 196 try: 197 if args.v: 198 print(' '.join(f'{k}={shlex.quote(v)}' for k, v in env.items()), 199 args.rustc, shlex.join(rustc_args)) 200 r = subprocess.run([args.rustc, *rustc_args], env=env, check=False) 201 finally: 202 if not args.v: 203 os.remove(out_rsp) 204 if r.returncode != 0: 205 sys.exit(r.returncode) 206 207 final_depfile_lines = [] 208 dirty = False 209 with open(args.depfile, encoding="utf-8") as d: 210 # Figure out which lines we want to keep in the depfile. If it's not the 211 # whole file, we will rewrite the file. 212 env_dep_re = re.compile("# env-dep:(.*)=.*") 213 for line in d: 214 m = env_dep_re.match(line) 215 if m and m.group(1) in fixed_env_vars: 216 dirty = True # We want to skip this line. 217 else: 218 final_depfile_lines.append(line) 219 220 # Verify each dependent file is listed in sources/inputs. 221 for line in final_depfile_lines: 222 if not verify_inputs(line, sources, abs_build_root): 223 return 1 224 225 if dirty: # we made a change, let's write out the file 226 with action_helpers.atomic_output(args.depfile) as output: 227 output.write("\n".join(final_depfile_lines).encode("utf-8")) 228 229 230 if __name__ == '__main__': 231 sys.exit(main())