reproxy.star (13458B)
1 # -*- bazel-starlark -*- 2 # Copyright 2023 The Chromium Authors 3 # Use of this source code is governed by a BSD-style license that can be 4 # found in the LICENSE file. 5 """Siso configuration for rewriting remote calls into reproxy config.""" 6 7 load("@builtin//encoding.star", "json") 8 load("@builtin//path.star", "path") 9 load("@builtin//runtime.star", "runtime") 10 load("@builtin//struct.star", "module") 11 load("./clang_code_coverage_wrapper.star", "clang_code_coverage_wrapper") 12 load("./config.star", "config") 13 load("./gn_logs.star", "gn_logs") 14 load("./platform.star", "platform") 15 load("./rewrapper_cfg.star", "rewrapper_cfg") 16 17 def __filegroups(ctx): 18 return {} 19 20 def __parse_rewrapper_cmdline(ctx, cmd): 21 if not "rewrapper" in cmd.args[0]: 22 return [], "", False 23 24 # Example command: 25 # ../../buildtools/reclient/rewrapper 26 # -cfg=../../buildtools/reclient_cfgs/chromium-browser-clang/rewrapper_linux.cfg 27 # -inputs=build/config/unsafe_buffers_paths.txt 28 # -exec_root=/path/to/your/chromium/src/ 29 # ../../third_party/llvm-build/Release+Asserts/bin/clang++ 30 # [rest of clang args] 31 # We don't need to care about: 32 # -exec_root: Siso already knows this. 33 wrapped_command_pos = -1 34 cfg_file = None 35 skip = "" 36 rw_cmd_opts = {} 37 for i, arg in enumerate(cmd.args): 38 if i == 0: 39 continue 40 if arg.startswith("-cfg="): 41 cfg_file = ctx.fs.canonpath(arg.removeprefix("-cfg=")) 42 continue 43 if arg.startswith("-inputs=") or skip == "-inputs": 44 rw_cmd_opts["inputs"] = arg.removeprefix("-inputs=").split(",") 45 skip = "" 46 continue 47 if arg == "-inputs": 48 skip = arg 49 continue 50 if not arg.startswith("-"): 51 wrapped_command_pos = i 52 break 53 if wrapped_command_pos < 1: 54 fail("couldn't find first non-arg passed to rewrapper from %s" % str(cmd.args)) 55 if not cfg_file: 56 fail("couldn't find rewrapper cfg file from %s" % str(cmd.args)) 57 58 # Config options are the lowest prioity. 59 rw_opts = rewrapper_cfg.parse(ctx, cfg_file) 60 61 # TODO: Read RBE_* envvars. 62 if runtime.os == "windows": 63 # Experimenting if longer timeouts resolve slow Windows developer builds. b/335525655 64 rw_opts.update({ 65 "exec_timeout": "4m", 66 "reclient_timeout": "8m", 67 }) 68 if runtime.os == "darwin": 69 # Mac gets timeouts occasionally on large input uploads (likely due to large invalidations) 70 # b/356981080 71 rw_opts.update({ 72 "exec_timeout": "3m", 73 "reclient_timeout": "6m", 74 }) 75 76 # Command line options are the highest priority. 77 rw_opts.update(rw_cmd_opts) 78 return cmd.args[wrapped_command_pos:], rw_opts, True 79 80 def __parse_cros_rewrapper_cmdline(ctx, cmd): 81 # fix cros sdk clang command line and extract rewrapper cfg. 82 # Example command: 83 # ../../build/cros_cache/chrome-sdk/symlinks/amd64-generic+15629.0.0+target_toolchain/bin/x86_64-cros-linux-gnu-clang++ 84 # -MMD -MF obj/third_party/abseil-cpp/absl/base/base/spinlock.o.d 85 # ... 86 # --rewrapper-path /usr/local/google/home/ukai/src/chromium/src/build/args/chromeos/rewrapper_amd64-generic 87 # --rewrapper-cfg ../../buildtools/reclient_cfgs/chromium-browser-clang/rewrapper_linux.cfg 88 # -pipe -march=x86-64 -msse3 ... 89 cfg_file = None 90 skip = "" 91 args = [] 92 toolchainpath = None 93 for i, arg in enumerate(cmd.args): 94 if i == 0: 95 toolchainpath = path.dir(path.dir(ctx.fs.canonpath(arg))) 96 args.append(arg) 97 continue 98 if skip: 99 if skip == "--rewrapper-cfg": 100 cfg_file = ctx.fs.canonpath(arg) 101 skip = "" 102 continue 103 if arg in ("--rewrapper-path", "--rewrapper-cfg"): 104 skip = arg 105 continue 106 args.append(arg) 107 if not cfg_file: 108 fail("couldn't find rewrapper cfg file in %s" % str(cmd.args)) 109 rwcfg = rewrapper_cfg.parse(ctx, cfg_file) 110 inputs = rwcfg.get("inputs", []) 111 inputs.extend([ 112 path.join(toolchainpath, "bin"), 113 path.join(toolchainpath, "lib"), 114 path.join(toolchainpath, "usr/bin"), 115 path.join(toolchainpath, "usr/lib64/clang"), 116 # TODO: b/320189180 - Simple Chrome builds should use libraries under usr/lib64. 117 # But, Ninja/Reclient also don't use them unexpectedly. 118 ]) 119 rwcfg["inputs"] = inputs 120 rwcfg["preserve_symlinks"] = True 121 return args, rwcfg 122 123 # TODO(b/278225415): change gn so this wrapper (and by extension this handler) becomes unnecessary. 124 def __parse_clang_code_coverage_wrapper_cmdline(ctx, cmd): 125 # Example command: 126 # python3 127 # ../../build/toolchain/clang_code_coverage_wrapper.py 128 # --target-os=... 129 # --files_to_instrument=... 130 # ../../buildtools/reclient/rewrapper 131 # -cfg=../../buildtools/reclient_cfgs/chromium-browser-clang/rewrapper_linux.cfg 132 # -inputs=build/config/unsafe_buffers_paths.txt 133 # -exec_root=/path/to/your/chromium/src/ 134 # ../../third_party/llvm-build/Release+Asserts/bin/clang++ 135 # [rest of clang args] 136 # We don't need to care about: 137 # most args to clang_code_coverage_wrapper (need --files_to_instrument as tool_input) 138 # -exec_root: Siso already knows this. 139 rewrapper_pos = -1 140 wrapped_command_pos = -1 141 cfg_file = None 142 skip = None 143 rw_ops = {} 144 for i, arg in enumerate(cmd.args): 145 if i < 2: 146 continue 147 if rewrapper_pos == -1 and not arg.startswith("-"): 148 rewrapper_pos = i 149 continue 150 if rewrapper_pos > 0 and arg.startswith("-cfg="): 151 cfg_file = ctx.fs.canonpath(arg.removeprefix("-cfg=")) 152 continue 153 if arg.startswith("-inputs=") or skip == "-inputs": 154 rw_ops["inputs"] = arg.removeprefix("-inputs=").split(",") 155 skip = "" 156 continue 157 if arg == "-inputs": 158 skip = arg 159 continue 160 if rewrapper_pos > 0 and not arg.startswith("-"): 161 wrapped_command_pos = i 162 break 163 if rewrapper_pos < 1: 164 fail("couldn't find rewrapper in %s" % str(cmd.args)) 165 if wrapped_command_pos < 1: 166 fail("couldn't find first non-arg passed to rewrapper for %s" % str(cmd.args)) 167 if not cfg_file: 168 fail("couldn't find rewrapper cfg file in %s" % str(cmd.args)) 169 coverage_wrapper_command = cmd.args[:rewrapper_pos] + cmd.args[wrapped_command_pos:] 170 clang_command = clang_code_coverage_wrapper.run(ctx, list(coverage_wrapper_command)) 171 if len(clang_command) > 1 and "/chrome-sdk/" in clang_command[0]: 172 # TODO: implement cros sdk support under code coverage wrapper 173 fail("need to fix handler for cros sdk under code coverage wrapper") 174 rw_cfg_opts = rewrapper_cfg.parse(ctx, cfg_file) 175 176 # Command line options have higher priority than the ones in the cfg file. 177 rw_cfg_opts.update(rw_ops) 178 return clang_command, rw_cfg_opts 179 180 def __rewrite_rewrapper(ctx, cmd, use_large = False): 181 # If clang-coverage, needs different handling. 182 if len(cmd.args) > 2 and "clang_code_coverage_wrapper.py" in cmd.args[1]: 183 args, rwcfg = __parse_clang_code_coverage_wrapper_cmdline(ctx, cmd) 184 elif len(cmd.args) > 1 and "/chrome-sdk/" in cmd.args[0]: 185 args, rwcfg = __parse_cros_rewrapper_cmdline(ctx, cmd) 186 else: 187 # handling for generic rewrapper. 188 args, rwcfg, wrapped = __parse_rewrapper_cmdline(ctx, cmd) 189 if not wrapped: 190 print("command doesn't have rewrapper. %s" % str(cmd.args)) 191 return 192 if not rwcfg: 193 fail("couldn't find rewrapper cfg file in %s" % str(cmd.args)) 194 if use_large: 195 platform = rwcfg.get("platform", {}) 196 if platform.get("OSFamily") == "Windows": 197 # Since there is no large Windows workers, it needs to run locally. 198 ctx.actions.fix(args = args) 199 return 200 if platform: 201 action_key = None 202 for key in rwcfg["platform"]: 203 if key.startswith("label:action_"): 204 action_key = key 205 break 206 if action_key: 207 rwcfg["platform"].pop(action_key) 208 else: 209 rwcfg["platform"] = {} 210 rwcfg["platform"].update({ 211 "label:action_large": "1", 212 }) 213 214 # Some large compiles take longer than the default timeout 2m. 215 # same as clang_exception.star. 216 rwcfg["exec_timeout"] = "10m" 217 rwcfg["reclient_timeout"] = "10m" 218 ctx.actions.fix( 219 args = args, 220 reproxy_config = json.encode(rwcfg), 221 ) 222 223 def __rewrite_rewrapper_large(ctx, cmd): 224 return __rewrite_rewrapper(ctx, cmd, use_large = True) 225 226 def __strip_rewrapper(ctx, cmd): 227 # If clang-coverage, needs different handling. 228 if len(cmd.args) > 2 and "clang_code_coverage_wrapper.py" in cmd.args[1]: 229 args, _ = __parse_clang_code_coverage_wrapper_cmdline(ctx, cmd) 230 else: 231 args, _, wrapped = __parse_rewrapper_cmdline(ctx, cmd) 232 if not wrapped: 233 print("command doesn't have rewrapper. %s" % str(cmd.args)) 234 return 235 ctx.actions.fix(args = args) 236 237 __handlers = { 238 "rewrite_rewrapper": __rewrite_rewrapper, 239 "rewrite_rewrapper_large": __rewrite_rewrapper_large, 240 "strip_rewrapper": __strip_rewrapper, 241 } 242 243 def __use_reclient(ctx): 244 return gn_logs.read(ctx).get("use_reclient") == "true" 245 246 def __step_config(ctx, step_config): 247 # New rules to convert commands calling rewrapper to use reproxy instead. 248 new_rules = [] 249 250 # Disable racing on builders since bots don't have many CPU cores. 251 # TODO: b/297807325 - Siso wants to handle local execution. 252 # However, Reclient's alerts require racing and local fallback to be 253 # done on Reproxy side. 254 exec_strategy = "racing" 255 if config.get(ctx, "builder"): 256 exec_strategy = "remote_local_fallback" 257 258 for rule in step_config["rules"]: 259 # Replace nacl-clang/clang++ rules without command_prefix, because they will incorrectly match rewrapper. 260 # Replace the original step rule with one that only rewrites rewrapper and convert its rewrapper config to reproxy config. 261 if rule["name"].find("nacl-clang") >= 0 and not rule.get("command_prefix"): 262 new_rule = { 263 "name": rule["name"], 264 "action": rule["action"], 265 "handler": "rewrite_rewrapper", 266 } 267 new_rules.append(new_rule) 268 continue 269 270 # clang cxx/cc/objcxx/objc will always have rewrapper config when use_remoteexec=true. 271 # Remove the native siso handling and replace with custom rewrapper-specific handling. 272 # All other rule values are not reused, instead use rewrapper config via handler. 273 # (In particular, command_prefix should be avoided because it will be rewrapper.) 274 if (rule["name"].startswith("clang/cxx") or rule["name"].startswith("clang/cc") or 275 rule["name"].startswith("clang-cl/cxx") or rule["name"].startswith("clang-cl/cc") or 276 rule["name"].startswith("clang/objc")): 277 if not rule.get("action"): 278 fail("clang rule %s found without action" % rule["name"]) 279 280 new_rule = { 281 "name": rule["name"], 282 "action": rule["action"], 283 "exclude_input_patterns": rule.get("exclude_input_patterns"), 284 "handler": "rewrite_rewrapper", 285 "input_root_absolute_path": rule.get("input_root_absolute_path"), 286 } 287 new_rules.append(new_rule) 288 continue 289 290 # clang-coverage/ is handled by the rewrite_rewrapper handler of clang/{cxx, cc} action rules above, so ignore these rules. 291 if rule["name"].startswith("clang-coverage/"): 292 continue 293 294 # Add non-remote rules as-is. 295 if not rule.get("remote"): 296 new_rules.append(rule) 297 continue 298 299 # Finally handle remaining remote rules. It's assumed it is enough to only convert native remote config to reproxy config. 300 platform_ref = rule.get("platform_ref") 301 if platform_ref: 302 p = step_config["platforms"].get(platform_ref) 303 if not p: 304 fail("Rule %s uses undefined platform '%s'" % (rule["name"], platform_ref)) 305 else: 306 p = step_config.get("platforms", {}).get("default") 307 if not p: 308 fail("Rule %s did not set platform_ref but no default platform exists" % rule["name"]) 309 rule["reproxy_config"] = { 310 "platform": p, 311 "labels": { 312 "type": "tool", 313 "siso_rule": rule["name"], 314 }, 315 "canonicalize_working_dir": rule.get("canonicalize_dir", False), 316 "exec_strategy": exec_strategy, 317 # TODO: crbug.com/380755128 - Make each compile unit smaller. 318 "exec_timeout": rule.get("timeout", "30m"), 319 "reclient_timeout": rule.get("timeout", "15m"), 320 "download_outputs": True, 321 } 322 new_rules.append(rule) 323 324 step_config["rules"] = new_rules 325 return step_config 326 327 reproxy = module( 328 "reproxy", 329 enabled = __use_reclient, 330 step_config = __step_config, 331 filegroups = __filegroups, 332 handlers = __handlers, 333 )