mach (10326B)
1 #!/usr/bin/env python3 2 # 3 # This Source Code Form is subject to the terms of the Mozilla Public 4 # License, v. 2.0. If a copy of the MPL was not distributed with this 5 # file, You can obtain one at http://mozilla.org/MPL/2.0/. 6 ########################################################################## 7 # 8 # This is a collection of helper tools to get stuff done in NSS. 9 # 10 11 import sys 12 import argparse 13 import fnmatch 14 import io 15 import subprocess 16 import os 17 import platform 18 import shutil 19 import tarfile 20 import tempfile 21 22 from hashlib import sha256 23 24 DEVNULL = open(os.devnull, 'wb') 25 cwd = os.path.dirname(os.path.abspath(__file__)) 26 27 def run_tests(test, cycles="standard", env={}, silent=False): 28 domsuf = os.getenv('DOMSUF', "localdomain") 29 host = os.getenv('HOST', "localhost") 30 env = env.copy() 31 env.update({ 32 "NSS_TESTS": test, 33 "NSS_CYCLES": cycles, 34 "DOMSUF": domsuf, 35 "HOST": host 36 }) 37 os_env = os.environ 38 os_env.update(env) 39 command = cwd + "/tests/all.sh" 40 stdout = stderr = DEVNULL if silent else None 41 subprocess.check_call(command, env=os_env, stdout=stdout, stderr=stderr) 42 43 44 class cfAction(argparse.Action): 45 docker_command = None 46 restorecon = None 47 48 def __call__(self, parser, args, values, option_string=None): 49 self.setDockerCommand(args) 50 51 if values: 52 files = [os.path.relpath(os.path.abspath(x), start=cwd) for x in values] 53 else: 54 files = self.modifiedFiles() 55 56 # First check if we can run docker. 57 try: 58 with open(os.devnull, "w") as f: 59 subprocess.check_call( 60 self.docker_command + ["images"], stdout=f) 61 except: 62 self.docker_command = None 63 64 if self.docker_command is None: 65 print("warning: running clang-format directly, which isn't guaranteed to be correct") 66 command = [cwd + "/automation/clang-format/run_clang_format.sh"] + files 67 repr(command) 68 subprocess.call(command) 69 return 70 71 files = [os.path.join('/home/worker/nss', x) for x in files] 72 docker_image = 'clang-format-service:latest' 73 cf_docker_folder = cwd + "/automation/clang-format" 74 75 # Build the image if necessary. 76 if self.filesChanged(cf_docker_folder): 77 self.buildImage(docker_image, cf_docker_folder) 78 79 # Check if we have the docker image. 80 try: 81 command = self.docker_command + [ 82 "image", "inspect", "clang-format-service:latest" 83 ] 84 with open(os.devnull, "w") as f: 85 subprocess.check_call(command, stdout=f) 86 except: 87 print("I have to build the docker image first.") 88 self.buildImage(docker_image, cf_docker_folder) 89 90 command = self.docker_command + [ 91 'run', '-v', cwd + ':/home/worker/nss:Z', '--rm', '-ti', docker_image 92 ] 93 # The clang format script returns 1 if something's to do. We don't 94 # care. 95 subprocess.call(command + files) 96 if self.restorecon is not None: 97 subprocess.call([self.restorecon, '-R', cwd]) 98 99 def filesChanged(self, path): 100 hash = sha256() 101 for dirname, dirnames, files in os.walk(path): 102 for file in files: 103 with open(os.path.join(dirname, file), "rb") as f: 104 hash.update(f.read()) 105 chk_file = cwd + "/.chk" 106 old_chk = "" 107 new_chk = hash.hexdigest() 108 if os.path.exists(chk_file): 109 with open(chk_file) as f: 110 old_chk = f.readline() 111 if old_chk != new_chk: 112 with open(chk_file, "w+") as f: 113 f.write(new_chk) 114 return True 115 return False 116 117 def buildImage(self, docker_image, cf_docker_folder): 118 command = self.docker_command + [ 119 "build", "-t", docker_image, cf_docker_folder 120 ] 121 subprocess.check_call(command) 122 return 123 124 def setDockerCommand(self, args): 125 if platform.system() == "Linux": 126 self.restorecon = shutil.which("restorecon") 127 dcmd = shutil.which("docker") 128 if dcmd is not None: 129 self.docker_command = [dcmd] 130 if not args.noroot: 131 self.docker_command = ["sudo"] + self.docker_command 132 else: 133 self.docker_command = None 134 135 def modifiedFiles(self): 136 files = [] 137 if os.path.exists(os.path.join(cwd, '.hg')): 138 st = subprocess.Popen(['hg', 'status', '-m', '-a'], 139 cwd=cwd, stdout=subprocess.PIPE, universal_newlines=True) 140 for line in iter(st.stdout.readline, ''): 141 files += [line[2:].rstrip()] 142 elif os.path.exists(os.path.join(cwd, '.git')): 143 st = subprocess.Popen(['git', 'status', '--porcelain'], 144 cwd=cwd, stdout=subprocess.PIPE) 145 for line in iter(st.stdout.readline, ''): 146 if line[1] == 'M' or line[1] != 'D' and \ 147 (line[0] == 'M' or line[0] == 'A' or 148 line[0] == 'C' or line[0] == 'U'): 149 files += [line[3:].rstrip()] 150 elif line[0] == 'R': 151 files += [line[line.index(' -> ', beg=4) + 4:]] 152 else: 153 print('Warning: neither mercurial nor git detected!') 154 155 def isFormatted(x): 156 return x[-2:] == '.c' or x[-3:] == '.cc' or x[-2:] == '.h' 157 return [x for x in files if isFormatted(x)] 158 159 160 class buildAction(argparse.Action): 161 162 def __call__(self, parser, args, values, option_string=None): 163 subprocess.check_call([cwd + "/build.sh"] + values) 164 165 166 class testAction(argparse.Action): 167 168 def __call__(self, parser, args, values, option_string=None): 169 run_tests(values) 170 171 172 class covAction(argparse.Action): 173 174 def runSslGtests(self, outdir): 175 env = { 176 "GTESTFILTER": "*", # Prevent parallel test runs. 177 "ASAN_OPTIONS": "coverage=1:coverage_dir=" + outdir, 178 "NSS_DEFAULT_DB_TYPE": "sql", 179 "NSS_DISABLE_UNLOAD": "1" 180 } 181 182 run_tests("ssl_gtests", env=env) 183 184 def findSanCovFile(self, outdir): 185 for file in os.listdir(outdir): 186 if fnmatch.fnmatch(file, 'ssl_gtest.*.sancov'): 187 return os.path.join(outdir, file) 188 189 return None 190 191 def __call__(self, parser, args, values, option_string=None): 192 outdir = args.outdir 193 print("Output directory: " + outdir) 194 195 print("\nBuild with coverage sanitizers...\n") 196 sancov_args = "edge,no-prune,trace-pc-guard,trace-cmp" 197 subprocess.check_call([ 198 os.path.join(cwd, "build.sh"), "-c", "--clang", "--asan", "--enable-legacy-db", 199 "--sancov=" + sancov_args 200 ]) 201 202 print("\nRun ssl_gtests to get a coverage report...") 203 self.runSslGtests(outdir) 204 print("Done.") 205 206 sancov_file = self.findSanCovFile(outdir) 207 if not sancov_file: 208 print("Couldn't find .sancov file.") 209 sys.exit(1) 210 211 symcov_file = os.path.join(outdir, "ssl_gtest.symcov") 212 out = open(symcov_file, 'wb') 213 # Don't exit immediately on error 214 symbol_retcode = subprocess.call([ 215 "sancov", 216 "-ignorelist=" + os.path.join(cwd, ".sancov-blacklist"), 217 "-symbolize", sancov_file, 218 os.path.join(cwd, "../dist/Debug/bin/ssl_gtest") 219 ], stdout=out) 220 out.close() 221 222 print("\nCopying ssl_gtests to artifacts...") 223 shutil.copyfile(os.path.join(cwd, "../dist/Debug/bin/ssl_gtest"), 224 os.path.join(outdir, "ssl_gtest")) 225 226 print("\nCoverage report: " + symcov_file) 227 if symbol_retcode > 0: 228 print("sancov failed to symbolize with return code {}".format(symbol_retcode)) 229 sys.exit(symbol_retcode) 230 231 class commandsAction(argparse.Action): 232 commands = [] 233 234 def __call__(self, parser, args, values, option_string=None): 235 for c in commandsAction.commands: 236 print(c) 237 238 def parse_arguments(): 239 parser = argparse.ArgumentParser( 240 description='NSS helper script. ' + 241 'Make sure to separate sub-command arguments with --.') 242 subparsers = parser.add_subparsers() 243 244 parser_build = subparsers.add_parser( 245 'build', help='All arguments are passed to build.sh') 246 parser_build.add_argument( 247 'build_args', nargs='*', help="build arguments", action=buildAction) 248 249 parser_cf = subparsers.add_parser( 250 'clang-format', 251 help=""" 252 Run clang-format. 253 254 By default this runs against any files that you have modified. If 255 there are no modified files, it checks everything. 256 """) 257 parser_cf.add_argument( 258 '--noroot', 259 help='On linux, suppress the use of \'sudo\' for running docker.', 260 action='store_true') 261 parser_cf.add_argument( 262 '<file/dir>', 263 nargs='*', 264 help="Specify files or directories to run clang-format on", 265 action=cfAction) 266 267 parser_test = subparsers.add_parser( 268 'tests', help='Run tests through tests/all.sh.') 269 tests = [ 270 "cipher", "lowhash", "chains", "cert", "dbtests", "tools", "fips", 271 "sdr", "crmf", "smime", "ssl", "ocsp", "merge", "pkits", "ec", 272 "gtests", "ssl_gtests", "bogo", "interop", "policy" 273 ] 274 parser_test.add_argument( 275 'test', choices=tests, help="Available tests", action=testAction) 276 277 parser_cov = subparsers.add_parser( 278 'coverage', help='Generate coverage report') 279 cov_modules = ["ssl_gtests"] 280 parser_cov.add_argument( 281 '--outdir', help='Output directory for coverage report data.', 282 default=tempfile.mkdtemp()) 283 parser_cov.add_argument( 284 'module', choices=cov_modules, help="Available coverage modules", 285 action=covAction) 286 287 parser_commands = subparsers.add_parser( 288 'mach-completion', 289 help="list commands") 290 parser_commands.add_argument( 291 'mach-completion', 292 nargs='*', 293 action=commandsAction) 294 295 commandsAction.commands = [c for c in subparsers.choices] 296 return parser.parse_args() 297 298 299 def main(): 300 parse_arguments() 301 302 303 if __name__ == '__main__': 304 main()