lint.py (7097B)
1 #! /usr/bin/env python3 2 import os 3 import subprocess 4 import re 5 import sys 6 import fnmatch 7 8 from collections import defaultdict 9 from optparse import OptionParser 10 11 lint_root = os.path.dirname(os.path.abspath(__file__)) 12 repo_root = os.path.dirname(os.path.dirname(lint_root)) 13 14 15 def git(command, *args): 16 args = list(args) 17 proc_kwargs = {"cwd": repo_root} 18 command_line = ["git", command] + args 19 20 try: 21 return subprocess.check_output(command_line, universal_newlines=True, **proc_kwargs) 22 except subprocess.CalledProcessError: 23 raise 24 25 26 def iter_files(flag=False, floder=""): 27 if floder != "" and floder != None: 28 os.chdir(repo_root) 29 for pardir, subdir, files in os.walk(floder): 30 for item in subdir + files: 31 if not os.path.isdir(os.path.join(pardir, item)): 32 yield os.path.join(pardir, item) 33 os.chdir(lint_root) 34 else: 35 if not flag: 36 os.chdir(repo_root) 37 for pardir, subdir, files in os.walk(repo_root): 38 for item in subdir + files: 39 if not os.path.isdir(os.path.join(pardir, item)): 40 yield os.path.join(pardir, item).split(repo_root + "/")[1] 41 os.chdir(lint_root) 42 else: 43 for item in git("diff", "--name-status", "HEAD~1").strip().split("\n"): 44 status = item.split("\t") 45 if status[0].strip() != "D": 46 yield status[1] 47 48 49 def check_filename_space(path): 50 bname = os.path.basename(path) 51 if re.compile(" ").search(bname): 52 return [("FILENAME WHITESPACE", "Filename of %s contains white space" % path, None)] 53 return [] 54 55 56 def check_permission(path): 57 bname = os.path.basename(path) 58 if not re.compile('\.py$|\.sh$').search(bname): 59 if os.access(os.path.join(repo_root, path), os.X_OK): 60 return [("UNNECESSARY EXECUTABLE PERMISSION", "%s contains unnecessary executable permission" % path, None)] 61 return [] 62 63 64 def parse_allowlist_file(filename): 65 data = defaultdict(lambda:defaultdict(set)) 66 67 with open(filename) as f: 68 for line in f: 69 line = line.strip() 70 if not line or line.startswith("#"): 71 continue 72 parts = [item.strip() for item in line.split(":")] 73 if len(parts) == 2: 74 parts.append(None) 75 else: 76 parts[-1] = int(parts[-1]) 77 78 error_type, file_match, line_number = parts 79 data[file_match][error_type].add(line_number) 80 81 def inner(path, errors): 82 allowlisted = [False for item in range(len(errors))] 83 84 for file_match, allowlist_errors in data.items(): 85 if fnmatch.fnmatch(path, file_match): 86 for i, (error_type, msg, line) in enumerate(errors): 87 if "*" in allowlist_errors: 88 allowlisted[i] = True 89 elif error_type in allowlist_errors: 90 allowed_lines = allowlist_errors[error_type] 91 if None in allowed_lines or line in allowed_lines: 92 allowlisted[i] = True 93 94 return [item for i, item in enumerate(errors) if not allowlisted[i]] 95 return inner 96 97 98 _allowlist_fn = None 99 def allowlist_errors(path, errors): 100 global _allowlist_fn 101 102 if _allowlist_fn is None: 103 _allowlist_fn = parse_allowlist_file(os.path.join(lint_root, "lint.allowlist")) 104 return _allowlist_fn(path, errors) 105 106 107 class Regexp(object): 108 pattern = None 109 file_extensions = None 110 error = None 111 _re = None 112 113 def __init__(self): 114 self._re = re.compile(self.pattern) 115 116 def applies(self, path): 117 return (self.file_extensions is None or 118 os.path.splitext(path)[1] in self.file_extensions) 119 120 def search(self, line): 121 return self._re.search(line) 122 123 124 class TrailingWhitespaceRegexp(Regexp): 125 pattern = " $" 126 error = "TRAILING WHITESPACE" 127 128 129 class TabsRegexp(Regexp): 130 pattern = "^\t" 131 error = "INDENT TABS" 132 133 134 class CRRegexp(Regexp): 135 pattern = "\r$" 136 error = "CR AT EOL" 137 138 regexps = [item() for item in 139 [TrailingWhitespaceRegexp, 140 TabsRegexp, 141 CRRegexp]] 142 143 144 def check_regexp_line(path, f): 145 errors = [] 146 147 applicable_regexps = [regexp for regexp in regexps if regexp.applies(path)] 148 149 try: 150 for i, line in enumerate(f): 151 for regexp in applicable_regexps: 152 if regexp.search(line): 153 errors.append((regexp.error, "%s line %i" % (path, i+1), i+1)) 154 except UnicodeDecodeError as e: 155 return [("INVALID UNICODE", "File %s contains non-UTF-8 Unicode characters" % path, None)] 156 157 return errors 158 159 160 def output_errors(errors): 161 for error_type, error, line_number in errors: 162 print("%s: %s" % (error_type, error)) 163 164 165 def output_error_count(error_count): 166 if not error_count: 167 return 168 169 by_type = " ".join("%s: %d" % item for item in error_count.items()) 170 count = sum(error_count.values()) 171 if count == 1: 172 print("There was 1 error (%s)" % (by_type,)) 173 else: 174 print("There were %d errors (%s)" % (count, by_type)) 175 176 177 def main(): 178 global repo_root 179 error_count = defaultdict(int) 180 181 parser = OptionParser() 182 parser.add_option('-p', '--pull', dest="pull_request", action='store_true', default=False) 183 parser.add_option("-d", '--dir', dest="dir", help="specify the checking dir, e.g. tools") 184 parser.add_option("-r", '--repo', dest="repo", help="specify the repo, e.g. WebGL") 185 options, args = parser.parse_args() 186 if options.pull_request == True: 187 options.pull_request = "WebGL" 188 repo_root = repo_root.replace("WebGL/sdk/tests", options.pull_request) 189 if options.repo == "" or options.repo == None: 190 options.repo = "WebGL/sdk/tests" 191 repo_root = repo_root.replace("WebGL/sdk/tests", options.repo) 192 193 def run_lint(path, fn, *args): 194 errors = allowlist_errors(path, fn(path, *args)) 195 output_errors(errors) 196 for error_type, error, line in errors: 197 error_count[error_type] += 1 198 199 for path in iter_files(options.pull_request, options.dir): 200 abs_path = os.path.join(repo_root, path) 201 if not os.path.exists(abs_path): 202 continue 203 for path_fn in file_path_lints: 204 run_lint(path, path_fn) 205 for state_fn in file_state_lints: 206 run_lint(path, state_fn) 207 208 if not os.path.isdir(abs_path): 209 if re.compile('\.html$|\.htm$|\.xhtml$|\.xhtm$|\.frag$|\.vert$|\.js$').search(abs_path): 210 with open(abs_path) as f: 211 for file_fn in file_content_lints: 212 run_lint(path, file_fn, f) 213 f.seek(0) 214 215 output_error_count(error_count) 216 return sum(error_count.values()) 217 218 file_path_lints = [check_filename_space] 219 file_content_lints = [check_regexp_line] 220 file_state_lints = [check_permission] 221 222 if __name__ == "__main__": 223 error_count = main() 224 if error_count > 0: 225 sys.exit(1)