redox.py (7590B)
1 #!/usr/bin/env python 2 # 3 # Copyright (c) 2008-2019, The Tor Project, Inc. 4 # See LICENSE for licensing information. 5 # 6 # Hi! 7 # I'm redox.py, the Tor redocumentation tool! 8 # I am a horrible hack! 9 # I read the output of doxygen from stderr, and add missing DOCDOC comments 10 # to tell you where documentation should go! 11 # To use me, edit the stuff below... 12 # ...and run 'make doxygen 2>doxygen.stderr' ... 13 # ...and run ./scripts/maint/redox.py < doxygen.stderr ! 14 # I'll make a bunch of new files by adding missing DOCDOC comments to your 15 # source. Those files will have names like ./src/common/util.c.newdoc. 16 # You will want to look over the changes by hand before checking them in. 17 # 18 # So, here's your workflow: 19 # 20 # 0. Make sure you're running a bourne shell for the redirects below. 21 # 1. make doxygen 1>doxygen.stdout 2>doxygen.stderr. 22 # 2. grep Warning doxygen.stderr | grep -v 'is not documented' | less 23 # [This will tell you about all the bogus doxygen output you have] 24 # 3. python ./scripts/maint/redox.py <doxygen.stderr 25 # [This will make lots of .newdoc files with DOCDOC comments for 26 # whatever was missing documentation.] 27 # 4. Look over those .newdoc files, and see which docdoc comments you 28 # want to merge into the main file. If it's all good, just run 29 # "mv fname.c.newdoc fname.c". Otherwise, you'll need to merge 30 # the parts you like by hand. 31 32 # Future imports for Python 2.7, mandatory in 3.0 33 from __future__ import division 34 from __future__ import print_function 35 from __future__ import unicode_literals 36 37 import re 38 import sys 39 40 try: 41 xrange # Python 2 42 except NameError: 43 xrange = range # Python 3 44 45 # Which files should we ignore warning from? Mostly, these are external 46 # files that we've snarfed in from somebody else, whose C we do no intend 47 # to document for them. 48 SKIP_FILES = [ "OpenBSD_malloc_Linux.c", 49 "strlcat.c", 50 "strlcpy.c", 51 "sha256.c", 52 "sha256.h", 53 "aes.c", 54 "aes.h" ] 55 56 # What names of things never need javadoc 57 SKIP_NAME_PATTERNS = [ r'^.*_c_id$', 58 r'^.*_H_ID$' ] 59 60 # Which types of things should get DOCDOC comments added if they are 61 # missing documentation? Recognized types are in KINDS below. 62 ADD_DOCDOCS_TO_TYPES = [ 'function', 'type', 'typedef' ] 63 ADD_DOCDOCS_TO_TYPES += [ 'variable', ] 64 65 # ==================== 66 # The rest of this should not need hacking. 67 68 KINDS = [ "type", "field", "typedef", "define", "function", "variable", 69 "enumeration" ] 70 71 NODOC_LINE_RE = re.compile(r'^([^:]+):(\d+): (\w+): (.*) is not documented\.$') 72 73 THING_RE = re.compile(r'^Member ([a-zA-Z0-9_]+).*\((typedef|define|function|variable|enumeration|macro definition)\) of (file|class) ') 74 75 SKIP_NAMES = [re.compile(s) for s in SKIP_NAME_PATTERNS] 76 77 def parsething(thing): 78 """I figure out what 'foobar baz in quux quum is not documented' means, 79 and return: the name of the foobar, and the kind of the foobar. 80 """ 81 if thing.startswith("Compound "): 82 tp, name = "type", thing.split()[1] 83 else: 84 m = THING_RE.match(thing) 85 if not m: 86 print(thing, "???? Format didn't match.") 87 return None, None 88 else: 89 name, tp, parent = m.groups() 90 if parent == 'class': 91 if tp == 'variable' or tp == 'function': 92 tp = 'field' 93 94 return name, tp 95 96 def read(): 97 """I snarf doxygen stderr from stdin, and parse all the "foo has no 98 documentation messages. I return a map from filename to lists 99 of tuples of (alleged line number, name of thing, kind of thing) 100 """ 101 errs = {} 102 for line in sys.stdin: 103 m = NODOC_LINE_RE.match(line) 104 if m: 105 file, line, tp, thing = m.groups() 106 assert tp.lower() == 'warning' 107 name, kind = parsething(thing) 108 errs.setdefault(file, []).append((int(line), name, kind)) 109 110 return errs 111 112 def findline(lines, lineno, ident): 113 """Given a list of all the lines in the file (adjusted so 1-indexing works), 114 a line number that ident is allegedly on, and ident, I figure out 115 the line where ident was really declared.""" 116 lno = lineno 117 for lineno in xrange(lineno, 0, -1): 118 try: 119 if ident in lines[lineno]: 120 return lineno 121 except IndexError: 122 continue 123 124 return None 125 126 FUNC_PAT = re.compile(r"^[A-Za-z0-9_]+\(") 127 128 def hascomment(lines, lineno, kind): 129 """I return true if it looks like there's already a good comment about 130 the thing on lineno of lines of type kind. """ 131 if "*/" in lines[lineno-1]: 132 return True 133 if kind == 'function' and FUNC_PAT.match(lines[lineno]): 134 if "*/" in lines[lineno-2]: 135 return True 136 return False 137 138 def hasdocdoc(lines, lineno, kind): 139 """I return true if it looks like there's already a docdoc comment about 140 the thing on lineno of lines of type kind.""" 141 try: 142 if "DOCDOC" in lines[lineno]: 143 return True 144 except IndexError: 145 pass 146 try: 147 if "DOCDOC" in lines[lineno-1]: 148 return True 149 except IndexError: 150 pass 151 if kind == 'function' and FUNC_PAT.match(lines[lineno]): 152 if "DOCDOC" in lines[lineno-2]: 153 return True 154 return False 155 156 def checkf(fn, errs): 157 """I go through the output of read() for a single file, and build a list 158 of tuples of things that want DOCDOC comments. Each tuple has: 159 the line number where the comment goes; the kind of thing; its name. 160 """ 161 for skip in SKIP_FILES: 162 if fn.endswith(skip): 163 print("Skipping",fn) 164 return 165 166 comments = [] 167 lines = [ None ] 168 try: 169 lines.extend( open(fn, 'r').readlines() ) 170 except IOError: 171 return 172 173 for line, name, kind in errs: 174 if any(pat.match(name) for pat in SKIP_NAMES): 175 continue 176 177 if kind not in ADD_DOCDOCS_TO_TYPES: 178 continue 179 180 ln = findline(lines, line, name) 181 if ln == None: 182 print("Couldn't find the definition of %s allegedly on %s of %s"%( 183 name, line, fn)) 184 else: 185 if hasdocdoc(lines, line, kind): 186 # print "Has a DOCDOC" 187 # print fn, line, name, kind 188 # print "\t",lines[line-2], 189 # print "\t",lines[line-1], 190 # print "\t",lines[line], 191 # print "-------" 192 pass 193 else: 194 if kind == 'function' and FUNC_PAT.match(lines[ln]): 195 ln = ln - 1 196 197 comments.append((ln, kind, name)) 198 199 return comments 200 201 def applyComments(fn, entries): 202 """I apply lots of comments to the file in fn, making a new .newdoc file. 203 """ 204 N = 0 205 206 lines = [ None ] 207 try: 208 lines.extend( open(fn, 'r').readlines() ) 209 except IOError: 210 return 211 212 # Process the comments in reverse order by line number, so that 213 # the line numbers for the ones we haven't added yet remain valid 214 # until we add them. Standard trick. 215 entries.sort() 216 entries.reverse() 217 218 for ln, kind, name in entries: 219 220 lines.insert(ln, "/* DOCDOC %s */\n"%name) 221 N += 1 222 223 outf = open(fn+".newdoc", 'w') 224 for line in lines[1:]: 225 outf.write(line) 226 outf.close() 227 228 print("Added %s DOCDOCs to %s" %(N, fn)) 229 230 e = read() 231 232 for fn, errs in e.iteritems(): 233 print(repr((fn, errs))) 234 comments = checkf(fn, errs) 235 if comments: 236 applyComments(fn, comments)