tor-browser

The Tor Browser
git clone https://git.dasho.dev/tor-browser.git
Log | Files | Refs | README | LICENSE

check_gn_headers.py (9628B)

      1 #!/usr/bin/env python3
      2 # Copyright 2017 The Chromium Authors
      3 # Use of this source code is governed by a BSD-style license that can be
      4 # found in the LICENSE file.
      5 
      6 """Find header files missing in GN.
      7 
      8 This script gets all the header files from ninja_deps, which is from the true
      9 dependency generated by the compiler, and report if they don't exist in GN.
     10 """
     11 
     12 import argparse
     13 import json
     14 import os
     15 import re
     16 import shutil
     17 import subprocess
     18 import sys
     19 import tempfile
     20 from multiprocessing import Process, Queue
     21 
     22 SRC_DIR = os.path.abspath(
     23    os.path.join(os.path.abspath(os.path.dirname(__file__)), os.path.pardir))
     24 DEPOT_TOOLS_DIR = os.path.join(SRC_DIR, 'third_party', 'depot_tools')
     25 
     26 SISO_PATH = os.path.join(SRC_DIR, 'third_party', 'siso', 'cipd', 'siso')
     27 NINJA_PATH = os.path.join(SRC_DIR, 'third_party', 'ninja', 'ninja')
     28 
     29 
     30 def IsSisoUsed(out_dir):
     31  return os.path.exists(os.path.join(out_dir, ".siso_deps"))
     32 
     33 
     34 def GetHeadersFromNinja(out_dir, skip_obj, q):
     35  """Return all the header files from ninja_deps"""
     36 
     37  def NinjaSource():
     38    if IsSisoUsed(out_dir):
     39      cmd = [
     40          SISO_PATH,
     41          'query',
     42          'deps',
     43          '-C',
     44          out_dir,
     45      ]
     46    else:
     47      cmd = [NINJA_PATH, '-C', out_dir, '-t', 'deps']
     48    # A negative bufsize means to use the system default, which usually
     49    # means fully buffered.
     50    popen = subprocess.Popen(cmd, stdout=subprocess.PIPE, bufsize=-1)
     51    for line in iter(popen.stdout.readline, b''):
     52      yield line.rstrip().decode('utf-8')
     53 
     54    popen.stdout.close()
     55    return_code = popen.wait()
     56    if return_code:
     57      raise subprocess.CalledProcessError(return_code, cmd)
     58 
     59  ans, err = set(), None
     60  try:
     61    ans = ParseNinjaDepsOutput(NinjaSource(), out_dir, skip_obj)
     62  except Exception as e:
     63    err = str(e)
     64  q.put((ans, err))
     65 
     66 
     67 def ParseNinjaDepsOutput(ninja_out, out_dir, skip_obj):
     68  """Parse ninja output and get the header files"""
     69  all_headers = {}
     70 
     71  # Ninja always uses "/", even on Windows.
     72  prefix = '../../'
     73 
     74  is_valid = False
     75  obj_file = ''
     76  for line in ninja_out:
     77    if line.startswith('    '):
     78      if not is_valid:
     79        continue
     80      if line.endswith('.h') or line.endswith('.hh'):
     81        f = line.strip()
     82        if f.startswith(prefix):
     83          f = f[6:]  # Remove the '../../' prefix
     84          # build/ only contains build-specific files like build_config.h
     85          # and buildflag.h, and system header files, so they should be
     86          # skipped.
     87          if f.startswith(out_dir) or f.startswith('out'):
     88            continue
     89          if not f.startswith('build'):
     90            all_headers.setdefault(f, [])
     91            if not skip_obj:
     92              all_headers[f].append(obj_file)
     93    else:
     94      is_valid = line.endswith('(VALID)')
     95      obj_file = line.split(':')[0]
     96 
     97  return all_headers
     98 
     99 
    100 def GetHeadersFromGN(out_dir, q):
    101  """Return all the header files from GN"""
    102 
    103  tmp = None
    104  ans, err = set(), None
    105  try:
    106    # Argument |dir| is needed to make sure it's on the same drive on Windows.
    107    # dir='' means dir='.', but doesn't introduce an unneeded prefix.
    108    tmp = tempfile.mkdtemp(dir='')
    109    shutil.copy2(os.path.join(out_dir, 'args.gn'),
    110                 os.path.join(tmp, 'args.gn'))
    111    # Do "gn gen" in a temp dir to prevent dirtying |out_dir|.
    112    gn_exe = 'gn.bat' if sys.platform == 'win32' else 'gn'
    113    subprocess.check_call([
    114        os.path.join(DEPOT_TOOLS_DIR, gn_exe), 'gen', tmp, '--ide=json', '-q'])
    115    gn_json = json.load(open(os.path.join(tmp, 'project.json')))
    116    ans = ParseGNProjectJSON(gn_json, out_dir, tmp)
    117  except Exception as e:
    118    err = str(e)
    119  finally:
    120    if tmp:
    121      shutil.rmtree(tmp)
    122  q.put((ans, err))
    123 
    124 
    125 def ParseGNProjectJSON(gn, out_dir, tmp_out):
    126  """Parse GN output and get the header files"""
    127  all_headers = set()
    128 
    129  for _target, properties in gn['targets'].items():
    130    sources = properties.get('sources', [])
    131    public = properties.get('public', [])
    132    # Exclude '"public": "*"'.
    133    if type(public) is list:
    134      sources += public
    135    for f in sources:
    136      if f.endswith('.h') or f.endswith('.hh'):
    137        if f.startswith('//'):
    138          f = f[2:]  # Strip the '//' prefix.
    139          if f.startswith(tmp_out):
    140            f = out_dir + f[len(tmp_out):]
    141          all_headers.add(f)
    142 
    143  return all_headers
    144 
    145 
    146 def GetDepsPrefixes(q):
    147  """Return all the folders controlled by DEPS file"""
    148  prefixes, err = set(), None
    149  try:
    150    gclient_exe = 'gclient.bat' if sys.platform == 'win32' else 'gclient'
    151    gclient_out = subprocess.check_output([
    152        os.path.join(DEPOT_TOOLS_DIR,
    153                     gclient_exe), 'recurse', '--no-progress', '-j1', 'python3',
    154        '-c', 'import os;print(os.environ["GCLIENT_DEP_PATH"])'
    155    ],
    156                                          universal_newlines=True)
    157    for i in gclient_out.split('\n'):
    158      if i.startswith('src/'):
    159        i = i[4:]
    160        prefixes.add(i)
    161  except Exception as e:
    162    err = str(e)
    163  q.put((prefixes, err))
    164 
    165 
    166 def IsBuildClean(out_dir):
    167  if IsSisoUsed(out_dir):
    168    cmd = [SISO_PATH, 'ninja', '-C', out_dir, '-n']
    169  else:
    170    cmd = [NINJA_PATH, '-C', out_dir, '-n']
    171  try:
    172    out = subprocess.check_output(cmd)
    173    return b'no work to do.' in out
    174  except Exception as e:
    175    print(e)
    176    return False
    177 
    178 
    179 def ParseAllowlist(allowlist):
    180  out = set()
    181  for line in allowlist.split('\n'):
    182    line = re.sub(r'#.*', '', line).strip()
    183    if line:
    184      out.add(line)
    185  return out
    186 
    187 
    188 def FilterOutDepsedRepo(files, deps):
    189  return {f for f in files if not any(f.startswith(d) for d in deps)}
    190 
    191 
    192 def GetNonExistingFiles(lst):
    193  out = set()
    194  for f in lst:
    195    if not os.path.isfile(f):
    196      out.add(f)
    197  return out
    198 
    199 
    200 def main():
    201 
    202  def DumpJson(data):
    203    if args.json:
    204      with open(args.json, 'w') as f:
    205        json.dump(data, f)
    206 
    207  def PrintError(msg):
    208    DumpJson([])
    209    parser.error(msg)
    210 
    211  parser = argparse.ArgumentParser(description='''
    212      NOTE: Use ninja to build all targets in OUT_DIR before running
    213      this script.''')
    214  parser.add_argument('--out-dir', metavar='OUT_DIR', default='out/Release',
    215                      help='output directory of the build')
    216  parser.add_argument('--json',
    217                      help='JSON output filename for missing headers')
    218  parser.add_argument('--allowlist', help='file containing allowlist')
    219  parser.add_argument('--skip-dirty-check',
    220                      action='store_true',
    221                      help='skip checking whether the build is dirty')
    222  parser.add_argument('--verbose', action='store_true',
    223                      help='print more diagnostic info')
    224 
    225  args, _extras = parser.parse_known_args()
    226 
    227  if not os.path.isdir(args.out_dir):
    228    parser.error('OUT_DIR "%s" does not exist.' % args.out_dir)
    229 
    230  if not args.skip_dirty_check and not IsBuildClean(args.out_dir):
    231    dirty_msg = 'OUT_DIR looks dirty. You need to build all there.'
    232    if args.json:
    233      # Assume running on the bots. Silently skip this step.
    234      # This is possible because "analyze" step can be wrong due to
    235      # underspecified header files. See crbug.com/725877
    236      print(dirty_msg)
    237      DumpJson([])
    238      return 0
    239    else:
    240      # Assume running interactively.
    241      parser.error(dirty_msg)
    242 
    243  d_q = Queue()
    244  d_p = Process(target=GetHeadersFromNinja, args=(args.out_dir, True, d_q,))
    245  d_p.start()
    246 
    247  gn_q = Queue()
    248  gn_p = Process(target=GetHeadersFromGN, args=(args.out_dir, gn_q,))
    249  gn_p.start()
    250 
    251  deps_q = Queue()
    252  deps_p = Process(target=GetDepsPrefixes, args=(deps_q,))
    253  deps_p.start()
    254 
    255  d, d_err = d_q.get()
    256  gn, gn_err = gn_q.get()
    257  missing = set(d.keys()) - gn
    258  nonexisting = GetNonExistingFiles(gn)
    259 
    260  deps, deps_err = deps_q.get()
    261  missing = FilterOutDepsedRepo(missing, deps)
    262  nonexisting = FilterOutDepsedRepo(nonexisting, deps)
    263 
    264  d_p.join()
    265  gn_p.join()
    266  deps_p.join()
    267 
    268  if d_err:
    269    PrintError(d_err)
    270  if gn_err:
    271    PrintError(gn_err)
    272  if deps_err:
    273    PrintError(deps_err)
    274  if len(GetNonExistingFiles(d)) > 0:
    275    print('Non-existing files in ninja deps:', GetNonExistingFiles(d))
    276    PrintError('Found non-existing files in ninja deps. You should ' +
    277               'build all in OUT_DIR.')
    278  if len(d) == 0:
    279    PrintError('OUT_DIR looks empty. You should build all there.')
    280  if any((('/gen/' in i) for i in nonexisting)):
    281    PrintError('OUT_DIR looks wrong. You should build all there.')
    282 
    283  if args.allowlist:
    284    allowlist = ParseAllowlist(open(args.allowlist).read())
    285    missing -= allowlist
    286    nonexisting -= allowlist
    287 
    288  missing = sorted(missing)
    289  nonexisting = sorted(nonexisting)
    290 
    291  DumpJson(sorted(missing + nonexisting))
    292 
    293  if len(missing) == 0 and len(nonexisting) == 0:
    294    return 0
    295 
    296  if len(missing) > 0:
    297    print('\nThe following files should be included in gn files:')
    298    for i in missing:
    299      print(i)
    300 
    301  if len(nonexisting) > 0:
    302    print('\nThe following non-existing files should be removed from gn files:')
    303    for i in nonexisting:
    304      print(i)
    305 
    306  if args.verbose:
    307    # Only get detailed obj dependency here since it is slower.
    308    GetHeadersFromNinja(args.out_dir, False, d_q)
    309    d, d_err = d_q.get()
    310    print('\nDetailed dependency info:')
    311    for f in missing:
    312      print(f)
    313      for cc in d[f]:
    314        print('  ', cc)
    315 
    316    print('\nMissing headers sorted by number of affected object files:')
    317    count = {k: len(v) for (k, v) in d.items()}
    318    for f in sorted(count, key=count.get, reverse=True):
    319      if f in missing:
    320        print(count[f], f)
    321 
    322  if args.json:
    323    # Assume running on the bots. Temporarily return 0 before
    324    # https://crbug.com/937847 is fixed.
    325    return 0
    326  return 1
    327 
    328 
    329 if __name__ == '__main__':
    330  sys.exit(main())