print_python_deps.py (7001B)
1 #!/usr/bin/env vpython3 2 # Copyright 2016 The Chromium Authors 3 # Use of this source code is governed by a BSD-style license that can be 4 # found in the LICENSE file. 5 6 """Prints all non-system dependencies for the given module. 7 8 The primary use-case for this script is to generate the list of python modules 9 required for .isolate files. 10 """ 11 12 import argparse 13 import os 14 import shlex 15 import sys 16 17 # Don't use any helper modules, or else they will end up in the results. 18 19 20 _SRC_ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir)) 21 22 23 def ComputePythonDependencies(): 24 """Gets the paths of imported non-system python modules. 25 26 A path is assumed to be a "system" import if it is outside of chromium's 27 src/. 28 29 Returns: 30 List of absolute paths. 31 """ 32 module_paths = (m.__file__ for m in sys.modules.values() 33 if m and hasattr(m, '__file__') and m.__file__ 34 and m.__name__ != '__main__') 35 36 src_paths = set() 37 for path in module_paths: 38 path = os.path.abspath(path) # paths can be relative before python 3.9. 39 if not path.startswith(_SRC_ROOT): 40 continue 41 42 if (path.endswith('.pyc') 43 or (path.endswith('c') and not os.path.splitext(path)[1])): 44 path = path[:-1] 45 src_paths.add(path) 46 47 return src_paths 48 49 50 def quote(string): 51 if string.count(' ') > 0: 52 return '"%s"' % string 53 else: 54 return string 55 56 57 def _NormalizeCommandLine(options): 58 """Returns a string that when run from SRC_ROOT replicates the command.""" 59 args = ['build/print_python_deps.py'] 60 root = os.path.relpath(options.root, _SRC_ROOT) 61 if root != '.': 62 args.extend(('--root', root)) 63 if options.output: 64 args.extend(('--output', os.path.relpath(options.output, _SRC_ROOT))) 65 if options.gn_paths: 66 args.extend(('--gn-paths',)) 67 for allowlist in sorted(options.allowlists): 68 args.extend(('--allowlist', os.path.relpath(allowlist, _SRC_ROOT))) 69 args.append(os.path.relpath(options.module, _SRC_ROOT)) 70 if os.name == 'nt': 71 return ' '.join(quote(x) for x in args).replace('\\', '/') 72 else: 73 return ' '.join(shlex.quote(x) for x in args) 74 75 76 def _FindPythonInDirectory(directory, allow_test): 77 """Returns an iterable of all non-test python files in the given directory.""" 78 for root, _dirnames, filenames in os.walk(directory): 79 for filename in filenames: 80 if filename.endswith('.py') and (allow_test 81 or not filename.endswith('_test.py')): 82 yield os.path.join(root, filename) 83 84 85 def _ImportModuleByPath(module_path): 86 """Imports a module by its source file.""" 87 # Replace the path entry for print_python_deps.py with the one for the given 88 # module. 89 sys.path[0] = os.path.dirname(module_path) 90 91 # https://docs.python.org/3/library/importlib.html#importing-a-source-file-directly 92 module_name = os.path.splitext(os.path.basename(module_path))[0] 93 import importlib.util # Python 3 only, since it's unavailable in Python 2. 94 spec = importlib.util.spec_from_file_location(module_name, module_path) 95 module = importlib.util.module_from_spec(spec) 96 sys.modules[module_name] = module 97 spec.loader.exec_module(module) 98 99 100 def main(): 101 parser = argparse.ArgumentParser( 102 description='Prints all non-system dependencies for the given module.') 103 parser.add_argument('module', 104 help='The python module to analyze.') 105 parser.add_argument('--root', default='.', 106 help='Directory to make paths relative to.') 107 parser.add_argument('--output', 108 help='Write output to a file rather than stdout.') 109 parser.add_argument('--inplace', action='store_true', 110 help='Write output to a file with the same path as the ' 111 'module, but with a .pydeps extension. Also sets the ' 112 'root to the module\'s directory.') 113 parser.add_argument('--no-header', action='store_true', 114 help='Do not write the "# Generated by" header.') 115 parser.add_argument('--gn-paths', action='store_true', 116 help='Write paths as //foo/bar/baz.py') 117 parser.add_argument('--did-relaunch', action='store_true', 118 help=argparse.SUPPRESS) 119 parser.add_argument('--allowlist', 120 default=[], 121 action='append', 122 dest='allowlists', 123 help='Recursively include all non-test python files ' 124 'within this directory. May be specified multiple times.') 125 options = parser.parse_args() 126 127 if options.inplace: 128 if options.output: 129 parser.error('Cannot use --inplace and --output at the same time!') 130 if not options.module.endswith('.py'): 131 parser.error('Input module path should end with .py suffix!') 132 options.output = options.module + 'deps' 133 options.root = os.path.dirname(options.module) 134 135 modules = [options.module] 136 if os.path.isdir(options.module): 137 modules = list(_FindPythonInDirectory(options.module, allow_test=True)) 138 if not modules: 139 parser.error('Input directory does not contain any python files!') 140 141 is_vpython = 'vpython' in sys.executable 142 if not is_vpython: 143 # Prevent infinite relaunch if something goes awry. 144 assert not options.did_relaunch 145 # Re-launch using vpython will cause us to pick up modules specified in 146 # //.vpython, but does not cause it to pick up modules defined inline via 147 # [VPYTHON:BEGIN] ... [VPYTHON:END] comments. 148 # TODO(agrieve): Add support for this if the need ever arises. 149 os.execvp('vpython3', ['vpython3'] + sys.argv + ['--did-relaunch']) 150 151 # Work-around for protobuf library not being loadable via importlib 152 # This is needed due to compile_resources.py. 153 import importlib._bootstrap_external 154 importlib._bootstrap_external._NamespacePath.sort = lambda self, **_: 0 155 156 paths_set = set() 157 try: 158 for module in modules: 159 _ImportModuleByPath(module) 160 paths_set.update(ComputePythonDependencies()) 161 except Exception: 162 # Output extra diagnostics when loading the script fails. 163 sys.stderr.write('Error running print_python_deps.py.\n') 164 sys.stderr.write('is_vpython={}\n'.format(is_vpython)) 165 sys.stderr.write('did_relanuch={}\n'.format(options.did_relaunch)) 166 sys.stderr.write('python={}\n'.format(sys.executable)) 167 raise 168 169 for path in options.allowlists: 170 paths_set.update( 171 os.path.abspath(p) 172 for p in _FindPythonInDirectory(path, allow_test=False)) 173 174 paths = [os.path.relpath(p, options.root) for p in paths_set] 175 176 normalized_cmdline = _NormalizeCommandLine(options) 177 out = open(options.output, 'w', newline='') if options.output else sys.stdout 178 with out: 179 if not options.no_header: 180 out.write('# Generated by running:\n') 181 out.write('# %s\n' % normalized_cmdline) 182 prefix = '//' if options.gn_paths else '' 183 for path in sorted(paths): 184 out.write(prefix + path.replace('\\', '/') + '\n') 185 186 187 if __name__ == '__main__': 188 sys.exit(main())