setup_development.py (9705B)
1 #!/usr/bin/env python 2 3 # This Source Code Form is subject to the terms of the Mozilla Public 4 # License, v. 2.0. If a copy of the MPL was not distributed with this file, 5 # You can obtain one at http://mozilla.org/MPL/2.0/. 6 7 """ 8 Setup mozbase packages for development. 9 10 Packages may be specified as command line arguments. 11 If no arguments are given, install all packages. 12 13 See https://wiki.mozilla.org/Auto-tools/Projects/Mozbase 14 """ 15 16 import os 17 import subprocess 18 import sys 19 from optparse import OptionParser 20 from subprocess import PIPE 21 22 try: 23 from subprocess import check_call as call 24 except ImportError: 25 from subprocess import call 26 27 28 # directory containing this file 29 here = os.path.dirname(os.path.abspath(__file__)) 30 31 # all python packages 32 mozbase_packages = [ 33 i for i in os.listdir(here) if os.path.exists(os.path.join(here, i, "setup.py")) 34 ] 35 36 # testing: https://wiki.mozilla.org/Auto-tools/Projects/Mozbase#Tests 37 test_packages = ["mock"] 38 39 # documentation: https://wiki.mozilla.org/Auto-tools/Projects/Mozbase#Documentation 40 extra_packages = ["sphinx"] 41 42 43 def cycle_check(order, dependencies): 44 """ensure no cyclic dependencies""" 45 order_dict = dict([(j, i) for i, j in enumerate(order)]) 46 for package, deps in dependencies.items(): 47 index = order_dict[package] 48 for d in deps: 49 assert index > order_dict[d], "Cyclic dependencies detected" 50 51 52 def info(directory): 53 "get the package setup.py information" 54 55 assert os.path.exists(os.path.join(directory, "setup.py")) 56 57 # setup the egg info 58 try: 59 call([sys.executable, "setup.py", "egg_info"], cwd=directory, stdout=PIPE) 60 except subprocess.CalledProcessError: 61 print("Error running setup.py in %s" % directory) 62 raise 63 64 # get the .egg-info directory 65 egg_info = [entry for entry in os.listdir(directory) if entry.endswith(".egg-info")] 66 assert len(egg_info) == 1, "Expected one .egg-info directory in %s, got: %s" % ( 67 directory, 68 egg_info, 69 ) 70 egg_info = os.path.join(directory, egg_info[0]) 71 assert os.path.isdir(egg_info), "%s is not a directory" % egg_info 72 73 # read the package information 74 pkg_info = os.path.join(egg_info, "PKG-INFO") 75 info_dict = {} 76 for line in open(pkg_info).readlines(): 77 if not line or line[0].isspace(): 78 continue # XXX neglects description 79 assert ":" in line 80 key, value = [i.strip() for i in line.split(":", 1)] 81 info_dict[key] = value 82 83 return info_dict 84 85 86 def get_dependencies(directory): 87 "returns the package name and dependencies given a package directory" 88 89 # get the package metadata 90 info_dict = info(directory) 91 92 # get the .egg-info directory 93 egg_info = [ 94 entry for entry in os.listdir(directory) if entry.endswith(".egg-info") 95 ][0] 96 97 # read the dependencies 98 requires = os.path.join(directory, egg_info, "requires.txt") 99 dependencies = [] 100 if os.path.exists(requires): 101 for line in open(requires): 102 line = line.strip() 103 # in requires.txt file, a dependency is a non empty line 104 # Also lines like [device] are sections to mark optional 105 # dependencies, we don't want those sections. 106 if line and not (line.startswith("[") and line.endswith("]")): 107 dependencies.append(line) 108 109 # return the information 110 return info_dict["Name"], dependencies 111 112 113 def dependency_info(dep): 114 "return dictionary of dependency information from a dependency string" 115 retval = dict(Name=None, Type=None, Version=None) 116 for joiner in ("==", "<=", ">="): 117 if joiner in dep: 118 retval["Type"] = joiner 119 name, version = [i.strip() for i in dep.split(joiner, 1)] 120 retval["Name"] = name 121 retval["Version"] = version 122 break 123 else: 124 retval["Name"] = dep.strip() 125 return retval 126 127 128 def unroll_dependencies(dependencies): 129 """ 130 unroll a set of dependencies to a flat list 131 132 dependencies = {'packageA': set(['packageB', 'packageC', 'packageF']), 133 'packageB': set(['packageC', 'packageD', 'packageE', 'packageG']), 134 'packageC': set(['packageE']), 135 'packageE': set(['packageF', 'packageG']), 136 'packageF': set(['packageG']), 137 'packageX': set(['packageA', 'packageG'])} 138 """ 139 140 order = [] 141 142 # flatten all 143 packages = set(dependencies.keys()) 144 for deps in dependencies.values(): 145 packages.update(deps) 146 147 while len(order) != len(packages): 148 for package in packages.difference(order): 149 if set(dependencies.get(package, set())).issubset(order): 150 order.append(package) 151 break 152 else: 153 raise AssertionError("Cyclic dependencies detected") 154 155 cycle_check(order, dependencies) # sanity check 156 157 return order 158 159 160 def main(args=sys.argv[1:]): 161 # parse command line options 162 usage = "%prog [options] [package] [package] [...]" 163 parser = OptionParser(usage=usage, description=__doc__) 164 parser.add_option( 165 "-d", 166 "--dependencies", 167 dest="list_dependencies", 168 action="store_true", 169 default=False, 170 help="list dependencies for the packages", 171 ) 172 parser.add_option( 173 "--list", action="store_true", default=False, help="list what will be installed" 174 ) 175 parser.add_option( 176 "--extra", 177 "--install-extra-packages", 178 action="store_true", 179 default=False, 180 help="installs extra supporting packages as well as core mozbase ones", 181 ) 182 options, packages = parser.parse_args(args) 183 184 if not packages: 185 # install all packages 186 packages = sorted(mozbase_packages) 187 188 # ensure specified packages are in the list 189 assert set(packages).issubset(mozbase_packages), ( 190 "Packages should be in %s (You gave: %s)" % (mozbase_packages, packages) 191 ) 192 193 if options.list_dependencies: 194 # list the package dependencies 195 for package in packages: 196 print("%s: %s" % get_dependencies(os.path.join(here, package))) 197 parser.exit() 198 199 # gather dependencies 200 # TODO: version conflict checking 201 deps = {} 202 alldeps = {} 203 mapping = {} # mapping from subdir name to package name 204 # core dependencies 205 for package in packages: 206 key, value = get_dependencies(os.path.join(here, package)) 207 deps[key] = [dependency_info(dep)["Name"] for dep in value] 208 mapping[package] = key 209 210 # keep track of all dependencies for non-mozbase packages 211 for dep in value: 212 alldeps[dependency_info(dep)["Name"]] = "".join(dep.split()) 213 214 # indirect dependencies 215 flag = True 216 while flag: 217 flag = False 218 for value in deps.values(): 219 for dep in value: 220 if dep in mozbase_packages and dep not in deps: 221 key, value = get_dependencies(os.path.join(here, dep)) 222 deps[key] = [dep for dep in value] 223 224 for dep in value: 225 alldeps[dep] = "".join(dep.split()) 226 mapping[package] = key 227 flag = True 228 break 229 if flag: 230 break 231 232 # get the remaining names for the mapping 233 for package in mozbase_packages: 234 if package in mapping: 235 continue 236 key, value = get_dependencies(os.path.join(here, package)) 237 mapping[package] = key 238 239 # unroll dependencies 240 unrolled = unroll_dependencies(deps) 241 242 # make a reverse mapping: package name -> subdirectory 243 reverse_mapping = dict([(j, i) for i, j in mapping.items()]) 244 245 # we only care about dependencies in mozbase 246 unrolled = [package for package in unrolled if package in reverse_mapping] 247 248 if options.list: 249 # list what will be installed 250 for package in unrolled: 251 print(package) 252 parser.exit() 253 254 # set up the packages for development 255 for package in unrolled: 256 call( 257 [sys.executable, "setup.py", "develop", "--no-deps"], 258 cwd=os.path.join(here, reverse_mapping[package]), 259 ) 260 261 # add the directory of sys.executable to path to aid the correct 262 # `easy_install` getting called 263 # https://bugzilla.mozilla.org/show_bug.cgi?id=893878 264 os.environ["PATH"] = "%s%s%s" % ( 265 os.path.dirname(os.path.abspath(sys.executable)), 266 os.path.pathsep, 267 os.environ.get("PATH", "").strip(os.path.pathsep), 268 ) 269 270 current_file_path = os.path.abspath(__file__) 271 topobjdir = os.path.dirname(os.path.dirname(os.path.dirname(current_file_path))) 272 mach = str(os.path.join(topobjdir, "mach")) 273 274 # install non-mozbase dependencies 275 # these need to be installed separately and the --no-deps flag 276 # subsequently used due to a bug in setuptools; see 277 # https://bugzilla.mozilla.org/show_bug.cgi?id=759836 278 pypi_deps = dict([(i, j) for i, j in alldeps.items() if i not in unrolled]) 279 for package, version in pypi_deps.items(): 280 # Originally, Mozilla used easy_install here. 281 # That tool is deprecated, therefore we swich to pip. 282 call([sys.executable, mach, "python", "-m", "pip", "install", version]) 283 284 # install packages required for unit testing 285 for package in test_packages: 286 call([sys.executable, mach, "python", "-m", "pip", "install", package]) 287 288 # install extra non-mozbase packages if desired 289 if options.extra: 290 for package in extra_packages: 291 call([sys.executable, mach, "python", "-m", "pip", "install", package]) 292 293 294 if __name__ == "__main__": 295 main()