make-source-package.py (11814B)
1 #!/usr/bin/env -S python3 -B 2 # This Source Code Form is subject to the terms of the Mozilla Public 3 # License, v. 2.0. If a copy of the MPL was not distributed with this file, 4 # You can obtain one at http://mozilla.org/MPL/2.0/. 5 6 import argparse 7 import enum 8 import logging 9 import os 10 import shutil 11 import subprocess 12 import sys 13 import tarfile 14 from pathlib import Path 15 16 logging.basicConfig(format="%(levelname)s: %(message)s", level=logging.INFO) 17 18 19 def find_command(names): 20 """Search for command in `names`, and returns the first one that exists.""" 21 22 for name in names: 23 path = shutil.which(name) 24 if path is not None: 25 return name 26 27 return None 28 29 30 def assert_command(env_var, name): 31 """Assert that the command is not empty 32 The command name comes from either environment variable or find_command. 33 """ 34 if not name: 35 logging.error(f"{env_var} command not found") 36 sys.exit(1) 37 38 39 def parse_version(topsrc_dir): 40 """Parse milestone.txt and return the entire milestone and major version.""" 41 milestone_file = topsrc_dir / "config" / "milestone.txt" 42 if not milestone_file.is_file(): 43 return ("", "", "") 44 45 with milestone_file.open("r") as f: 46 for line in f: 47 line = line.strip() 48 if not line: 49 continue 50 if line.startswith("#"): 51 continue 52 53 v = line.split(".") 54 return tuple((v + ["", ""])[:3]) 55 56 return ("", "", "") 57 58 59 tmp_dir = Path("/tmp") 60 61 rsync = os.environ.get("RSYNC", find_command(["rsync"])) 62 assert_command("RSYNC", rsync) 63 64 src_dir = Path(os.environ.get("SRC_DIR", Path(__file__).parent.absolute())) 65 mozjs_name = os.environ.get("MOZJS_NAME", "mozjs") 66 staging_dir = Path(os.environ.get("STAGING", tmp_dir / "mozjs-src-pkg")) 67 dist_dir = Path(os.environ.get("DIST", tmp_dir)) 68 topsrc_dir = src_dir.parent.parent.absolute() 69 70 parsed_major_version, parsed_minor_version, parsed_patch_version = parse_version( 71 topsrc_dir 72 ) 73 74 major_version = os.environ.get("MOZJS_MAJOR_VERSION", parsed_major_version) 75 minor_version = os.environ.get("MOZJS_MINOR_VERSION", parsed_minor_version) 76 patch_version = os.environ.get("MOZJS_PATCH_VERSION", parsed_patch_version) 77 alpha = os.environ.get("MOZJS_ALPHA", "") 78 79 version = "{}-{}.{}.{}".format( 80 mozjs_name, major_version, minor_version, patch_version or alpha or "0" 81 ) 82 target_dir = staging_dir / version 83 package_name = f"{version}.tar.xz" 84 package_file = dist_dir / package_name 85 86 # Given there might be some external program that reads the following output, 87 # use raw `print`, instead of logging. 88 print("Environment:") 89 print(f" RSYNC = {rsync}") 90 print(f" STAGING = {staging_dir}") 91 print(f" DIST = {dist_dir}") 92 print(f" SRC_DIR = {src_dir}") 93 print(f" MOZJS_NAME = {mozjs_name}") 94 print(f" MOZJS_MAJOR_VERSION = {major_version}") 95 print(f" MOZJS_MINOR_VERSION = {minor_version}") 96 print(f" MOZJS_PATCH_VERSION = {patch_version}") 97 print(f" MOZJS_ALPHA = {alpha}") 98 print("") 99 100 rsync_filter_list = """ 101 # Top-level config and build files 102 103 + /client.mk 104 + /configure.py 105 + /LICENSE 106 + /mach 107 + /Makefile.in 108 + /moz.build 109 + /moz.configure 110 + /test.mozbuild 111 + /.babel-eslint.rc.js 112 + /.eslintrc*.js 113 + /.flake8 114 + /.gitignore 115 + /.hgignore 116 + /.lldbinit 117 + /.prettierignore 118 + /.prettierrc 119 + /.ycm_extra_conf.py 120 121 # Additional libraries (optionally) used by SpiderMonkey 122 123 + /mfbt/** 124 + /nsprpub/** 125 126 + /intl/bidi/** 127 128 - /intl/icu/source/data 129 - /intl/icu/source/test 130 - /intl/icu/source/tools 131 + /intl/icu/** 132 133 + /intl/icu_capi/** 134 + /intl/icu_segmenter_data/** 135 136 - /intl/components/gtest 137 + /intl/components/** 138 139 + /memory/replace/dmd/dmd.py 140 + /memory/build/** 141 + /memory/moz.build 142 + /memory/mozalloc/** 143 144 + /modules/fdlibm/** 145 + /modules/zlib/** 146 147 + /mozglue/baseprofiler/** 148 + /mozglue/build/** 149 + /mozglue/interposers/** 150 + /mozglue/misc/** 151 + /mozglue/moz.build 152 + /mozglue/static/** 153 154 + /tools/rb/fix_stacks.py 155 + /tools/fuzzing/moz.build 156 + /tools/fuzzing/interface/** 157 + /tools/fuzzing/registry/** 158 + /tools/fuzzing/libfuzzer/** 159 + /tools/fuzzing/*.mozbuild 160 161 # Build system and dependencies 162 163 + /Cargo.lock 164 + /build/** 165 + /config/** 166 + /python/** 167 168 + /.cargo/config.toml.in 169 170 + /third_party/fmt/** 171 + /third_party/function2/** 172 - /third_party/python/gyp 173 + /third_party/python/** 174 + /third_party/rust/** 175 + /third_party/gemmology/** 176 + /third_party/xsimd/** 177 + /layout/tools/reftest/reftest/** 178 179 + /testing/*.py 180 + /testing/manifest/** 181 + /testing/moz.build 182 + /testing/mozbase/** 183 + /testing/mozharness/** 184 + /testing/performance/** 185 + /testing/test/** 186 + /testing/web-platform/*.ini 187 + /testing/web-platform/*.py 188 + /testing/web-platform/meta/streams/** 189 + /testing/web-platform/mozilla/** 190 + /testing/web-platform/tests/resources/** 191 + /testing/web-platform/tests/streams/** 192 + /testing/web-platform/tests/tools/** 193 194 + /toolkit/crashreporter/tools/symbolstore.py 195 + /toolkit/mozapps/installer/package-name.mk 196 197 + /xpcom/geckoprocesstypes_generator/** 198 199 # List of prefs. 200 + /modules/libpref/init/StaticPrefList.yaml 201 202 # SpiderMonkey itself 203 204 + /js/src/** 205 + /js/app.mozbuild 206 + /js/*.configure 207 + /js/examples/** 208 + /js/public/** 209 210 + */ 211 - /** 212 """ 213 214 INSTALL_CONTENT = """\ 215 Documentation for SpiderMonkey is available at: 216 217 https://spidermonkey.dev/ 218 219 In particular, it points to build documentation at 220 221 https://firefox-source-docs.mozilla.org/js/build.html 222 223 Note that the libraries produced by the build system include symbols, 224 causing the binaries to be extremely large. It is highly suggested that `strip` 225 be run over the binaries before deploying them. 226 227 Building with default options may be performed as follows: 228 229 ./mach build 230 231 This will produce a debug build (much more suitable for developing against the 232 SpiderMonkey JSAPI). To produce an optimized build: 233 234 export MOZCONFIG=$(pwd)/mozconfig.opt 235 ./mach build 236 237 You may edit the mozconfig and mozconfig.opt files to configure your own build 238 appropriately. 239 """ 240 241 MOZCONFIG_DEBUG_CONTENT = """\ 242 # Much slower when running, but adds assertions that are much better for 243 # developing against the JSAPI. 244 ac_add_options --enable-debug 245 246 # Much faster when running, worse for debugging. 247 ac_add_options --enable-optimize 248 249 mk_add_options MOZ_OBJDIR=obj-debug 250 """ 251 252 MOZCONFIG_OPT_CONTENT = """\ 253 # Much faster when running, but very error-prone to develop against because 254 # this will skip all the assertions critical to using the JSAPI properly. 255 ac_add_options --disable-debug 256 257 # Much faster when running, worse for debugging. 258 ac_add_options --enable-optimize 259 260 mk_add_options MOZ_OBJDIR=obj-opt 261 """ 262 263 README_CONTENT = f"""\ 264 This directory contains SpiderMonkey {major_version}. 265 266 This release is based on a revision of Mozilla {major_version}: 267 https://hg.mozilla.org/releases/ 268 The changes in the patches/ directory were applied. 269 270 See https://spidermonkey.dev/ for documentation, examples, and release notes. 271 """ 272 273 274 def is_mozjs_cargo_member(line): 275 """Checks if the line in workspace.members is mozjs-related""" 276 277 return '"js/' in line 278 279 280 def is_mozjs_crates_io_local_patch(line): 281 """Checks if the line in patch.crates-io is mozjs-related""" 282 283 return any( 284 f'path = "{p}' in line for p in ("js", "build", "third_party/rust", "intl") 285 ) 286 287 288 def clean(): 289 """Remove temporary directory and package file.""" 290 logging.info(f"Cleaning {package_file} and {target_dir} ...") 291 if package_file.exists(): 292 package_file.unlink() 293 if target_dir.exists(): 294 shutil.rmtree(str(target_dir)) 295 296 297 def assert_clean(): 298 """Assert that target directory does not contain generated files.""" 299 makefile_file = target_dir / "js" / "src" / "Makefile" 300 if makefile_file.exists(): 301 logging.error("found js/src/Makefile. Please clean before packaging.") 302 sys.exit(1) 303 304 305 def create_target_dir(): 306 if target_dir.exists(): 307 logging.warning(f"dist tree {target_dir} already exists!") 308 else: 309 target_dir.mkdir(parents=True) 310 311 312 def sync_files(): 313 # Output of the command should directly go to stdout/stderr. 314 p = subprocess.Popen( 315 [ 316 str(rsync), 317 "--delete-excluded", 318 "--prune-empty-dirs", 319 "--quiet", 320 "--recursive", 321 f"{topsrc_dir}/", 322 f"{target_dir}/", 323 "--filter=. -", 324 ], 325 stdin=subprocess.PIPE, 326 ) 327 328 p.communicate(rsync_filter_list.encode()) 329 330 if p.returncode != 0: 331 sys.exit(p.returncode) 332 333 334 def copy_cargo_toml(): 335 cargo_toml_file = topsrc_dir / "Cargo.toml" 336 target_cargo_toml_file = target_dir / "Cargo.toml" 337 338 with cargo_toml_file.open("r") as f: 339 340 class State(enum.Enum): 341 BEFORE_MEMBER = 1 342 INSIDE_MEMBER = 2 343 AFTER_MEMBER = 3 344 INSIDE_PATCH = 4 345 AFTER_PATCH = 5 346 347 content = "" 348 state = State.BEFORE_MEMBER 349 for line in f: 350 if state == State.BEFORE_MEMBER: 351 if line.strip() == "members = [": 352 state = State.INSIDE_MEMBER 353 elif state == State.INSIDE_MEMBER: 354 if line.strip() == "]": 355 state = State.AFTER_MEMBER 356 elif not is_mozjs_cargo_member(line): 357 continue 358 elif state == State.AFTER_MEMBER: 359 if line.strip() == "[patch.crates-io]": 360 state = State.INSIDE_PATCH 361 elif state == State.INSIDE_PATCH: 362 if line.startswith("["): 363 state = State.AFTER_PATCH 364 if "path = " in line: 365 if not is_mozjs_crates_io_local_patch(line): 366 continue 367 368 content += line 369 370 with target_cargo_toml_file.open("w") as f: 371 f.write(content) 372 373 374 def copy_file(filename, content): 375 """Copy an existing file from the staging area, or create a new file 376 with the given contents if it does not exist.""" 377 378 staging_file = staging_dir / filename 379 target_file = target_dir / filename 380 381 if staging_file.exists(): 382 shutil.copy2(str(staging_file), str(target_file)) 383 else: 384 with target_file.open("w") as f: 385 f.write(content) 386 387 388 def copy_patches(): 389 """Copy patches dir, if it exists.""" 390 391 staging_patches_dir = staging_dir / "patches" 392 top_patches_dir = topsrc_dir / "patches" 393 target_patches_dir = target_dir / "patches" 394 395 if staging_patches_dir.is_dir(): 396 shutil.copytree(str(staging_patches_dir), str(target_patches_dir)) 397 elif top_patches_dir.is_dir(): 398 shutil.copytree(str(top_patches_dir), str(target_patches_dir)) 399 400 401 def remove_python_cache(): 402 """Remove *.pyc and *.pyo files if any.""" 403 for f in target_dir.glob("**/*.pyc"): 404 f.unlink() 405 for f in target_dir.glob("**/*.pyo"): 406 f.unlink() 407 408 409 def stage(): 410 """Stage source tarball content.""" 411 logging.info(f"Staging source tarball in {target_dir}...") 412 413 create_target_dir() 414 sync_files() 415 copy_cargo_toml() 416 copy_file("INSTALL", INSTALL_CONTENT) 417 copy_file("README", README_CONTENT) 418 copy_file("mozconfig", MOZCONFIG_DEBUG_CONTENT) 419 copy_file("mozconfig.opt", MOZCONFIG_OPT_CONTENT) 420 copy_patches() 421 remove_python_cache() 422 423 424 def create_tar(): 425 """Roll the tarball.""" 426 427 logging.info(f"Packaging source tarball at {package_file}...") 428 429 with tarfile.open(str(package_file), "w:xz") as stream: 430 stream.add(staging_dir / version, version) 431 432 433 def build(): 434 assert_clean() 435 stage() 436 create_tar() 437 438 439 parser = argparse.ArgumentParser(description="Make SpiderMonkey source package") 440 subparsers = parser.add_subparsers(dest="COMMAND") 441 subparser_update = subparsers.add_parser("clean", help="") 442 subparser_update = subparsers.add_parser("build", help="") 443 args = parser.parse_args() 444 445 if args.COMMAND == "clean": 446 clean() 447 elif not args.COMMAND or args.COMMAND == "build": 448 build()