unpack-sdk.py (3790B)
1 # This Source Code Form is subject to the terms of the Mozilla Public 2 # License, v. 2.0. If a copy of the MPL was not distributed with this 3 # file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 5 import hashlib 6 import os 7 import shutil 8 import stat 9 import sys 10 import tempfile 11 from io import BytesIO 12 from urllib.request import urlopen 13 14 from mozpack.macpkg import Pbzx, uncpio, unxar 15 16 17 def unpack_sdk(url, sha512, extract_prefix, out_dir="."): 18 if "MOZ_AUTOMATION" in os.environ: 19 url = f"http://taskcluster/tooltool.mozilla-releng.net/sha512/{sha512}" 20 with tempfile.TemporaryFile() as pkg: 21 hash = hashlib.sha512() 22 for attempt in range(3): 23 if attempt != 0: 24 print(f"Failed to download from {url}. Retrying", file=sys.stderr) 25 26 with urlopen(url) as fh: 27 # Equivalent to shutil.copyfileobj, but computes sha512 at the same time. 28 while True: 29 buf = fh.read(1024 * 1024) 30 if not buf: 31 break 32 hash.update(buf) 33 pkg.write(buf) 34 digest = hash.hexdigest() 35 if digest == sha512: 36 break 37 else: 38 raise Exception(f"(actual) {digest} != (expected) {sha512}") 39 40 pkg.seek(0, os.SEEK_SET) 41 42 for name, content in unxar(pkg): 43 if name in ("Payload", "Content"): 44 extract_payload(content, extract_prefix, out_dir) 45 46 47 def extract_payload(fileobj, extract_prefix, out_dir="."): 48 hardlinks = {} 49 for path, st, content in uncpio(Pbzx(fileobj)): 50 if not path: 51 continue 52 path = path.decode() 53 matches = path.startswith(extract_prefix) 54 if matches: 55 path = os.path.join(out_dir, path[len(extract_prefix) :].lstrip("/")) 56 57 # When there are hardlinks, normally a cpio stream is supposed to 58 # contain the data for all of them, but, even with compression, that 59 # can be a waste of space, so in some cpio streams (*cough* *cough*, 60 # Apple's, e.g. in Xcode), the files after the first one have dummy 61 # data. 62 # As we may be filtering the first file out (if it doesn't match 63 # extract_prefix), we need to keep its data around (we're not going 64 # to be able to rewind). 65 if stat.S_ISREG(st.mode) and st.nlink > 1: 66 key = (st.dev, st.ino) 67 hardlink = hardlinks.get(key) 68 if hardlink: 69 hardlink[0] -= 1 70 if hardlink[0] == 0: 71 del hardlinks[key] 72 content = hardlink[1] 73 if isinstance(content, BytesIO): 74 content.seek(0) 75 if matches: 76 hardlink[1] = path 77 elif matches: 78 hardlink = hardlinks[key] = [st.nlink - 1, path] 79 else: 80 hardlink = hardlinks[key] = [st.nlink - 1, BytesIO(content.read())] 81 content = hardlink[1] 82 83 if not matches: 84 continue 85 if stat.S_ISDIR(st.mode): 86 os.makedirs(path, exist_ok=True) 87 else: 88 parent = os.path.dirname(path) 89 if parent: 90 os.makedirs(parent, exist_ok=True) 91 92 if stat.S_ISLNK(st.mode): 93 os.symlink(content.read(), path) 94 elif stat.S_ISREG(st.mode): 95 if isinstance(content, str): 96 os.link(content, path) 97 else: 98 with open(path, "wb") as out: 99 shutil.copyfileobj(content, out) 100 else: 101 raise Exception(f"File mode {st.mode:o} is not supported") 102 103 104 if __name__ == "__main__": 105 unpack_sdk(*sys.argv[1:])