tor-browser

bootstrap.configure (16266B)

---

# -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*-
# vim: set filetype=python:
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.

option(
    "--with-tor-browser-build-out",
    env="TOR_BROWSER_BUILD_OUT",
    nargs=1,
    default="https://tb-build-06.torproject.org/~tb-builder/tor-browser-build/out",
    help="URL pointing to a Tor Browser Build out folder, served over HTTP[S].",
)


@depends("--with-tor-browser-build-out")
def tor_browser_build_out(value):
    if value:
        return value[0]


option(
    "--enable-tor-browser-build-only-bootstrap",
    env="TBB_ONLY_BOOTSTRAP",
    default=False,
    help="Flag that disables bootstrapping any artifact from Mozilla's Taskcluster. Will only bootstrap artifacts from tor-browser-build.",
)


option(
    env="MOZ_FETCHES_DIR",
    nargs=1,
    when=moz_automation,
    help="Directory containing fetched artifacts",
)


@depends("MOZ_FETCHES_DIR", when=moz_automation)
def moz_fetches_dir(value):
    if value:
        return value[0]


@depends(vcs_checkout_type, milestone.is_nightly, moz_automation)
def bootstrap_default(vcs_checkout_type, is_nightly, automation):
    if automation:
        return False
    # We only enable if building off a VCS checkout of central.
    if is_nightly and vcs_checkout_type:
        return True


option(
    "--enable-bootstrap",
    nargs="*",
    default=bootstrap_default,
    help="{Automatically bootstrap or update some toolchains|Disable bootstrap or update of toolchains}",
)


@depends("--enable-bootstrap")
def want_bootstrap(bootstrap):
    include = set()
    exclude = set()
    for item in bootstrap:
        if item == "no-update":
            continue
        if item.startswith("-"):
            exclude.add(item.lstrip("-"))
        else:
            include.add(item)

    def match(name):
        if not bootstrap:
            return False

        if name in exclude:
            return False
        if include and name in include:
            return True
        return not bool(include)

    return match


toolchains_base_dir = moz_fetches_dir | mozbuild_state_path


@dependable
@imports("os")
@imports(_from="os", _import="environ")
def original_path():
    return environ["PATH"].split(os.pathsep)


@depends(host, when="--enable-bootstrap")
@imports("os")
@imports("traceback")
@imports(_from="mozbuild.toolchains", _import="toolchain_task_definitions")
@imports(_from="__builtin__", _import="Exception")
def bootstrap_toolchain_tasks(host):
    prefix = {
        ("x86_64", "GNU", "Linux"): "linux64",
        ("aarch64", "GNU", "Linux"): "linux64-aarch64",
        ("x86_64", "OSX", "Darwin"): "macosx64",
        ("aarch64", "OSX", "Darwin"): "macosx64-aarch64",
        ("x86_64", "WINNT", "WINNT"): "win64",
        ("aarch64", "WINNT", "WINNT"): "win64-aarch64",
    }.get((host.cpu, host.os, host.kernel))
    try:
        tasks = toolchain_task_definitions()
    except Exception as e:
        message = traceback.format_exc()
        log.warning(str(e))
        log.debug(message)
        return None

    def task_data(t):
        result = {
            "index": t.optimization["index-search"],
            "artifact": t.attributes[f"{t.kind}-artifact"],
            "extract": t.attributes.get(f"{t.kind}-extract", True),
        }
        command = t.attributes.get("toolchain-command")
        if command:
            result["command"] = command
        return result

    # We only want to use toolchains annotated with "local-toolchain". We also limit the
    # amount of data to what we use, so that trace logs can be more useful.
    tasks = {
        k: task_data(t)
        for k, t in tasks.items()
        if t.attributes.get(f"local-{t.kind}")
        and t.optimization
        and "index-search" in t.optimization
    }

    return namespace(prefix=prefix, tasks=tasks)


@template
def bootstrap_path(path, **kwargs):
    when = kwargs.pop("when", None)
    allow_failure = kwargs.pop("allow_failure", None)
    no_unpack = kwargs.pop("no_unpack", False)
    if kwargs:
        configure_error(
            "bootstrap_path only takes `when`, `allow_failure` and `no_unpack` as keyword arguments"
        )

    @depends(
        "--enable-bootstrap",
        want_bootstrap,
        toolchains_base_dir,
        moz_fetches_dir,
        bootstrap_toolchain_tasks,
        build_environment,
        dependable(path),
        dependable(allow_failure),
        dependable(no_unpack),
        tor_browser_build_out,
        "--enable-tor-browser-build-only-bootstrap",
        target,
        when=when,
    )
    @imports("os")
    @imports("re")
    @imports("subprocess")
    @imports("sys")
    @imports("mozbuild.tbbutils")
    @imports(_from="mozbuild.dirutils", _import="ensureParentDir")
    @imports(_from="importlib", _import="import_module")
    @imports(_from="shutil", _import="rmtree")
    @imports(_from="__builtin__", _import="open")
    @imports(_from="__builtin__", _import="Exception")
    def bootstrap_path(
        enable_bootstrap,
        want_bootstrap,
        toolchains_base_dir,
        moz_fetches_dir,
        tasks,
        build_env,
        path,
        allow_failure,
        no_unpack,
        tor_browser_build_out,
        tbb_only_bootstrap,
        target,
    ):
        if not path:
            return
        path_parts = path.split("/")
        path_prefix = ""
        # Small hack until clang-tidy stops being a separate toolchain in a
        # weird location.
        if path_parts[0] == "clang-tools":
            path_prefix = path_parts.pop(0)

        # Small hack because extensions are inside the browser folder.
        if path_parts[0] in ("noscript",):
            path_prefix = "browser"

        def try_tbb_bootstrap(exists):
            if not tor_browser_build_out:
                return False

            # Tor browser build doesn't have artifacts for all targets supported
            # by the Firefox build system. When this is empty it means we are
            # building for a platform which tbb doesn't support.
            if not target.tor_browser_build_alias:
                return False

            artifact = mozbuild.tbbutils.get_artifact_name(path_parts[0], tasks.prefix)
            if not artifact:
                log.info("%s is not mapped to a tbb artifact", path_parts[0])
                return False

            artifact_path = mozbuild.tbbutils.get_artifact_path(
                tor_browser_build_out,
                artifact,
                target,
                prefix=path_prefix,
                log=log.warning,
            )
            if not artifact_path:
                log.info("no path found in tbb/out for %s", artifact)
                return False

            artifact_index = mozbuild.tbbutils.get_artifact_index(
                artifact_path, artifact
            )
            index_file = os.path.join(toolchains_base_dir, "indices", artifact)
            try:
                with open(index_file) as fh:
                    index = fh.read().strip()
            except Exception:
                index = None
            if index == artifact_index and exists:
                log.debug("%s is up-to-date", artifact)
                return True

            command = ["artifact", "toolchain", "--from-url", artifact_path]

            if no_unpack:
                command.append("--no-unpack")

            # Note to rebasers:
            # From here on, it's a slightly modified copy/paste
            # from the end of the try_bootstrap function
            log.info(
                "%s bootstrapped toolchain from TBB in %s",
                "Updating" if exists else "Installing",
                os.path.join(toolchains_base_dir, path_prefix, artifact),
            )
            os.makedirs(os.path.join(toolchains_base_dir, path_prefix), exist_ok=True)
            proc = subprocess.run(
                [
                    sys.executable,
                    os.path.join(build_env.topsrcdir, "mach"),
                    "--log-no-times",
                ]
                + command,
                cwd=os.path.join(toolchains_base_dir, path_prefix),
                check=not allow_failure,
            )
            if proc.returncode != 0 and allow_failure:
                return False
            ensureParentDir(index_file)
            with open(index_file, "w") as fh:
                fh.write(artifact_index)

            return True

        def try_bootstrap(exists):
            if not tasks:
                return False
            prefixes = [""]
            if tasks.prefix:
                prefixes.insert(0, "{}-".format(tasks.prefix))
            for prefix in prefixes:
                for kind in ("toolchain", "fetch"):
                    label = f"{kind}-{prefix}{path_parts[0]}"
                    task = tasks.tasks.get(label)
                    if task:
                        break
                if task:
                    break
            log.debug("Trying to bootstrap %s", label)
            if not task:
                return False
            task_index = task["index"]
            log.debug("Resolved %s to %s", label, task_index[0])
            task_index = task_index[0].split(".")[-1]
            artifact = task["artifact"]
            # `mach artifact toolchain` doesn't support authentication for
            # private artifacts. Some toolchains may provide a command that can be
            # used for local production of the artifact.
            command = None
            if not artifact.startswith("public/"):
                command = task.get("command")
                if not command:
                    log.debug("Cannot bootstrap %s: not a public artifact", label)
                    return False
            index_file = os.path.join(toolchains_base_dir, "indices", path_parts[0])
            try:
                with open(index_file) as fh:
                    index = fh.read().strip()
            except Exception:
                # On automation, if there's an artifact in MOZ_FETCHES_DIR, we assume it's
                # up-to-date.
                index = task_index if moz_fetches_dir else None
            if index == task_index and exists:
                log.debug("%s is up-to-date", label)
                return True
            # Manually import with import_module so that we can gracefully disable bootstrap
            # when e.g. building from a js standalone tarball, that doesn't contain the
            # taskgraph code. In those cases, `mach artifact toolchain --from-build` would
            # also fail.
            task_id = None
            if not command:
                try:
                    IndexSearch = import_module(
                        "taskgraph.optimize.strategies"
                    ).IndexSearch
                except Exception:
                    log.debug("Cannot bootstrap %s: missing taskgraph module", label)
                    return False
                task_id = IndexSearch().should_replace_task(
                    task, {}, None, task["index"]
                )
            if task_id:
                # If we found the task in the index, use the `mach artifact toolchain`
                # fast path.
                command = [
                    "artifact",
                    "toolchain",
                    "--from-task",
                    f"{task_id}:{artifact}",
                ]
                if not task["extract"]:
                    command.append("--no-unpack")

            elif command:
                # For private local toolchains, run the associated command.
                command = (
                    [
                        "python",
                        "--virtualenv",
                        "build",
                        os.path.join(
                            build_env.topsrcdir,
                            "taskcluster/scripts/misc",
                            command["script"],
                        ),
                    ]
                    + command["arguments"]
                    + [path_parts[0]]
                )

                # BIG HACK: Replace the Apple CDN link with our mirror,
                # otherwise bootstrapping will fail whenever a new MacOS SDK
                # is released, since Apple seems to retire the previous link everytime.
                # Our mirror serves an _unmodified_ version of the file.
                macosx_sdk_match = re.match(r"^MacOSX(.*)\.sdk$", path_parts[0])
                if macosx_sdk_match:
                    version = macosx_sdk_match.group(1)
                    command = [
                        re.sub(
                            r"https://swcdn\.apple\.com/.*/CLTools_macOSNMOS_SDK\.pkg",
                            f"https://build-sources.tbb.torproject.org/CLTools_macOSNMOS_SDK-{version}.pkg",
                            c,
                        )
                        for c in command
                    ]

                # Clean up anything that was bootstrapped previously before going
                # forward. In other cases, that's taken care of by mach artifact toolchain.
                rmtree(
                    os.path.join(toolchains_base_dir, path_prefix, path_parts[0]),
                    ignore_errors=True,
                )
            else:
                # Otherwise, use the slower path, which will print a better error than
                # we would be able to.
                command = ["artifact", "toolchain", "--from-build", label]

            log.info(
                "%s bootstrapped toolchain in %s",
                "Updating" if exists else "Installing",
                os.path.join(toolchains_base_dir, path_prefix, path_parts[0]),
            )
            os.makedirs(os.path.join(toolchains_base_dir, path_prefix), exist_ok=True)
            proc = subprocess.run(
                [
                    sys.executable,
                    os.path.join(build_env.topsrcdir, "mach"),
                    "--log-no-times",
                ]
                + command,
                cwd=os.path.join(toolchains_base_dir, path_prefix),
                check=not allow_failure,
            )
            if proc.returncode != 0 and allow_failure:
                return False
            ensureParentDir(index_file)
            with open(index_file, "w") as fh:
                fh.write(task_index)
            return True

        path = os.path.join(toolchains_base_dir, path_prefix, *path_parts)
        if want_bootstrap(path_parts[0]):
            exists = os.path.exists(path)
            try:
                # With --enable-bootstrap=no-update, we don't `try_bootstrap`, except
                # when the toolchain can't be found.
                if ("no-update" not in enable_bootstrap or not exists) and not (
                    try_tbb_bootstrap(exists)
                    or (not tbb_only_bootstrap and try_bootstrap(exists))
                ):
                    # If there aren't toolchain artifacts to use for this build,
                    # don't return a path.
                    return None
            except Exception as e:
                log.error("%s", e)
                die(
                    "If you can't fix the above, retry with --enable-bootstrap=no-update."
                )
        if enable_bootstrap or enable_bootstrap.origin == "default":
            # We re-test whether the path exists because it may have been created by
            # try_bootstrap. Automation will not have gone through the bootstrap
            # process, but we want to return the path if it exists.
            if os.path.exists(path):
                return path

    return bootstrap_path


@template
def bootstrap_search_path(path, paths=original_path, **kwargs):
    @depends(
        bootstrap_path(path, **kwargs),
        paths,
        original_path,
    )
    def bootstrap_search_path(path, paths, original_path):
        if paths is None:
            paths = original_path
        if not path:
            return paths
        return [path] + paths

    return bootstrap_search_path