commit 2b10a74305692eefb9aa51dd725db5890d5221f6 parent c11a036fe841f6e2e14b04bed62a20831f3e2794 Author: serge-sans-paille <sguelton@mozilla.com> Date: Tue, 7 Oct 2025 08:43:32 +0000 Bug 1986311 - Use zst instead of gz to store test_archive r=jmaher,webdriver-reviewers,jgraham,glandium,whimboo zst is faster, can run in parallel, and has better compression ratio. With this change target.perftests.tests.tar.gz's size does down from 92M to 83M, and the package_test step goes from 1m25s down to 37s Differential Revision: https://phabricator.services.mozilla.com/D263314 Diffstat:
24 files changed, 179 insertions(+), 146 deletions(-)
diff --git a/build/gen_test_packages_manifest.py b/build/gen_test_packages_manifest.py @@ -110,14 +110,14 @@ def generate_package_data(args): harness_requirements = dict([(k, [tests_common]) for k in ALL_HARNESSES]) harness_requirements["jittest"].append(jsshell) harness_requirements["jsreftest"].append(args.reftest) - harness_requirements["common"].append("target.condprof.tests.tar.gz") + harness_requirements["common"].append("target.condprof.tests.tar.zst") for harness in PACKAGE_SPECIFIED_HARNESSES + OPTIONAL_PACKAGES: pkg_name = getattr(args, harness, None) if pkg_name is None: continue harness_requirements[harness].append(pkg_name) - harness_requirements[harness].append("target.condprof.tests.tar.gz") - harness_requirements[harness].append("target.trainhop.tests.tar.gz") + harness_requirements[harness].append("target.condprof.tests.tar.zst") + harness_requirements[harness].append("target.trainhop.tests.tar.zst") return harness_requirements diff --git a/python/mozbuild/mozbuild/action/test_archive.py b/python/mozbuild/mozbuild/action/test_archive.py @@ -17,7 +17,7 @@ import time import buildconfig import mozpack.path as mozpath from manifestparser import TestManifest -from mozpack.archive import create_tar_gz_from_files +from mozpack.archive import create_tar_gz_from_files, create_tar_zst_from_files from mozpack.copier import FileRegistry from mozpack.files import ExistingFile, FileFinder from mozpack.manifests import InstallManifest @@ -903,8 +903,8 @@ def main(argv): args = parser.parse_args(argv) out_file = args.outputfile - if not out_file.endswith((".tar.gz", ".zip")): - raise Exception("expected tar.gz or zip output file") + if not out_file.endswith((".tar.gz", ".tar.zst", ".zip")): + raise Exception("expected tar.gz, tar.zst or zip output file") file_count = 0 t_start = time.monotonic() @@ -919,6 +919,10 @@ def main(argv): files = dict(res) create_tar_gz_from_files(fh, files, compresslevel=5) file_count = len(files) + elif out_file.endswith(".tar.zst"): + files = dict(res) + create_tar_zst_from_files(fh, files, compresslevel=5, threads=-1) + file_count = len(files) elif out_file.endswith(".zip"): with JarWriter(fileobj=fh, compress_level=5) as writer: for p, f in res: diff --git a/python/mozbuild/mozbuild/artifacts.py b/python/mozbuild/mozbuild/artifacts.py @@ -175,7 +175,7 @@ class ArtifactJob: # We can tell our input is a test archive by this suffix, which happens to # be the same across platforms. _test_zip_archive_suffix = ".common.tests.zip" - _test_tar_archive_suffix = ".common.tests.tar.gz" + _test_tar_archive_suffix = ".common.tests.tar.zst" # A map of extra archives to fetch and unpack. An extra archive might # include optional build output to incorporate into the local artifact @@ -225,7 +225,7 @@ class ArtifactJob: self._tests_re = None if download_tests: self._tests_re = re.compile( - r"public/build/(en-US/)?target\.common\.tests\.(zip|tar\.gz)$" + r"public/build/(en-US/)?target\.common\.tests\.(zip|tar\.zst)$" ) self._maven_zip_re = None if download_maven_zip: diff --git a/python/mozbuild/mozpack/archive.py b/python/mozbuild/mozpack/archive.py @@ -118,6 +118,22 @@ def create_tar_gz_from_files(fp, files, filename=None, compresslevel=9): create_tar_from_files(gf, files) +def create_tar_zst_from_files(fp, files, filename=None, compresslevel=9, threads=1): + """Create a tar.zst file deterministically from files. + + This is a glorified wrapper around ``create_tar_from_files`` that + adds zstandard compression. + + The passed file handle should be opened for writing in binary mode. + When the function returns, all data has been written to the handle. + """ + import zstandard + + cctx = zstandard.ZstdCompressor(level=compresslevel, threads=threads) + with cctx.stream_writer(writer=fp) as compressor: + create_tar_from_files(compressor, files) + + class _BZ2Proxy: """File object that proxies writes to a bz2 compressor.""" diff --git a/python/mozperftest/perfdocs/writing.rst b/python/mozperftest/perfdocs/writing.rst @@ -54,7 +54,7 @@ Here's an example of such a metrics call:: XPCShell Tests in CI ^^^^^^^^^^^^^^^^^^^^ -To run your test in CI, you may need to modify the ``_TRY_MAPPING`` variable `found here <https://searchfox.org/mozilla-central/rev/7d1b5c88343879056168aa710a9ee743392604c0/python/mozperftest/mozperftest/utils.py#299>`_. This will allow us to find your test file in CI, and is needed because the file mappings differ from local runs. The mapping maps the top-level folder of the test to it's location in CI. To find this location/mapping, download the ``target.xpcshell.tests.tar.gz`` archive from the build task and search for your test file in it. +To run your test in CI, you may need to modify the ``_TRY_MAPPING`` variable `found here <https://searchfox.org/mozilla-central/rev/7d1b5c88343879056168aa710a9ee743392604c0/python/mozperftest/mozperftest/utils.py#299>`_. This will allow us to find your test file in CI, and is needed because the file mappings differ from local runs. The mapping maps the top-level folder of the test to it's location in CI. To find this location/mapping, download the ``target.xpcshell.tests.tar.zst`` archive from the build task and search for your test file in it. The XPCShell test that is written can also be run as a unit test, however, if this is not desired, set the `disabled = reason` flag in the test TOML file to prevent it from running there. `See here for an example <https://searchfox.org/mozilla-central/rev/7d1b5c88343879056168aa710a9ee743392604c0/toolkit/components/ml/tests/browser/perftest.toml#7>`_. @@ -121,7 +121,7 @@ Only the GeckoView Test Runner, and GeckoView Example are currently supported in Mochitest Tests in CI ^^^^^^^^^^^^^^^^^^^^^ -To run your test in CI, you may need to modify the ``_TRY_MAPPING`` variable `found here <https://searchfox.org/mozilla-central/rev/7d1b5c88343879056168aa710a9ee743392604c0/python/mozperftest/mozperftest/utils.py#299>`_. This will allow us to find your test file in CI, and is needed because the file mappings differ from local runs. The mapping maps the top-level folder of the test to it's location in CI. To find this location/mapping, download the ``target.mochitest.tests.tar.gz`` archive from the build task and search for your test file in it. +To run your test in CI, you may need to modify the ``_TRY_MAPPING`` variable `found here <https://searchfox.org/mozilla-central/rev/7d1b5c88343879056168aa710a9ee743392604c0/python/mozperftest/mozperftest/utils.py#299>`_. This will allow us to find your test file in CI, and is needed because the file mappings differ from local runs. The mapping maps the top-level folder of the test to it's location in CI. To find this location/mapping, download the ``target.mochitest.tests.tar.zst`` archive from the build task and search for your test file in it. The Mochitest test that is written can also be run as a unit test, however, if this is not desired, set the `disabled = reason` flag in the test TOML file to prevent it from running there. `See here for an example <https://searchfox.org/mozilla-central/rev/7d1b5c88343879056168aa710a9ee743392604c0/toolkit/components/ml/tests/browser/perftest.toml#7>`_. @@ -133,10 +133,10 @@ For Mochitest Android tests in CI, everything that applies to desktop tests also build: - artifact: geckoview_example.apk extract: false - - artifact: en-US/target.perftests.tests.tar.gz - - artifact: en-US/target.condprof.tests.tar.gz - - artifact: en-US/target.common.tests.tar.gz - - artifact: en-US/target.mochitest.tests.tar.gz + - artifact: en-US/target.perftests.tests.tar.zst + - artifact: en-US/target.condprof.tests.tar.zst + - artifact: en-US/target.common.tests.tar.zst + - artifact: en-US/target.mochitest.tests.tar.zst toolchain: - linux64-hostutils diff --git a/remote/doc/marionette/Testing.md b/remote/doc/marionette/Testing.md @@ -212,7 +212,7 @@ possibly to run the Marionette tests _without_ a local build and with a downloaded test archive from [Taskcluster](Taskcluster.md) If you want to run tests from a downloaded test archive, you will -need to download the `target.common.tests.tar.gz` artifact attached to +need to download the `target.common.tests.tar.zst` artifact attached to Treeherder [build jobs] `B` for your system. Extract the archive and set up the Python Marionette client and harness by executing the following command in a virtual environment: diff --git a/taskcluster/docker/periodic-updates/scripts/periodic_file_updates.sh b/taskcluster/docker/periodic-updates/scripts/periodic_file_updates.sh @@ -131,7 +131,7 @@ case "${BRANCH}" in esac BROWSER_ARCHIVE="target.tar.xz" -TESTS_ARCHIVE="target.common.tests.tar.gz" +TESTS_ARCHIVE="target.common.tests.tar.zst" UNPACK_CMD="tar Jxf" COMMIT_AUTHOR='ffxbld <ffxbld@mozilla.com>' diff --git a/taskcluster/kinds/build/macosx.yml b/taskcluster/kinds/build/macosx.yml @@ -433,39 +433,39 @@ macosx64-devedition/opt: - artifact: target.test_packages.json extract: false dest: ../artifacts - - artifact: target.perftests.tests.tar.gz + - artifact: target.perftests.tests.tar.zst extract: false dest: ../artifacts - artifact: target.xpt_artifacts.zip extract: false dest: ../artifacts - - artifact: target.common.tests.tar.gz + - artifact: target.common.tests.tar.zst dest: x64/common - - artifact: target.cppunittest.tests.tar.gz + - artifact: target.cppunittest.tests.tar.zst dest: x64/cppunittest - - artifact: target.gtest.tests.tar.gz + - artifact: target.gtest.tests.tar.zst dest: x64/gtest - - artifact: target.jittest.tests.tar.gz + - artifact: target.jittest.tests.tar.zst dest: x64/jittest - - artifact: target.jsreftest.tests.tar.gz + - artifact: target.jsreftest.tests.tar.zst dest: x64/jsreftest - - artifact: target.mochitest.tests.tar.gz + - artifact: target.mochitest.tests.tar.zst dest: x64/mochitest - - artifact: target.reftest.tests.tar.gz + - artifact: target.reftest.tests.tar.zst dest: x64/reftest - - artifact: target.talos.tests.tar.gz + - artifact: target.talos.tests.tar.zst dest: x64/talos - - artifact: target.raptor.tests.tar.gz + - artifact: target.raptor.tests.tar.zst dest: x64/raptor - - artifact: target.condprof.tests.tar.gz + - artifact: target.condprof.tests.tar.zst dest: x64/confprof - - artifact: target.awsy.tests.tar.gz + - artifact: target.awsy.tests.tar.zst dest: x64/awsy - - artifact: target.xpcshell.tests.tar.gz + - artifact: target.xpcshell.tests.tar.zst dest: x64/xpcshell - - artifact: target.web-platform.tests.tar.gz + - artifact: target.web-platform.tests.tar.zst dest: x64/web-platform - - artifact: target.updater-dep.tests.tar.gz + - artifact: target.updater-dep.tests.tar.zst dest: x64/updater-dep - artifact: target.crashreporter-symbols.zip dest: x64/crashreporter-symbols @@ -489,33 +489,33 @@ macosx64-devedition/opt: - artifact: target.dmg extract: false dest: aarch64 - - artifact: target.common.tests.tar.gz + - artifact: target.common.tests.tar.zst dest: aarch64/common - - artifact: target.cppunittest.tests.tar.gz + - artifact: target.cppunittest.tests.tar.zst dest: aarch64/cppunittest - - artifact: target.gtest.tests.tar.gz + - artifact: target.gtest.tests.tar.zst dest: aarch64/gtest - - artifact: target.jittest.tests.tar.gz + - artifact: target.jittest.tests.tar.zst dest: aarch64/jittest - - artifact: target.jsreftest.tests.tar.gz + - artifact: target.jsreftest.tests.tar.zst dest: aarch64/jsreftest - - artifact: target.mochitest.tests.tar.gz + - artifact: target.mochitest.tests.tar.zst dest: aarch64/mochitest - - artifact: target.reftest.tests.tar.gz + - artifact: target.reftest.tests.tar.zst dest: aarch64/reftest - - artifact: target.talos.tests.tar.gz + - artifact: target.talos.tests.tar.zst dest: aarch64/talos - - artifact: target.raptor.tests.tar.gz + - artifact: target.raptor.tests.tar.zst dest: aarch64/raptor - - artifact: target.condprof.tests.tar.gz + - artifact: target.condprof.tests.tar.zst dest: aarch64/confprof - - artifact: target.awsy.tests.tar.gz + - artifact: target.awsy.tests.tar.zst dest: aarch64/awsy - - artifact: target.xpcshell.tests.tar.gz + - artifact: target.xpcshell.tests.tar.zst dest: aarch64/xpcshell - - artifact: target.web-platform.tests.tar.gz + - artifact: target.web-platform.tests.tar.zst dest: aarch64/web-platform - - artifact: target.updater-dep.tests.tar.gz + - artifact: target.updater-dep.tests.tar.zst dest: aarch64/updater-dep - artifact: target.crashreporter-symbols.zip dest: aarch64/crashreporter-symbols @@ -871,39 +871,39 @@ macosx64-shippable/opt: - artifact: target.test_packages.json extract: false dest: ../artifacts - - artifact: target.perftests.tests.tar.gz + - artifact: target.perftests.tests.tar.zst extract: false dest: ../artifacts - artifact: target.xpt_artifacts.zip extract: false dest: ../artifacts - - artifact: target.common.tests.tar.gz + - artifact: target.common.tests.tar.zst dest: x64/common - - artifact: target.cppunittest.tests.tar.gz + - artifact: target.cppunittest.tests.tar.zst dest: x64/cppunittest - - artifact: target.gtest.tests.tar.gz + - artifact: target.gtest.tests.tar.zst dest: x64/gtest - - artifact: target.jittest.tests.tar.gz + - artifact: target.jittest.tests.tar.zst dest: x64/jittest - - artifact: target.jsreftest.tests.tar.gz + - artifact: target.jsreftest.tests.tar.zst dest: x64/jsreftest - - artifact: target.mochitest.tests.tar.gz + - artifact: target.mochitest.tests.tar.zst dest: x64/mochitest - - artifact: target.reftest.tests.tar.gz + - artifact: target.reftest.tests.tar.zst dest: x64/reftest - - artifact: target.talos.tests.tar.gz + - artifact: target.talos.tests.tar.zst dest: x64/talos - - artifact: target.raptor.tests.tar.gz + - artifact: target.raptor.tests.tar.zst dest: x64/raptor - - artifact: target.condprof.tests.tar.gz + - artifact: target.condprof.tests.tar.zst dest: x64/confprof - - artifact: target.awsy.tests.tar.gz + - artifact: target.awsy.tests.tar.zst dest: x64/awsy - - artifact: target.xpcshell.tests.tar.gz + - artifact: target.xpcshell.tests.tar.zst dest: x64/xpcshell - - artifact: target.web-platform.tests.tar.gz + - artifact: target.web-platform.tests.tar.zst dest: x64/web-platform - - artifact: target.updater-dep.tests.tar.gz + - artifact: target.updater-dep.tests.tar.zst dest: x64/updater-dep - artifact: target.crashreporter-symbols.zip dest: x64/crashreporter-symbols @@ -920,40 +920,40 @@ macosx64-shippable/opt: - artifact: mozharness.zip extract: false dest: ../artifacts - - artifact: target.trainhop.tests.tar.gz + - artifact: target.trainhop.tests.tar.zst extract: false dest: ../artifacts macosx64-aarch64-shippable-opt: - artifact: target.dmg extract: false dest: aarch64 - - artifact: target.common.tests.tar.gz + - artifact: target.common.tests.tar.zst dest: aarch64/common - - artifact: target.cppunittest.tests.tar.gz + - artifact: target.cppunittest.tests.tar.zst dest: aarch64/cppunittest - - artifact: target.gtest.tests.tar.gz + - artifact: target.gtest.tests.tar.zst dest: aarch64/gtest - - artifact: target.jittest.tests.tar.gz + - artifact: target.jittest.tests.tar.zst dest: aarch64/jittest - - artifact: target.jsreftest.tests.tar.gz + - artifact: target.jsreftest.tests.tar.zst dest: aarch64/jsreftest - - artifact: target.mochitest.tests.tar.gz + - artifact: target.mochitest.tests.tar.zst dest: aarch64/mochitest - - artifact: target.reftest.tests.tar.gz + - artifact: target.reftest.tests.tar.zst dest: aarch64/reftest - - artifact: target.talos.tests.tar.gz + - artifact: target.talos.tests.tar.zst dest: aarch64/talos - - artifact: target.raptor.tests.tar.gz + - artifact: target.raptor.tests.tar.zst dest: aarch64/raptor - - artifact: target.condprof.tests.tar.gz + - artifact: target.condprof.tests.tar.zst dest: aarch64/confprof - - artifact: target.awsy.tests.tar.gz + - artifact: target.awsy.tests.tar.zst dest: aarch64/awsy - - artifact: target.xpcshell.tests.tar.gz + - artifact: target.xpcshell.tests.tar.zst dest: aarch64/xpcshell - - artifact: target.web-platform.tests.tar.gz + - artifact: target.web-platform.tests.tar.zst dest: aarch64/web-platform - - artifact: target.updater-dep.tests.tar.gz + - artifact: target.updater-dep.tests.tar.zst dest: aarch64/updater-dep - artifact: target.crashreporter-symbols.zip dest: aarch64/crashreporter-symbols diff --git a/taskcluster/kinds/condprof/kind.yml b/taskcluster/kinds/condprof/kind.yml @@ -48,7 +48,7 @@ tasks: - full fetches: build: - - artifact: target.condprof.tests.tar.gz + - artifact: target.condprof.tests.tar.zst - artifact: target.zip toolchain: - win64-geckodriver @@ -82,7 +82,7 @@ tasks: - full fetches: build: - - artifact: target.condprof.tests.tar.gz + - artifact: target.condprof.tests.tar.zst - artifact: target.tar.xz toolchain: - linux64-geckodriver @@ -115,7 +115,7 @@ tasks: build: build-macosx64-shippable/opt fetches: build: - - artifact: target.condprof.tests.tar.gz + - artifact: target.condprof.tests.tar.zst - artifact: target.dmg extract: false toolchain: diff --git a/taskcluster/kinds/fuzzing/kind.yml b/taskcluster/kinds/fuzzing/kind.yml @@ -44,7 +44,7 @@ tasks: fetches: build: - target.jsshell.zip - - target.fuzztest.tests.tar.gz + - target.fuzztest.tests.tar.zst worker: docker-image: {in-tree: ubuntu2404-test} env: @@ -71,7 +71,7 @@ tasks: fetches: build: - artifact: target.tar.xz - - artifact: target.gtest.tests.tar.gz + - artifact: target.gtest.tests.tar.zst - artifact: target.mozinfo.json grizzly-linux64-debug: @@ -89,7 +89,7 @@ tasks: fetches: build: - artifact: target.tar.xz - - artifact: target.gtest.tests.tar.gz + - artifact: target.gtest.tests.tar.zst - artifact: target.mozinfo.json grizzly-linux64-tsan: @@ -107,7 +107,7 @@ tasks: fetches: build: - artifact: target.tar.xz - - artifact: target.gtest.tests.tar.gz + - artifact: target.gtest.tests.tar.zst - artifact: target.mozinfo.json grizzly-windows-debug: diff --git a/taskcluster/kinds/perftest/linux.yml b/taskcluster/kinds/perftest/linux.yml @@ -11,11 +11,11 @@ task-defaults: - browsertime build: - artifact: target.mozinfo.json - - artifact: target.common.tests.tar.gz - - artifact: target.condprof.tests.tar.gz - - artifact: target.perftests.tests.tar.gz - - artifact: target.xpcshell.tests.tar.gz - - artifact: target.mochitest.tests.tar.gz + - artifact: target.common.tests.tar.zst + - artifact: target.condprof.tests.tar.zst + - artifact: target.perftests.tests.tar.zst + - artifact: target.xpcshell.tests.tar.zst + - artifact: target.mochitest.tests.tar.zst - artifact: target.tar.xz platform: linux1804-64-shippable/opt require-build: diff --git a/taskcluster/kinds/perftest/macosx.yml b/taskcluster/kinds/perftest/macosx.yml @@ -7,11 +7,11 @@ task-defaults: fetches: build: - artifact: target.mozinfo.json - - artifact: target.common.tests.tar.gz - - artifact: target.condprof.tests.tar.gz - - artifact: target.mochitest.tests.tar.gz - - artifact: target.xpcshell.tests.tar.gz - - artifact: target.perftests.tests.tar.gz + - artifact: target.common.tests.tar.zst + - artifact: target.condprof.tests.tar.zst + - artifact: target.mochitest.tests.tar.zst + - artifact: target.xpcshell.tests.tar.zst + - artifact: target.perftests.tests.tar.zst - artifact: target.dmg extract: false toolchain: diff --git a/taskcluster/kinds/perftest/windows11-24h2-ref.yml b/taskcluster/kinds/perftest/windows11-24h2-ref.yml @@ -6,10 +6,10 @@ task-defaults: worker-type: win11-64-24h2-hw-ref fetches: build: - - artifact: target.condprof.tests.tar.gz - - artifact: target.common.tests.tar.gz - - artifact: target.mochitest.tests.tar.gz - - artifact: target.perftests.tests.tar.gz + - artifact: target.condprof.tests.tar.zst + - artifact: target.common.tests.tar.zst + - artifact: target.mochitest.tests.tar.zst + - artifact: target.perftests.tests.tar.zst - artifact: target.zip toolchain: - win64-node diff --git a/taskcluster/kinds/perftest/windows11-24h2.yml b/taskcluster/kinds/perftest/windows11-24h2.yml @@ -6,10 +6,10 @@ task-defaults: worker-type: win11-64-24h2-hw fetches: build: - - artifact: target.condprof.tests.tar.gz - - artifact: target.common.tests.tar.gz - - artifact: target.mochitest.tests.tar.gz - - artifact: target.perftests.tests.tar.gz + - artifact: target.condprof.tests.tar.zst + - artifact: target.common.tests.tar.zst + - artifact: target.mochitest.tests.tar.zst + - artifact: target.perftests.tests.tar.zst - artifact: target.zip toolchain: - win64-node diff --git a/taskcluster/kinds/source-test/python.yml b/taskcluster/kinds/source-test/python.yml @@ -144,9 +144,9 @@ mochitest-harness: fetches: build: - target.tar.xz - - artifact: target.common.tests.tar.gz + - artifact: target.common.tests.tar.zst dest: tests - - artifact: target.mochitest.tests.tar.gz + - artifact: target.mochitest.tests.tar.zst dest: tests toolchain: - linux64-fix-stacks @@ -327,9 +327,9 @@ reftest-harness: fetches: build: - target.tar.xz - - artifact: target.common.tests.tar.gz + - artifact: target.common.tests.tar.zst dest: tests - - artifact: target.reftest.tests.tar.gz + - artifact: target.reftest.tests.tar.zst dest: tests toolchain: - linux64-fix-stacks @@ -456,7 +456,7 @@ nimbus: fetches: build: - artifact: target.tar.xz - - artifact: target.common.tests.tar.gz + - artifact: target.common.tests.tar.zst worker: by-platform: linux2404-64.*: @@ -497,7 +497,7 @@ fxms-schemas: fetches: build: - artifact: target.tar.xz - - artifact: target.common.tests.tar.gz + - artifact: target.common.tests.tar.zst worker: by-platform: linux2404-64.*: @@ -596,9 +596,9 @@ telemetry-integration-tests: fetches: build: - target.tar.bz2 - - artifact: target.common.tests.tar.gz + - artifact: target.common.tests.tar.zst dest: tests - - artifact: target.reftest.tests.tar.gz + - artifact: target.reftest.tests.tar.zst dest: tests when: files-changed: diff --git a/taskcluster/kinds/webrender/kind.yml b/taskcluster/kinds/webrender/kind.yml @@ -256,8 +256,8 @@ tasks: name: public/build/wrench-debug.apk path: /builds/worker/checkouts/gecko/gfx/wr/target/debug/apk/wrench.apk - type: file - name: public/build/reftests.tar.gz - path: /builds/worker/checkouts/gecko/gfx/wr/wrench/reftests.tar.gz + name: public/build/reftests.tar.zst + path: /builds/worker/checkouts/gecko/gfx/wr/wrench/reftests.tar.zst - type: file name: public/build/mozdevice.tar.gz path: /builds/worker/checkouts/gecko/testing/mozbase/mozdevice.tar.gz @@ -269,7 +269,7 @@ tasks: cwd: '{checkout}/gfx/wr/wrench' command: >- $GECKO_PATH/taskcluster/scripts/misc/wrench-android-build.sh debug && - tar czf reftests.tar.gz reftests/ && + tar -c --zstd -f reftests.tar.zst reftests/ && cd $GECKO_PATH/testing/mozbase && tar czf mozdevice.tar.gz mozdevice/ && cd $GECKO_PATH/third_party/python && @@ -306,8 +306,8 @@ tasks: name: public/build/wrench-release.apk path: /builds/worker/checkouts/gecko/gfx/wr/target/release/apk/wrench.apk - type: file - name: public/build/reftests.tar.gz - path: /builds/worker/checkouts/gecko/gfx/wr/wrench/reftests.tar.gz + name: public/build/reftests.tar.zst + path: /builds/worker/checkouts/gecko/gfx/wr/wrench/reftests.tar.zst - type: file name: public/build/mozdevice.tar.gz path: /builds/worker/checkouts/gecko/testing/mozbase/mozdevice.tar.gz @@ -319,7 +319,7 @@ tasks: cwd: '{checkout}/gfx/wr/wrench' command: >- $GECKO_PATH/taskcluster/scripts/misc/wrench-android-build.sh release && - tar czf reftests.tar.gz reftests/ && + tar -c --zstd -f reftests.tar.zst reftests/ && cd $GECKO_PATH/testing/mozbase && tar czf mozdevice.tar.gz mozdevice/ && cd $GECKO_PATH/third_party/python && @@ -458,7 +458,7 @@ tasks: webrender-wrench-android-debug: - artifact: 'wrench-debug.apk' extract: false - - 'reftests.tar.gz' + - 'reftests.tar.zst' - 'mozdevice.tar.gz' - 'six.tar.gz' android-build: @@ -506,7 +506,7 @@ tasks: webrender-wrench-android-release: - artifact: 'wrench-release.apk' extract: false - - 'reftests.tar.gz' + - 'reftests.tar.zst' - 'mozdevice.tar.gz' - 'six.tar.gz' android-build: @@ -554,7 +554,7 @@ tasks: webrender-wrench-android-debug: - artifact: 'wrench-debug.apk' extract: false - - 'reftests.tar.gz' + - 'reftests.tar.zst' - 'mozdevice.tar.gz' - 'six.tar.gz' android-build: @@ -602,7 +602,7 @@ tasks: webrender-wrench-android-release: - artifact: 'wrench-release.apk' extract: false - - 'reftests.tar.gz' + - 'reftests.tar.zst' - 'mozdevice.tar.gz' - 'six.tar.gz' android-build: @@ -650,7 +650,7 @@ tasks: webrender-wrench-android-debug: - artifact: 'wrench-debug.apk' extract: false - - 'reftests.tar.gz' + - 'reftests.tar.zst' - 'mozdevice.tar.gz' - 'six.tar.gz' android-build: @@ -698,7 +698,7 @@ tasks: webrender-wrench-android-release: - artifact: 'wrench-release.apk' extract: false - - 'reftests.tar.gz' + - 'reftests.tar.zst' - 'mozdevice.tar.gz' - 'six.tar.gz' android-build: diff --git a/testing/condprofile/condprof/check_install.py b/testing/condprofile/condprof/check_install.py @@ -23,7 +23,7 @@ def install_reqs(): except Exception: # we're detecting here that this is running in Taskcluster # by checking for the presence of the mozfile directory - # that was decompressed from target.condprof.tests.tar.gz + # that was decompressed from target.condprof.tests.tar.zst run_in_ci = os.path.exists(os.path.join(TOPDIR, "mozfile")) # On Python 2 we only install what's required for condprof.client diff --git a/testing/condprofile/requirements/ci-client.txt b/testing/condprofile/requirements/ci-client.txt @@ -3,7 +3,7 @@ requests==2.32.3 pyyaml==5.1.2 -# the target.condprof.tests.tar.gz archive pulls those dependencies +# the target.condprof.tests.tar.zst archive pulls those dependencies # directly into the condprof project root ./mozfile ./mozlog diff --git a/testing/condprofile/requirements/ci.txt b/testing/condprofile/requirements/ci.txt @@ -1,5 +1,5 @@ # pulled when running in TaskCluster -# the target.condprof.tests.tar.gz archive pulls those dependencies +# the target.condprof.tests.tar.zst archive pulls those dependencies # directly into the condprof project root. ./mozfile ./mozlog diff --git a/testing/mozharness/mozharness/base/script.py b/testing/mozharness/mozharness/base/script.py @@ -722,8 +722,16 @@ class ScriptMixin(PlatformMixin): mode (str): string of the form 'filemode[:compression]' (e.g. 'r:gz' or 'r:bz2') extract_to (str, optional): where to extract the compressed file. """ - with tarfile.open(fileobj=compressed_file, mode=mode) as t: - _safe_extract(t, path=extract_to) + if mode == "r|zst": + import zstandard + + unzstd = zstandard.ZstdDecompressor() + with unzstd.stream_reader(compressed_file) as stream: + with tarfile.open(mode="r|", fileobj=stream) as t: + _safe_extract(t, path=extract_to) + else: + with tarfile.open(fileobj=compressed_file, mode=mode) as t: + _safe_extract(t, path=extract_to) def download_unpack(self, url, extract_to=".", extract_dirs="*", verbose=False): """Generic method to download and extract a compressed file without writing it @@ -744,6 +752,7 @@ class ScriptMixin(PlatformMixin): EXTENSION_TO_MIMETYPE = { "bz2": "application/x-bzip2", "xz": "application/x-xz", + "zst": "application/zstd", "gz": "application/x-gzip", "tar": "application/x-tar", "zip": "application/zip", @@ -753,6 +762,10 @@ class ScriptMixin(PlatformMixin): "function": self.deflate, "kwargs": {"mode": "r:xz"}, }, + "application/zstd": { + "function": self.deflate, + "kwargs": {"mode": "r|zst"}, + }, "application/x-bzip2": { "function": self.deflate, "kwargs": {"mode": "r:bz2"}, diff --git a/testing/perfdocs/generated/writing.rst b/testing/perfdocs/generated/writing.rst @@ -54,7 +54,7 @@ Here's an example of such a metrics call:: XPCShell Tests in CI ^^^^^^^^^^^^^^^^^^^^ -To run your test in CI, you may need to modify the ``_TRY_MAPPING`` variable `found here <https://searchfox.org/mozilla-central/rev/7d1b5c88343879056168aa710a9ee743392604c0/python/mozperftest/mozperftest/utils.py#299>`_. This will allow us to find your test file in CI, and is needed because the file mappings differ from local runs. The mapping maps the top-level folder of the test to it's location in CI. To find this location/mapping, download the ``target.xpcshell.tests.tar.gz`` archive from the build task and search for your test file in it. +To run your test in CI, you may need to modify the ``_TRY_MAPPING`` variable `found here <https://searchfox.org/mozilla-central/rev/7d1b5c88343879056168aa710a9ee743392604c0/python/mozperftest/mozperftest/utils.py#299>`_. This will allow us to find your test file in CI, and is needed because the file mappings differ from local runs. The mapping maps the top-level folder of the test to it's location in CI. To find this location/mapping, download the ``target.xpcshell.tests.tar.zst`` archive from the build task and search for your test file in it. The XPCShell test that is written can also be run as a unit test, however, if this is not desired, set the `disabled = reason` flag in the test TOML file to prevent it from running there. `See here for an example <https://searchfox.org/mozilla-central/rev/7d1b5c88343879056168aa710a9ee743392604c0/toolkit/components/ml/tests/browser/perftest.toml#7>`_. @@ -121,7 +121,7 @@ Only the GeckoView Test Runner, and GeckoView Example are currently supported in Mochitest Tests in CI ^^^^^^^^^^^^^^^^^^^^^ -To run your test in CI, you may need to modify the ``_TRY_MAPPING`` variable `found here <https://searchfox.org/mozilla-central/rev/7d1b5c88343879056168aa710a9ee743392604c0/python/mozperftest/mozperftest/utils.py#299>`_. This will allow us to find your test file in CI, and is needed because the file mappings differ from local runs. The mapping maps the top-level folder of the test to it's location in CI. To find this location/mapping, download the ``target.mochitest.tests.tar.gz`` archive from the build task and search for your test file in it. +To run your test in CI, you may need to modify the ``_TRY_MAPPING`` variable `found here <https://searchfox.org/mozilla-central/rev/7d1b5c88343879056168aa710a9ee743392604c0/python/mozperftest/mozperftest/utils.py#299>`_. This will allow us to find your test file in CI, and is needed because the file mappings differ from local runs. The mapping maps the top-level folder of the test to it's location in CI. To find this location/mapping, download the ``target.mochitest.tests.tar.zst`` archive from the build task and search for your test file in it. The Mochitest test that is written can also be run as a unit test, however, if this is not desired, set the `disabled = reason` flag in the test TOML file to prevent it from running there. `See here for an example <https://searchfox.org/mozilla-central/rev/7d1b5c88343879056168aa710a9ee743392604c0/toolkit/components/ml/tests/browser/perftest.toml#7>`_. @@ -133,10 +133,10 @@ For Mochitest Android tests in CI, everything that applies to desktop tests also build: - artifact: geckoview_example.apk extract: false - - artifact: en-US/target.perftests.tests.tar.gz - - artifact: en-US/target.condprof.tests.tar.gz - - artifact: en-US/target.common.tests.tar.gz - - artifact: en-US/target.mochitest.tests.tar.gz + - artifact: en-US/target.perftests.tests.tar.zst + - artifact: en-US/target.condprof.tests.tar.zst + - artifact: en-US/target.common.tests.tar.zst + - artifact: en-US/target.mochitest.tests.tar.zst toolchain: - linux64-hostutils diff --git a/testing/testsuite-targets.mk b/testing/testsuite-targets.mk @@ -103,7 +103,7 @@ ifdef COMPILE_ENVIRONMENT stage-all: stage-cppunittests endif -TEST_PKGS_TARGZ := \ +TEST_PKGS_TARZST := \ common \ condprof \ cppunittest \ @@ -124,7 +124,7 @@ TEST_PKGS_TARGZ := \ ifdef LINK_GTEST_DURING_COMPILE stage-all: stage-gtest -TEST_PKGS_TARGZ += gtest +TEST_PKGS_TARZST += gtest endif PKG_ARG = --$(1) '$(PKG_BASENAME).$(1).tests.$(2)' @@ -136,7 +136,7 @@ test-packages-manifest: --jsshell $(JSSHELL_NAME) \ --dest-file '$(MOZ_TEST_PACKAGES_FILE)' \ $(call PKG_ARG,common,zip) \ - $(foreach pkg,$(TEST_PKGS_TARGZ),$(call PKG_ARG,$(pkg),tar.gz)) + $(foreach pkg,$(TEST_PKGS_TARZST),$(call PKG_ARG,$(pkg),tar.zst)) ifdef UPLOAD_PATH test_archive_dir = $(UPLOAD_PATH) @@ -158,7 +158,7 @@ package-tests-$(1): stage-all package-tests-prepare-dest download-wpt-manifest package-tests: package-tests-$(1) endef -$(foreach name,$(TEST_PKGS_TARGZ),$(eval $(call package_archive,$(name),tar.gz))) +$(foreach name,$(TEST_PKGS_TARZST),$(eval $(call package_archive,$(name),tar.zst))) ifeq ($(MOZ_BUILD_APP),mobile/android) stage-all: stage-android diff --git a/toolkit/mozapps/installer/package-name.mk b/toolkit/mozapps/installer/package-name.mk @@ -97,15 +97,15 @@ MOZSEARCH_JAVA_INDEX_BASENAME = $(PKG_BASENAME).mozsearch-java-index MOZHARNESS_PACKAGE = mozharness.zip # Test package naming -TEST_PACKAGE = $(PKG_BASENAME).common.tests.tar.gz -CPP_TEST_PACKAGE = $(PKG_BASENAME).cppunittest.tests.tar.gz -XPC_TEST_PACKAGE = $(PKG_BASENAME).xpcshell.tests.tar.gz -MOCHITEST_PACKAGE = $(PKG_BASENAME).mochitest.tests.tar.gz -REFTEST_PACKAGE = $(PKG_BASENAME).reftest.tests.tar.gz -WP_TEST_PACKAGE = $(PKG_BASENAME).web-platform.tests.tar.gz -TALOS_PACKAGE = $(PKG_BASENAME).talos.tests.tar.gz -AWSY_PACKAGE = $(PKG_BASENAME).awsy.tests.tar.gz -GTEST_PACKAGE = $(PKG_BASENAME).gtest.tests.tar.gz +TEST_PACKAGE = $(PKG_BASENAME).common.tests.tar.zst +CPP_TEST_PACKAGE = $(PKG_BASENAME).cppunittest.tests.tar.zst +XPC_TEST_PACKAGE = $(PKG_BASENAME).xpcshell.tests.tar.zst +MOCHITEST_PACKAGE = $(PKG_BASENAME).mochitest.tests.tar.zst +REFTEST_PACKAGE = $(PKG_BASENAME).reftest.tests.tar.zst +WP_TEST_PACKAGE = $(PKG_BASENAME).web-platform.tests.tar.zst +TALOS_PACKAGE = $(PKG_BASENAME).talos.tests.tar.zst +AWSY_PACKAGE = $(PKG_BASENAME).awsy.tests.tar.zst +GTEST_PACKAGE = $(PKG_BASENAME).gtest.tests.tar.zst # `.xpt` artifacts: for use in artifact builds. XPT_ARTIFACTS_ARCHIVE_BASENAME = $(PKG_BASENAME).xpt_artifacts diff --git a/tools/fuzzing/smoke/smoke.py b/tools/fuzzing/smoke/smoke.py @@ -7,11 +7,11 @@ This script can be used to perform simple calls using `jsshell` or whatever other tools you may add. The call is done via `taskcluster/kinds/fuzzing/kind.yml` and -files contained in the `target.jsshell.zip` and `target.fuzztest.tests.tar.gz` +files contained in the `target.jsshell.zip` and `target.fuzztest.tests.tar.zst` build artifacts are downloaded to run things. Everything included in this directory will be added in -`target.fuzztest.tests.tar.gz` at build time, so you can add more scripts and +`target.fuzztest.tests.tar.zst` at build time, so you can add more scripts and tools if you need. They will be located in `$MOZ_FETCHES_DIR` and follow the same directory structure than the source tree. """