commit 4e51ed50ec463e2ebd76032e6cbf3a11ccd8c722
parent 481dc19ee50c025dca22de274ca838a8ae0d1114
Author: Atila Butkovits <abutkovits@mozilla.com>
Date: Wed, 22 Oct 2025 17:24:14 +0300
Revert "Bug 1994352: download and extract files in mozharness in parallel r=jmaher,releng-reviewers,jcristau" for causing Bug 1995767 and 1995571.
This reverts commit 162aa79a73d16d9ee605e18756a1f223a0cf5723.
Diffstat:
1 file changed, 27 insertions(+), 57 deletions(-)
diff --git a/testing/mozharness/mozharness/mozilla/testing/testbase.py b/testing/mozharness/mozharness/mozilla/testing/testbase.py
@@ -3,13 +3,11 @@
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
# You can obtain one at http://mozilla.org/MPL/2.0/.
-import concurrent.futures
import copy
import json
import os
import platform
import ssl
-import traceback
from six.moves import urllib
from six.moves.urllib.parse import ParseResult, urlparse
@@ -387,8 +385,7 @@ You can set this by specifying --test-url URL
)
return package_requirements
- def _download_test_packages(self, suite_categories, extract_dirs, executor):
- futures = set()
+ def _download_test_packages(self, suite_categories, extract_dirs):
# Some platforms define more suite categories/names than others.
# This is a difference in the convention of the configs more than
# to how these tests are run, so we pave over these differences here.
@@ -472,13 +469,7 @@ You can set this by specifying --test-url URL
unpack_dirs = None
url = self.query_build_dir_url(file_name)
- futures.add(
- executor.submit(
- self.download_unpack, url, target_dir, extract_dirs=unpack_dirs
- )
- )
-
- return futures
+ self.download_unpack(url, target_dir, extract_dirs=unpack_dirs)
def _download_test_zip(self, extract_dirs=None):
dirs = self.query_abs_dirs()
@@ -576,54 +567,33 @@ You can set this by specifying --test-url URL
self.info("Replacing url %s -> %s" % (url, new_url))
setattr(self, attr, new_url)
- futures = set()
- with concurrent.futures.ThreadPoolExecutor() as executor:
- if "test_url" in self.config:
- # A user has specified a test_url directly, any test_packages_url will
- # be ignored.
- if self.test_packages_url:
- self.error(
- 'Test data will be downloaded from "%s", the specified test '
- ' package data at "%s" will be ignored.'
- % (self.config.get("test_url"), self.test_packages_url)
- )
-
- futures.add(executor.submit(self._download_test_zip, extract_dirs))
- else:
- if not self.test_packages_url:
- # The caller intends to download harness specific packages, but doesn't know
- # where the packages manifest is located. This is the case when the
- # test package manifest isn't set as a property, which is true
- # for some self-serve jobs and platforms using parse_make_upload.
- self.test_packages_url = self.query_prefixed_build_dir_url(
- ".test_packages.json"
- )
-
- suite_categories = suite_categories or ["common"]
- futures.update(
- self._download_test_packages(
- suite_categories, extract_dirs, executor
- )
+ if "test_url" in self.config:
+ # A user has specified a test_url directly, any test_packages_url will
+ # be ignored.
+ if self.test_packages_url:
+ self.error(
+ 'Test data will be downloaded from "%s", the specified test '
+ ' package data at "%s" will be ignored.'
+ % (self.config.get("test_url"), self.test_packages_url)
)
- futures.add(executor.submit(self._download_installer))
- if self.config.get("download_symbols"):
- futures.add(executor.submit(self._download_and_extract_symbols))
-
- concurrent.futures.wait(futures)
-
- exceptions = []
- for future in futures:
- if exc := future.exception():
- exceptions.append(exc)
- if exceptions:
- # It looks weird to be adding our own `\n` here but it's necessary
- # because `format_exception` does so, so we need to match it to be
- # able to call `join` later to format the final message properly.
- msg = ["Got download error(s):\n"]
- for exc in exceptions:
- msg.extend(traceback.format_exception(exc))
- self.fatal("".join(msg))
+ self._download_test_zip(extract_dirs)
+ else:
+ if not self.test_packages_url:
+ # The caller intends to download harness specific packages, but doesn't know
+ # where the packages manifest is located. This is the case when the
+ # test package manifest isn't set as a property, which is true
+ # for some self-serve jobs and platforms using parse_make_upload.
+ self.test_packages_url = self.query_prefixed_build_dir_url(
+ ".test_packages.json"
+ )
+
+ suite_categories = suite_categories or ["common"]
+ self._download_test_packages(suite_categories, extract_dirs)
+
+ self._download_installer()
+ if self.config.get("download_symbols"):
+ self._download_and_extract_symbols()
# create_virtualenv is in VirtualenvMixin.
