commit 6e1e4fa31c44ca6005870799699332980257207a
parent 74bbf1e7d5220ba08ac7ebe1a3c1bca4b6a7c199
Author: Alex Hochheiden <ahochheiden@mozilla.com>
Date: Thu, 4 Dec 2025 17:28:43 +0000
Bug 2001450 - Centralize `nimbus-fml` archive caching r=nalexander,geckoview-reviewers
- Also added checksum verification and cache validation for downloaded `nimbus-fml` archives
Differential Revision: https://phabricator.services.mozilla.com/D273496
Diffstat:
2 files changed, 121 insertions(+), 22 deletions(-)
diff --git a/mobile/android/gradle/plugins/nimbus-gradle-plugin/src/main/groovy/org/mozilla/appservices/tooling/nimbus/NimbusAssembleToolsTask.groovy b/mobile/android/gradle/plugins/nimbus-gradle-plugin/src/main/groovy/org/mozilla/appservices/tooling/nimbus/NimbusAssembleToolsTask.groovy
@@ -27,6 +27,8 @@ import javax.inject.Inject
import groovy.transform.Immutable
+import java.security.MessageDigest
+
/**
* A task that fetches a prebuilt `nimbus-fml` binary for the current platform.
*
@@ -56,14 +58,14 @@ abstract class NimbusAssembleToolsTask extends DefaultTask {
abstract UnzipSpec getUnzipSpec()
/** The location of the fetched ZIP archive. */
- @LocalState
+ @Internal
abstract RegularFileProperty getArchiveFile()
/**
* The location of the fetched hash file, which contains the
* archive's checksum.
*/
- @LocalState
+ @Internal
abstract RegularFileProperty getHashFile()
/** The location of the unzipped binary. */
@@ -82,6 +84,10 @@ abstract class NimbusAssembleToolsTask extends DefaultTask {
@Internal
abstract Property<Integer> getReadTimeout()
+ /** The cache root directory */
+ @Internal
+ abstract Property<File> getCacheRoot()
+
NimbusAssembleToolsTask() {
platform.convention(detectPlatform(providers))
connectTimeout.convention(30000)
@@ -145,6 +151,10 @@ abstract class NimbusAssembleToolsTask extends DefaultTask {
@TaskAction
void assembleTools() {
+ def binaryFile = fmlBinary.get().asFile
+ def archiveFileObj = archiveFile.get().asFile
+ def hashFileObj = hashFile.get().asFile
+
def sources = [fetchSpec, *fetchSpec.fallbackSources.get()].collect {
new Source(
new URI(it.archive.get()),
@@ -154,30 +164,84 @@ abstract class NimbusAssembleToolsTask extends DefaultTask {
)
}
- def successfulSource = sources.find { it.trySaveArchiveTo(archiveFile.get().asFile) }
+ // Check if we have valid cached files by verifying against source hashes
+ def cachedHash = hashFileObj.exists() ? hashFileObj.text.trim() : null
+ if (cachedHash) {
+ for (source in sources) {
+ try {
+ def sourceHash = source.fetchHashString()
+ if (cachedHash.equalsIgnoreCase(sourceHash)) {
+ // Hash matches. Use cached binary if it exists, otherwise extract from archive
+ if (binaryFile.exists()) {
+ logger.info("nimbus-fml binary is up-to-date")
+ return
+ }
+ if (archiveFileObj.exists()) {
+ logger.info("Extracting nimbus-fml binary from cached archive")
+ extractBinary(archiveFileObj)
+ return
+ }
+ // We have a hash file, but neither binary nor archive, so we need to fetch the archive
+ break
+ }
+ } catch (IOException ignored) {
+ // Try next source
+ }
+ }
+ }
+
+ logger.info("Fetching nimbus-fml for platform: {}", platform.get())
+
+ // Clear cache before downloading a new archive
+ if (cacheRoot.isPresent()) {
+ def root = cacheRoot.get()
+ if (root.exists()) {
+ root.deleteDir()
+ }
+ }
+
+ // Download the archive and verify with hash from the same source
+ Source successfulSource = null
+ String sourceHash = null
+ for (source in sources) {
+ try {
+ sourceHash = source.fetchHashString()
+ } catch (IOException ignored) {
+ continue
+ }
+
+ if (source.trySaveArchiveTo(archiveFileObj)) {
+ successfulSource = source
+ break
+ }
+ }
+
if (successfulSource == null) {
- throw new GradleException("Couldn't fetch archive from any of: ${sources*.archiveURI.collect { "`$it`" }.join(', ')}")
+ throw new GradleException("Failed to fetch archive from any of: ${sources*.archiveURI.collect { "`$it`" }.join(', ')}")
}
- // We get the checksum, although don't do anything with it yet;
- // Checking it here would be able to detect if the zip file was tampered with
- // in transit between here and the server.
- // It won't detect compromise of the CI server.
- try {
- successfulSource.saveHashTo(hashFile.get().asFile)
- } catch (IOException e) {
- throw new GradleException("Couldn't fetch hash from `${successfulSource.hashURI}`", e)
+ def actualHash = computeSha256(archiveFileObj)
+ if (!actualHash.equalsIgnoreCase(sourceHash)) {
+ archiveFileObj.delete()
+ throw new GradleException("Archive checksum mismatch! Expected: $sourceHash, got: $actualHash")
}
+ hashFileObj.text = sourceHash
- def zipTree = archiveOperations.zipTree(archiveFile.get())
+ extractBinary(archiveFileObj)
+ }
+
+ private void extractBinary(File archiveFileObj) {
+ def binaryFile = fmlBinary.get().asFile
+ def zipTree = archiveOperations.zipTree(archiveFileObj)
def visitedFilePaths = []
zipTree.matching {
include unzipSpec.includePatterns.get()
}.visit { FileVisitDetails details ->
if (!details.directory) {
if (visitedFilePaths.empty) {
- details.copyTo(fmlBinary.get().asFile)
- fmlBinary.get().asFile.setExecutable(true)
+ binaryFile.parentFile?.mkdirs()
+ details.copyTo(binaryFile)
+ binaryFile.setExecutable(true)
}
visitedFilePaths.add(details.relativePath)
}
@@ -192,6 +256,18 @@ abstract class NimbusAssembleToolsTask extends DefaultTask {
}
}
+ private static String computeSha256(File file) {
+ def digest = MessageDigest.getInstance("SHA-256")
+ file.withInputStream { is ->
+ byte[] buffer = new byte[8192]
+ int read
+ while ((read = is.read(buffer)) != -1) {
+ digest.update(buffer, 0, read)
+ }
+ }
+ return digest.digest().encodeHex().toString()
+ }
+
/**
* Specifies the source from which to fetch the archive and
* its hash file.
@@ -270,6 +346,25 @@ abstract class NimbusAssembleToolsTask extends DefaultTask {
saveURITo(hashURI, destination)
}
+ String fetchHashString() {
+ def connection = hashURI.toURL().openConnection() as HttpURLConnection
+ connection.connectTimeout = connectTimeout
+ connection.readTimeout = readTimeout
+ connection.instanceFollowRedirects = true
+ connection.requestMethod = 'GET'
+
+ try {
+ if (connection.responseCode != 200) {
+ throw new IOException("HTTP ${connection.responseCode}: ${connection.responseMessage}")
+ }
+ return connection.inputStream.withStream { is ->
+ is.text.trim().split(/\s+/)[0]
+ }
+ } finally {
+ connection.disconnect()
+ }
+ }
+
private void saveURITo(URI source, File destination) {
def connection = source.toURL().openConnection() as HttpURLConnection
connection.connectTimeout = connectTimeout
diff --git a/mobile/android/gradle/plugins/nimbus-gradle-plugin/src/main/groovy/org/mozilla/appservices/tooling/nimbus/NimbusGradlePlugin.groovy b/mobile/android/gradle/plugins/nimbus-gradle-plugin/src/main/groovy/org/mozilla/appservices/tooling/nimbus/NimbusGradlePlugin.groovy
@@ -188,20 +188,24 @@ class NimbusPlugin implements Plugin<Project> {
def setupAssembleNimbusTools(Project project) {
def applicationServicesDir = project.nimbus.applicationServicesDir
def asVersionProvider = getProjectVersionProvider(project)
+ def projectDir = project.layout.projectDirectory
return project.tasks.register('assembleNimbusTools', NimbusAssembleToolsTask) { task ->
group "Nimbus"
description "Fetch the Nimbus FML tools from Application Services"
- def fmlRoot = project.layout.buildDirectory.dir(asVersionProvider.map { version ->
- "bin/nimbus/$version"
- })
+ def cacheDir = asVersionProvider.map { version ->
+ projectDir.dir(".gradle/caches/nimbus-fml/$version")
+ }
- archiveFile = fmlRoot.map { it.file('nimbus-fml.zip') }
- hashFile = fmlRoot.map { it.file('nimbus-fml.sha256') }
- fmlBinary = fmlRoot.zip(platform) { root, plat ->
- root.file(NimbusAssembleToolsTask.getBinaryName(plat))
+ archiveFile = cacheDir.map { it.file('nimbus-fml.zip') }
+ hashFile = cacheDir.map { it.file('nimbus-fml.sha256') }
+ fmlBinary = project.layout.buildDirectory.flatMap { buildDir ->
+ asVersionProvider.zip(platform) { version, plat ->
+ buildDir.dir("bin/nimbus/$version").file(NimbusAssembleToolsTask.getBinaryName(plat))
+ }
}
+ cacheRoot = projectDir.dir(".gradle/caches/nimbus-fml").asFile
fetch {
// Try archive.mozilla.org release first
