commit 6cb60358701289240ec6849d21b8768ef4de4215
parent 3d53db66e8166d817968c9c4b7d6f496b44ec595
Author: Beatriz Rizental <beatriz.rizental@gmail.com>
Date: Wed, 19 Jun 2024 09:46:19 +0200
Add CI for Base Browser
Diffstat:
7 files changed, 336 insertions(+), 1 deletion(-)
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
@@ -0,0 +1,14 @@
+stages:
+ - lint
+ - test
+ - update-translations
+
+variables:
+ IMAGE_PATH: containers.torproject.org/tpo/applications/tor-browser/base:latest
+ LOCAL_REPO_PATH: /srv/apps-repos/tor-browser.git
+
+include:
+ - local: '.gitlab/ci/mixins.yml'
+ - local: '.gitlab/ci/jobs/lint/lint.yml'
+ - local: '.gitlab/ci/jobs/test/python-test.yml'
+ - local: '.gitlab/ci/jobs/update-translations.yml'
diff --git a/.gitlab/ci/jobs/helpers.py b/.gitlab/ci/jobs/helpers.py
@@ -0,0 +1,129 @@
+#!/usr/bin/env python3
+
+import argparse
+import os
+import re
+import shlex
+import subprocess
+
+
+def git(command):
+ result = subprocess.run(
+ ["git"] + shlex.split(command), check=True, capture_output=True, text=True
+ )
+ return result.stdout.strip()
+
+
+def get_firefox_tag(reference):
+ """Extracts the Firefox tag associated with a branch or tag name.
+
+ The "firefox tag" is the tag that marks
+ the end of the Mozilla commits and the start of the Tor Project commits.
+
+ Know issue: If ever there is more than one tag per Firefox ESR version,
+ this function may return the incorrect reference number.
+
+ Args:
+ reference: The branch or tag name to extract the Firefox tag from.
+ Expected format is tor-browser-91.2.0esr-11.0-1,
+ where 91.2.0esr is the Firefox version.
+
+ Returns:
+ The reference specifier of the matching Firefox tag.
+ An exception will be raised if anything goes wrong.
+ """
+
+ # Extracts the version number from a branch or tag name.
+ firefox_version = ""
+ match = re.search(r"(?<=browser-)([^-]+)", reference)
+ if match:
+ # TODO: Validate that what we got is actually a valid semver string?
+ firefox_version = match.group(1)
+ else:
+ raise ValueError(f"Failed to extract version from reference '{reference}'.")
+
+ major_version = firefox_version.split(".")[0]
+ minor_patch_version = "_".join(firefox_version.split(".")[1:])
+
+ remote_tags = git("ls-remote --tags origin")
+
+ # Each line looks like:
+ # 9edd658bfd03a6b4743ecb75fd4a9ad968603715 refs/tags/FIREFOX_91_9_0esr_BUILD1
+ pattern = (
+ rf"(.*)FIREFOX_{re.escape(major_version)}_{re.escape(minor_patch_version)}(.*)$"
+ )
+ match = re.search(pattern, remote_tags, flags=re.MULTILINE)
+ if not match:
+ # Attempt to match with a nightly tag, in case the ESR tag is not found
+ pattern = rf"(.*)FIREFOX_NIGHTLY_{re.escape(major_version)}(.*)$"
+ match = re.search(pattern, remote_tags, flags=re.MULTILINE)
+
+ if match:
+ return match.group(0).split()[0]
+ else:
+ raise ValueError(
+ f"Failed to find reference specifier for Firefox tag of version '{firefox_version}' from '{reference}'."
+ )
+
+
+def get_list_of_changed_files():
+ """Gets a list of files changed in the working directory.
+
+ This function is meant to be run inside the Gitlab CI environment.
+
+ When running in a default branch, get the list of changed files since the last Firefox tag.
+ When running for a new MR commit, get a list of changed files in the current MR.
+
+ Returns:
+ A list of filenames of changed files (excluding deleted files).
+ An exception wil be raised if anything goes wrong.
+ """
+
+ base_reference = ""
+
+ if os.getenv("CI_PIPELINE_SOURCE") == "merge_request_event":
+ # For merge requests, the base_reference is the common ancestor between the MR and the target branch
+ base_reference = os.getenv("CI_MERGE_REQUEST_DIFF_BASE_SHA")
+ if not base_reference:
+ # Probably because there has been no overall change.
+ # See gitlab.com/gitlab-org/gitlab/-/issues/375047#note_2648459916
+ return []
+ else:
+ # When not in merge requests, the base reference is the Firefox tag
+ base_reference = get_firefox_tag(os.getenv("CI_COMMIT_BRANCH"))
+
+ if not base_reference:
+ raise RuntimeError("No base reference found. There might be more errors above.")
+
+ # Fetch the tag reference
+ git(f"fetch origin {base_reference} --depth=1 --filter=blob:none")
+ # Return but filter the issue_templates files because those file names have spaces which can cause issues
+ return git("diff --diff-filter=d --name-only FETCH_HEAD HEAD").split("\n")
+
+
+if __name__ == "__main__":
+ parser = argparse.ArgumentParser(description="")
+
+ parser.add_argument(
+ "--get-firefox-tag",
+ help="Get the Firefox tag related to a given (tor-mullvad-base)-browser tag or branch name.",
+ type=str,
+ )
+ parser.add_argument(
+ "--get-changed-files",
+ help="Get list of changed files."
+ "When running from a merge request gets the list of changed files since the merge-base of the current branch."
+ "When running from a protected branch i.e. any branch that starts with <something>-browser-, gets the list of files changed since the FIREFOX_ tag.",
+ action="store_true",
+ )
+
+ args = parser.parse_args()
+
+ if args.get_firefox_tag:
+ print(get_firefox_tag(args.get_firefox_tag))
+ elif args.get_changed_files:
+ # Separate the file names with a 0 byte to be parsed by xargs -0. Also
+ # drop the trailing '\n'.
+ print("\0".join(get_list_of_changed_files()), end="")
+ else:
+ print("No valid option provided.")
diff --git a/.gitlab/ci/jobs/lint/lint.yml b/.gitlab/ci/jobs/lint/lint.yml
@@ -0,0 +1,25 @@
+lint-all:
+ extends: .with-local-repo-bash
+ stage: lint
+ image: $IMAGE_PATH
+ interruptible: true
+ variables:
+ # Has to be the same as defined in `containers/base/Containerfile`
+ MOZBUILD_STATE_PATH: "/var/tmp/mozbuild"
+ cache:
+ paths:
+ - node_modules
+ # Store the cache regardless on job outcome
+ when: 'always'
+ # Share the cache throughout all pipelines running for a given branch
+ key: $CI_COMMIT_REF_SLUG
+ tags:
+ # Run these jobs in the browser dedicated runners.
+ - firefox
+ script:
+ - ./mach configure --with-base-browser-version=0.0.0
+ - .gitlab/ci/jobs/helpers.py --get-changed-files | xargs -0 --no-run-if-empty ./mach lint -v
+ rules:
+ - if: $CI_PIPELINE_SOURCE == 'merge_request_event'
+ # Run job whenever a commit is merged to a protected branch
+ - if: ($CI_COMMIT_BRANCH && $CI_COMMIT_REF_PROTECTED == 'true' && $CI_PIPELINE_SOURCE == 'push')
diff --git a/.gitlab/ci/jobs/test/python-test.yml b/.gitlab/ci/jobs/test/python-test.yml
@@ -0,0 +1,24 @@
+python-test:
+ extends: .with-local-repo-bash
+ stage: test
+ image: $IMAGE_PATH
+ interruptible: true
+ variables:
+ MOZBUILD_STATE_PATH: "/var/tmp/mozbuild"
+ cache:
+ paths:
+ - node_modules
+ # Store the cache regardless on job outcome
+ when: 'always'
+ # Share the cache throughout all pipelines running for a given branch
+ key: $CI_COMMIT_REF_SLUG
+ tags:
+ # Run these jobs in the browser dedicated runners.
+ - firefox
+ script:
+ - ./mach configure --with-base-browser-version=0.0.0
+ - ./mach python-test --subsuite base-browser
+ rules:
+ - if: $CI_PIPELINE_SOURCE == 'merge_request_event' || ($CI_COMMIT_BRANCH && $CI_COMMIT_REF_PROTECTED == 'true' && $CI_PIPELINE_SOURCE == 'push')
+ changes:
+ - "**/test_*.py"
diff --git a/.gitlab/ci/jobs/update-translations.yml b/.gitlab/ci/jobs/update-translations.yml
@@ -0,0 +1,52 @@
+.update-translation-base:
+ stage: update-translations
+ rules:
+ - if: ($TRANSLATION_FILES != "" && $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH && $CI_PIPELINE_SOURCE == "push")
+ changes:
+ - "**/*.ftl"
+ - "**/*.properties"
+ - "**/*.dtd"
+ - "**/*strings.xml"
+ - "**/update-translations.yml"
+ - "**/l10n/combine/combine.py"
+ - "**/l10n/combine-translation-versions.py"
+ - if: ($TRANSLATION_FILES != "" && $FORCE_UPDATE_TRANSLATIONS == "true")
+ variables:
+ COMBINED_FILES_JSON: "combined-translation-files.json"
+ TRANSLATION_FILES: ''
+
+
+combine-en-US-translations:
+ extends:
+ - .with-local-repo-bash
+ - .update-translation-base
+ needs: []
+ image: $IMAGE_PATH
+ # Artifact is for translation project job
+ artifacts:
+ paths:
+ - "$COMBINED_FILES_JSON"
+ expire_in: "60 min"
+ reports:
+ dotenv: job_id.env
+ # Don't load artifacts for this job.
+ dependencies: []
+ tags:
+ # Run these jobs in the browser dedicated runners.
+ - firefox
+ script:
+ # Save this CI_JOB_ID to the dotenv file to be used in the variables for the
+ # push-en-US-translations job.
+ - echo 'COMBINE_TRANSLATIONS_JOB_ID='"$CI_JOB_ID" >job_id.env
+ - ./mach python ./tools/base_browser/l10n/combine-translation-versions.py "$CI_COMMIT_BRANCH" "$TRANSLATION_FILES" "$COMBINED_FILES_JSON"
+
+push-en-US-translations:
+ extends: .update-translation-base
+ needs:
+ - job: combine-en-US-translations
+ variables:
+ COMBINED_FILES_JSON_URL: "${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/jobs/${COMBINE_TRANSLATIONS_JOB_ID}/artifacts/${COMBINED_FILES_JSON}"
+ trigger:
+ strategy: depend
+ project: tor-browser-translation-bot/translation
+ branch: tor-browser-ci
diff --git a/.gitlab/ci/mixins.yml b/.gitlab/ci/mixins.yml
@@ -0,0 +1,91 @@
+.with-local-repo-bash:
+ variables:
+ GIT_STRATEGY: "none"
+ FETCH_TIMEOUT: 180 # 3 minutes
+ before_script:
+ - git init
+ - git remote add local "$LOCAL_REPO_PATH"
+ - |
+ # Determine the reference of the target branch in the local repository copy.
+ #
+ # 1. List all references in the local repository
+ # 2. Filter the references to the target branch
+ # 3. Remove tags
+ # 4. Keep a single line, in case there are too many matches
+ # 5. Clean up the output
+ # 6. Remove everything before the last two slashes, because the output is like `refs/heads/...` or `refs/remotes/...`
+ TARGET_BRANCH=$(git ls-remote local | grep ${CI_COMMIT_BRANCH:-$CI_MERGE_REQUEST_TARGET_BRANCH_NAME} | grep -v 'refs/tags/' | awk '{print $2}' | tail -1 | sed 's|[^/]*/[^/]*/||')
+ if [ -z "$TARGET_BRANCH" ]; then
+ echo "Target branch $TARGET_BRANCH is not yet in local repository. Stopping the pipeline."
+ exit 1
+ fi
+ - git fetch --depth 500 local $TARGET_BRANCH
+ - git --no-pager log FETCH_HEAD --oneline -n 5
+ - git remote add origin "$CI_REPOSITORY_URL"
+ - |
+ if [ -z "${CI_COMMIT_BRANCH:-$CI_MERGE_REQUEST_SOURCE_BRANCH_NAME}" ]; then
+ echo "No branch specified. Stopping the pipeline."
+ exit 1
+ fi
+ - echo "Fetching from remote branch ${CI_COMMIT_BRANCH:-$CI_MERGE_REQUEST_SOURCE_BRANCH_NAME} with a ${FETCH_TIMEOUT}s timeout."
+ - |
+ fetch_with_timeout() {
+ local remote=$1
+ local branch=$2
+
+ set +e
+ timeout ${FETCH_TIMEOUT} git fetch "$remote" "$branch"
+ local fetch_exit=$?
+ set -e
+
+ if [ "$fetch_exit" -eq 124 ]; then
+ echo "Fetching failed for branch ${remote}/${branch} due to a timeout. Try again later."
+ echo "Gitlab may be experiencing slowness or the local copy of the repository on the CI server may be oudated."
+ return 1
+ fi
+
+ return $fetch_exit
+ }
+
+ if ! fetch_with_timeout origin "${CI_COMMIT_BRANCH:-$CI_MERGE_REQUEST_SOURCE_BRANCH_NAME}"; then
+ echo "Fetching failed for branch ${CI_COMMIT_BRANCH:-$CI_MERGE_REQUEST_SOURCE_BRANCH_NAME}."
+ echo "Attempting to fetch the merge request branch, assuming this pipeline is not running in a fork."
+
+ fetch_with_timeout origin "merge-requests/${CI_MERGE_REQUEST_IID}/head" || exit 1
+ fi
+ - git checkout FETCH_HEAD
+
+.with-local-repo-pwsh:
+ variables:
+ GIT_STRATEGY: "none"
+ before_script:
+ - git init
+ - git remote add local $env:LOCAL_REPO_PATH
+ - |
+ $branchName = $env:CI_COMMIT_BRANCH
+ if ([string]::IsNullOrEmpty($branchName)) {
+ $branchName = $env:CI_MERGE_REQUEST_TARGET_BRANCH_NAME
+ }
+ $TARGET_BRANCH = git ls-remote local | Select-String -Pattern $branchName | Select-String -Pattern -NotMatch 'refs/tags/' | Select-Object -Last 1 | ForEach-Object { $_.ToString().Split()[1] -replace '^[^/]*/[^/]*/', '' }
+ if ([string]::IsNullOrEmpty($TARGET_BRANCH)) {
+ Write-Output "Target branch $TARGET_BRANCH is not yet in local repository. Stopping the pipeline."
+ exit 1
+ }
+ - git remote add origin $env:CI_REPOSITORY_URL
+ - |
+ $branchName = $env:CI_COMMIT_BRANCH
+ if ([string]::IsNullOrEmpty($branchName)) {
+ $branchName = $env:CI_MERGE_REQUEST_SOURCE_BRANCH_NAME
+ }
+ if ([string]::IsNullOrEmpty($branchName)) {
+ Write-Output "No branch specified. Stopping the pipeline."
+ exit 1
+ }
+ - Write-Output "Fetching from remote branch $branchName"
+ - |
+ if (! git fetch origin $branchName) {
+ Write-Output "Fetching failed for branch $branchName from $env:CI_REPOSITORY_URL."
+ Write-Output "Attempting to fetch the merge request branch, assuming this pipeline is not running in a fork."
+ git fetch origin "merge-requests/$env:CI_MERGE_REQUEST_IID/head"
+ }
+ - git checkout FETCH_HEAD
diff --git a/eslint.config.mjs b/eslint.config.mjs
@@ -434,7 +434,7 @@ let config = [
ignores: ["toolkit/**/test/**", "toolkit/**/tests/**"],
plugins: { mozilla },
rules: {
- "mozilla/no-browser-refs-in-toolkit": "error",
+ "mozilla/no-browser-refs-in-toolkit": "warn",
},
},
{
