run.sh (7434B)
1 #!/bin/bash 2 3 set -ex 4 5 mkdir -p /builds/worker/artifacts/ 6 # for PGO logs 7 export UPLOAD_PATH=/builds/worker/artifacts/ 8 9 mkdir -p /builds/worker/.local/state/snapcraft/ 10 11 BRANCH=$1 12 ARCH=$2 13 DEBUG=${3:-0} 14 15 SOURCE_REPO=${SOURCE_REPO:-https://github.com/canonical/firefox-snap/} 16 SOURCE_BRANCH=${SOURCE_BRANCH:-${BRANCH}} 17 18 export LC_ALL=C.UTF-8 19 export LANG=C.UTF-8 20 export SNAP_ARCH=amd64 21 export SNAPCRAFT_BUILD_INFO=1 22 # Used to discern this Docker build, done by Mozilla, from the 'normal', 23 # production snap build. 24 export MOZ_SNAP_BUILD=1 25 26 export PATH=$PATH:$HOME/.local/bin/ 27 unset MOZ_AUTOMATION 28 29 MOZCONFIG=mozconfig.in 30 31 USE_SNAP_FROM_STORE_OR_MC=${USE_SNAP_FROM_STORE_OR_MC:-0} 32 33 sudo mkdir -p /run/user/1000 && sudo chown 1000:1000 /run/user/1000 34 35 TRY=0 36 if [ "${BRANCH}" = "try" ]; then 37 if [ "${SOURCE_BRANCH}" = "try" ]; then 38 # This would happen when e.g.: mach try fuzzy -q "'snap 'build 'try" 39 # In this case we want to default to nightly from upstream, except if the 40 # user passes a SOURCE_BRANCH, with e.g.: 41 # SOURCE_REPO=... SOURCE_BRANCH=... mach try fuzzy -q "'snap 'build 'try" 42 SOURCE_BRANCH=nightly 43 fi 44 TRY=1 45 fi 46 47 if [ "${USE_SNAP_FROM_STORE_OR_MC}" = "0" ]; then 48 # ESR currently still has a hard dependency against zstandard==0.17.0 so 49 # install this specific version here 50 if [ "${BRANCH}" = "esr" ]; then 51 sudo apt-get remove -y python3-zstandard && sudo apt-get install -y python3-pip && sudo pip3 install --no-input zstandard==0.17.0 52 MOZCONFIG=mozconfig 53 fi 54 55 # Stable and beta runs out of file descriptors during link with gold 56 ulimit -n 65536 57 58 git clone --single-branch --depth 1 --branch "${SOURCE_BRANCH}" "${SOURCE_REPO}" firefox-snap/ 59 cd firefox-snap/ 60 61 if [ "${TRY}" = "1" ]; then 62 # Symlink so that we can directly re-use Gecko mercurial checkout 63 ln -s /builds/worker/checkouts/gecko gecko 64 fi 65 66 # Force an update to avoid the case of a stale docker image and repos updated 67 # after 68 sudo apt-get update 69 70 # shellcheck disable=SC2046 71 sudo apt-get install -y $(/usr/bin/python3 /builds/worker/parse.py snapcraft.yaml "${ARCH}") 72 73 if [ -d "/builds/worker/patches/${BRANCH}/" ]; then 74 for p in /builds/worker/patches/"${BRANCH}"/*.patch; do 75 patch -p1 < "$p" 76 done; 77 fi 78 79 if [ "${TRY}" = "1" ]; then 80 # don't remove hg source, and don't force changeset so we get correct stamp 81 # still force repo because the try clone is from mozilla-unified but the 82 # generated link does not work 83 sed -ri 's|rm -rf .hg||g' snapcraft.yaml 84 # shellcheck disable=SC2016 85 sed -ri 's|MOZ_SOURCE_REPO=\$\{REPO\}|MOZ_SOURCE_REPO=${GECKO_HEAD_REPOSITORY}|g' snapcraft.yaml 86 # shellcheck disable=SC2016 87 sed -ri 's|MOZ_SOURCE_CHANGESET=\$\{REVISION\}|MOZ_SOURCE_CHANGESET=${GECKO_HEAD_REV}|g' snapcraft.yaml 88 # shellcheck disable=SC2016 89 sed -ri 's|hg clone --stream \$REPO -u \$REVISION|cp -r \$SNAPCRAFT_PROJECT_DIR/gecko/. |g' snapcraft.yaml 90 fi 91 92 if [ "${DEBUG}" = "--debug" ]; then 93 { 94 echo "ac_add_options --enable-debug" 95 echo "ac_add_options --disable-install-strip" 96 } >> ${MOZCONFIG} 97 echo "MOZ_DEBUG=1" >> ${MOZCONFIG} 98 99 # No PGO on debug builds 100 sed -ri 's/ac_add_options --enable-linker=gold//g' snapcraft.yaml 101 sed -ri 's/ac_add_options --enable-lto=cross//g' snapcraft.yaml 102 sed -ri 's/ac_add_options MOZ_PGO=1//g' snapcraft.yaml 103 fi 104 105 # Until launchpad is able to handle platforms definition, the snapcraft yaml 106 # hides them and we want to unhide. 107 sed -ri 's/^##CROSS-COMPILATION##//g' snapcraft.yaml 108 109 MAX_MEMORY_GB=$(free -g | awk '/Mem:/ { print $2 - 1 }') 110 111 # setting parallelism does not work with core24 ? 112 # 113 # SNAPCRAFT_BUILD_ENVIRONMENT_CPU=$(nproc) \ 114 # SNAPCRAFT_PARALLEL_BUILD_COUNT=$(nproc) \ 115 # CRAFT_PARALLEL_BUILD_COUNT=$(nproc) \ 116 117 # Get the value and overwrite the snap's content. 118 MAX_CPUS=$(nproc) 119 sed -ri "s|\\\$CRAFT_PARALLEL_BUILD_COUNT|${MAX_CPUS}|g" snapcraft.yaml 120 grep "MACH build .*-j${MAX_CPUS}" snapcraft.yaml 121 122 SNAPCRAFT_BUILD_ENVIRONMENT_MEMORY="${MAX_MEMORY_GB}G" \ 123 snapcraft --destructive-mode --verbosity verbose --build-for "${ARCH}" 124 elif [ "${USE_SNAP_FROM_STORE_OR_MC}" = "store" ]; then 125 mkdir from-snap-store && cd from-snap-store 126 127 CHANNEL="${BRANCH}" 128 if [ "${CHANNEL}" = "try" ] || [ "${CHANNEL}" = "nightly" ]; then 129 CHANNEL=edge 130 fi; 131 132 if [ "${CHANNEL}" = "stable-core24" ]; then 133 CHANNEL="latest/candidate" 134 fi 135 136 if [ "${CHANNEL}" = "beta-core24" ]; then 137 CHANNEL="latest/beta/core24" 138 fi 139 140 snap download --channel="${CHANNEL}" firefox 141 SNAP_DEBUG_NAME=$(find . -maxdepth 1 -type f -name "firefox*.snap" | sed -e 's/\.snap$/.debug/g') 142 touch "${SNAP_DEBUG_NAME}" 143 else 144 mkdir from-mc && cd from-mc 145 146 # index.gecko.v2.mozilla-central.latest.firefox.snap-amd64-esr-debug 147 # => https://firefox-ci-tc.services.mozilla.com/api/index/v1/task/gecko.v2.mozilla-central.latest.firefox.snap-amd64-esr-debug/artifacts/public%2Fbuild%2Ffirefox.snap 148 # index.gecko.v2.mozilla-central.revision.bf0897ec442e625c185407cc615a6adc0e40fa75.firefox.snap-amd64-esr-debug 149 # => https://firefox-ci-tc.services.mozilla.com/api/index/v1/task/gecko.v2.mozilla-central.revision.bf0897ec442e625c185407cc615a6adc0e40fa75.firefox.snap-amd64-esr-debug/artifacts/public%2Fbuild%2Ffirefox.snap 150 # index.gecko.v2.mozilla-central.latest.firefox.snap-amd64-nightly 151 # => https://firefox-ci-tc.services.mozilla.com/api/index/v1/task/gecko.v2.mozilla-central.latest.firefox.snap-amd64-nightly/artifacts/public%2Fbuild%2Ffirefox.snap 152 # index.gecko.v2.mozilla-central.revision.bf0897ec442e625c185407cc615a6adc0e40fa75.firefox.snap-amd64-nightly 153 # => https://firefox-ci-tc.services.mozilla.com/api/index/v1/task/gecko.v2.mozilla-central.revision.bf0897ec442e625c185407cc615a6adc0e40fa75.firefox.snap-amd64-nightly/artifacts/public%2Fbuild%2Ffirefox.snap 154 155 TASKCLUSTER_API_ROOT="https://firefox-ci-tc.services.mozilla.com/api" 156 if [ "${USE_SNAP_FROM_STORE_OR_MC}" != "task" ]; then 157 # Remove "-" so we get e.g., esr128 from esr-128 158 INDEX_NAME=${BRANCH//-/} 159 if [ "${INDEX_NAME}" = "try" ]; then 160 INDEX_NAME=nightly 161 fi; 162 163 if [ "${DEBUG}" = "--debug" ]; then 164 INDEX_NAME="${INDEX_NAME}-debug" 165 else 166 INDEX_NAME="${INDEX_NAME}-opt" 167 fi; 168 169 URL_TASK="${TASKCLUSTER_API_ROOT}/index/v1/task/gecko.v2.mozilla-central.${USE_SNAP_FROM_STORE_OR_MC}.firefox.snap-${ARCH}-${INDEX_NAME}" 170 PKGS_TASK_ID=$(curl "${URL_TASK}" | jq -r '.taskId') 171 172 if [ -z "${PKGS_TASK_ID}" ]; then 173 echo "Failure to find matching taskId for ${USE_SNAP_FROM_STORE_OR_MC} + ${INDEX_NAME}" 174 exit 1 175 fi 176 fi 177 178 PKGS_URL="${TASKCLUSTER_API_ROOT}/queue/v1/task/${PKGS_TASK_ID}/artifacts" 179 for pkg in $(curl "${PKGS_URL}" | jq -r '.artifacts | . [] | select(.name | contains("public/build/firefox_")) | .name'); 180 do 181 url="${TASKCLUSTER_API_ROOT}/queue/v1/task/${PKGS_TASK_ID}/artifacts/${pkg}" 182 target_name="$(basename "${pkg}")" 183 echo "$url => $target_name" 184 curl -SL "${url}" -o "${target_name}" 185 done; 186 fi 187 188 cp ./*.snap ./*.debug /builds/worker/artifacts/ 189 190 # Those are for fetches usage by the test task 191 cp ./*.snap /builds/worker/artifacts/firefox.snap 192 cp ./*.debug /builds/worker/artifacts/firefox.debug 193 194 # Those are for running snap-upstream-test 195 cd /builds/worker/checkouts/gecko/taskcluster/docker/snap-coreXX-build/snap-tests/ && zip -r9 /builds/worker/artifacts/snap-tests.zip ./*