periodic_file_updates.sh (25450B)
1 #!/bin/bash 2 3 set -ex 4 5 function usage { 6 cat <<EOF 7 8 Usage: $(basename "$0") -h # Displays this usage/help text 9 Usage: $(basename "$0") -x # lists exit codes 10 Usage: $(basename "$0") [-p product] 11 # Use mozilla-central builds to check HSTS & HPKP 12 [--use-mozilla-central] 13 # Use archive.m.o instead of the taskcluster index to get xpcshell 14 [--use-ftp-builds] 15 # Use git rather than hg. Using git does not currently support cloning (use 16 # --skip-repo as well). 17 [--use-git] 18 # One (or more) of the following actions must be specified. 19 --hsts | --hpkp | --remote-settings | --suffix-list | --mobile-experiments | --ct-logs 20 -b branch 21 # The name of top source directory to use for the repository clone. 22 [-t topsrcdir] 23 # Skips cloning of the repository. 24 [--skip-clone] 25 # Performs a dry run - no commits are created. 26 [-n] 27 # Skips pushing of the repository - create a commit but does not try 28 # to push it. 29 [--skip-push] 30 31 EOF 32 } 33 34 # Defaults 35 PRODUCT="firefox" 36 DRY_RUN=false 37 CLOSED_TREE=false 38 DONTBUILD=false 39 APPROVAL=false 40 41 DO_PRELOAD_PINSET=false 42 DO_HSTS=false 43 DO_HPKP=false 44 DO_REMOTE_SETTINGS=false 45 DO_SUFFIX_LIST=false 46 DO_MOBILE_EXPERIMENTS=false 47 DO_CT_LOGS=false 48 49 CLONE_REPO=true 50 HGHOST="hg.mozilla.org" 51 STAGEHOST="archive.mozilla.org" 52 53 USE_MC=false 54 USE_TC=true 55 USE_GIT=false 56 SKIP_PUSH=false 57 58 # Parse our command-line options. 59 while [ $# -gt 0 ]; do 60 case "$1" in 61 -h) usage; exit 0 ;; 62 -p) PRODUCT="$2"; shift ;; 63 -b) BRANCH="$2"; shift ;; 64 -n) DRY_RUN=true ;; 65 -c) CLOSED_TREE=true ;; 66 -d) DONTBUILD=true ;; 67 -a) APPROVAL=true ;; 68 --pinset) DO_PRELOAD_PINSET=true ;; 69 --hsts) DO_HSTS=true ;; 70 --hpkp) DO_HPKP=true ;; 71 --remote-settings) DO_REMOTE_SETTINGS=true ;; 72 --suffix-list) DO_SUFFIX_LIST=true ;; 73 --mobile-experiments) DO_MOBILE_EXPERIMENTS=true ;; 74 --ct-logs) DO_CT_LOGS=true ;; 75 --skip-clone) CLONE_REPO=false ;; 76 --skip-push) SKIP_PUSH=true ;; 77 -t) TOPSRCDIR="$2"; shift ;; 78 --use-mozilla-central) USE_MC=true ;; 79 --use-ftp-builds) USE_TC=false ;; 80 --use-git) USE_GIT=true ;; 81 -*) usage 82 exit 11 ;; 83 *) break ;; # terminate while loop 84 esac 85 shift 86 done 87 88 # Must supply a code branch to work with. 89 if [ "${BRANCH}" == "" ]; then 90 echo "Error: You must specify a branch with -b branchname." >&2 91 usage 92 exit 12 93 fi 94 95 # Must choose at least one update action. 96 if [ "$DO_HSTS" == "false" ] && [ "$DO_HPKP" == "false" ] && [ "$DO_REMOTE_SETTINGS" == "false" ] && [ "$DO_SUFFIX_LIST" == "false" ] && [ "$DO_MOBILE_EXPERIMENTS" == false ] && [ "$DO_CT_LOGS" == false ] 97 then 98 echo "Error: you must specify at least one action from: --hsts, --hpkp, --remote-settings, or --suffix-list" >&2 99 usage 100 exit 13 101 fi 102 103 # per-product constants 104 case "${PRODUCT}" in 105 thunderbird) 106 COMMIT_AUTHOR="tbirdbld <tbirdbld@thunderbird.net>" 107 ;; 108 firefox) 109 ;; 110 *) 111 echo "Error: Invalid product specified" 112 usage 113 exit 14 114 ;; 115 esac 116 117 if [ "${TOPSRCDIR}" == "" ]; then 118 TOPSRCDIR="$(basename "${BRANCH}")" 119 fi 120 121 case "${BRANCH}" in 122 mozilla-central|comm-central|try ) 123 HGREPO="https://${HGHOST}/${BRANCH}" 124 ;; 125 mozilla-*|comm-* ) 126 HGREPO="https://${HGHOST}/releases/${BRANCH}" 127 ;; 128 * ) 129 HGREPO="https://${HGHOST}/projects/${BRANCH}" 130 ;; 131 esac 132 133 BROWSER_ARCHIVE="target.tar.xz" 134 TESTS_ARCHIVE="target.common.tests.tar.zst" 135 136 UNPACK_CMD="tar xf" 137 COMMIT_AUTHOR='ffxbld <ffxbld@mozilla.com>' 138 WGET="wget -nv" 139 DIFF="$(command -v diff) -u" 140 JQ="$(command -v jq)" 141 142 if [ "${USE_GIT}" == "true" ]; then 143 GIT="$(command -v git)" 144 else 145 HG="$(command -v hg)" 146 fi 147 148 BASEDIR="${HOME}" 149 SCRIPTDIR="$(realpath "$(dirname "$0")")" 150 DATADIR="${BASEDIR}/data" 151 152 HSTS_PRELOAD_SCRIPT="${SCRIPTDIR}/getHSTSPreloadList.js" 153 HSTS_PRELOAD_ERRORS="nsSTSPreloadList.errors" 154 HSTS_PRELOAD_INC_OLD="${DATADIR}/nsSTSPreloadList.inc" 155 HSTS_PRELOAD_INC_NEW="${BASEDIR}/${PRODUCT}/nsSTSPreloadList.inc" 156 HSTS_UPDATED=false 157 158 HPKP_PRELOAD_SCRIPT="${SCRIPTDIR}/genHPKPStaticPins.js" 159 HPKP_PRELOAD_ERRORS="StaticHPKPins.errors" 160 HPKP_PRELOAD_JSON="${DATADIR}/PreloadedHPKPins.json" 161 HPKP_PRELOAD_INC="StaticHPKPins.h" 162 HPKP_PRELOAD_INPUT="${DATADIR}/${HPKP_PRELOAD_INC}" 163 HPKP_PRELOAD_OUTPUT="${DATADIR}/${HPKP_PRELOAD_INC}.out" 164 HPKP_UPDATED=false 165 166 REMOTE_SETTINGS_SERVER='' 167 REMOTE_SETTINGS_DIR="${TOPSRCDIR}/services/settings/dumps" 168 REMOTE_SETTINGS_UPDATED=false 169 170 PUBLIC_SUFFIX_URL="https://publicsuffix.org/list/public_suffix_list.dat" 171 PUBLIC_SUFFIX_LOCAL="public_suffix_list.dat" 172 HG_SUFFIX_LOCAL="effective_tld_names.dat" 173 HG_SUFFIX_PATH="/netwerk/dns/${HG_SUFFIX_LOCAL}" 174 SUFFIX_LIST_UPDATED=false 175 176 EXPERIMENTER_URL="https://experimenter.services.mozilla.com/api/v6/experiments-first-run/" 177 FENIX_INITIAL_EXPERIMENTS="mobile/android/fenix/app/src/main/res/raw/initial_experiments.json" 178 FOCUS_INITIAL_EXPERIMENTS="mobile/android/focus-android/app/src/main/res/raw/initial_experiments.json" 179 MOBILE_EXPERIMENTS_UPDATED=false 180 181 CT_LOG_UPDATE_SCRIPT="${SCRIPTDIR}/getCTKnownLogs.py" 182 183 ARTIFACTS_DIR="${ARTIFACTS_DIR:-.}" 184 # Defaults 185 HSTS_DIFF_ARTIFACT="${ARTIFACTS_DIR}/${HSTS_DIFF_ARTIFACT:-"nsSTSPreloadList.diff"}" 186 HPKP_DIFF_ARTIFACT="${ARTIFACTS_DIR}/${HPKP_DIFF_ARTIFACT:-"StaticHPKPins.h.diff"}" 187 REMOTE_SETTINGS_DIFF_ARTIFACT="${ARTIFACTS_DIR}/${REMOTE_SETTINGS_DIFF_ARTIFACT:-"remote-settings.diff"}" 188 SUFFIX_LIST_DIFF_ARTIFACT="${ARTIFACTS_DIR}/${SUFFIX_LIST_DIFF_ARTIFACT:-"effective_tld_names.diff"}" 189 EXPERIMENTER_DIFF_ARTIFACT="${ARTIFACTS_DIR}/initial_experiments.diff" 190 191 # duplicate the functionality of taskcluster-lib-urls, but in bash.. 192 queue_base="$TASKCLUSTER_ROOT_URL/api/queue/v1" 193 index_base="$TASKCLUSTER_ROOT_URL/api/index/v1" 194 195 function create_repo_diff() { 196 if [ "${USE_GIT}" == "true" ]; then 197 ${GIT} -C "${TOPSRCDIR}" diff -u "$1" > "$2" 198 else 199 ${HG} -R "${TOPSRCDIR}" diff "$1" > "$2" 200 fi 201 } 202 203 # Cleanup common artifacts. 204 function preflight_cleanup { 205 cd "${BASEDIR}" 206 rm -rf "${PRODUCT}" tests "${BROWSER_ARCHIVE}" "${TESTS_ARCHIVE}" 207 } 208 209 function download_shared_artifacts_from_ftp { 210 cd "${BASEDIR}" 211 212 # Download everything we need to run js with xpcshell 213 echo "INFO: Downloading all the necessary pieces from ${STAGEHOST}..." 214 ARTIFACT_DIR="nightly/latest-${BRANCH}" 215 if [ "${USE_MC}" == "true" ]; then 216 ARTIFACT_DIR="nightly/latest-mozilla-central" 217 fi 218 219 BROWSER_ARCHIVE_URL="https://${STAGEHOST}/pub/mozilla.org/${PRODUCT}/${ARTIFACT_DIR}/${BROWSER_ARCHIVE}" 220 TESTS_ARCHIVE_URL="https://${STAGEHOST}/pub/mozilla.org/${PRODUCT}/${ARTIFACT_DIR}/${TESTS_ARCHIVE}" 221 222 echo "INFO: ${WGET} ${BROWSER_ARCHIVE_URL}" 223 ${WGET} "${BROWSER_ARCHIVE_URL}" 224 echo "INFO: ${WGET} ${TESTS_ARCHIVE_URL}" 225 ${WGET} "${TESTS_ARCHIVE_URL}" 226 } 227 228 function download_shared_artifacts_from_tc { 229 cd "${BASEDIR}" 230 TASKID_FILE="taskId.json" 231 232 # Download everything we need to run js with xpcshell 233 echo "INFO: Downloading all the necessary pieces from the taskcluster index..." 234 TASKID_URL="$index_base/task/gecko.v2.${BRANCH}.shippable.latest.${PRODUCT}.linux64-opt" 235 if [ "${USE_MC}" == "true" ]; then 236 TASKID_URL="$index_base/task/gecko.v2.mozilla-central.shippable.latest.${PRODUCT}.linux64-opt" 237 fi 238 ${WGET} -O ${TASKID_FILE} "${TASKID_URL}" 239 INDEX_TASK_ID="$($JQ -r '.taskId' ${TASKID_FILE})" 240 if [ -z "${INDEX_TASK_ID}" ]; then 241 echo "Failed to look up taskId at ${TASKID_URL}" 242 exit 22 243 else 244 echo "INFO: Got taskId of $INDEX_TASK_ID" 245 fi 246 247 TASKSTATUS_FILE="taskstatus.json" 248 STATUS_URL="$queue_base/task/${INDEX_TASK_ID}/status" 249 ${WGET} -O "${TASKSTATUS_FILE}" "${STATUS_URL}" 250 LAST_RUN_INDEX=$(($(jq '.status.runs | length' ${TASKSTATUS_FILE}) - 1)) 251 echo "INFO: Examining run number ${LAST_RUN_INDEX}" 252 253 BROWSER_ARCHIVE_URL="$queue_base/task/${INDEX_TASK_ID}/runs/${LAST_RUN_INDEX}/artifacts/public/build/${BROWSER_ARCHIVE}" 254 echo "INFO: ${WGET} ${BROWSER_ARCHIVE_URL}" 255 ${WGET} "${BROWSER_ARCHIVE_URL}" 256 257 TESTS_ARCHIVE_URL="$queue_base/task/${INDEX_TASK_ID}/runs/${LAST_RUN_INDEX}/artifacts/public/build/${TESTS_ARCHIVE}" 258 echo "INFO: ${WGET} ${TESTS_ARCHIVE_URL}" 259 ${WGET} "${TESTS_ARCHIVE_URL}" 260 } 261 262 function unpack_artifacts { 263 cd "${BASEDIR}" 264 if [ ! -f "${BROWSER_ARCHIVE}" ]; then 265 echo "Downloaded file '${BROWSER_ARCHIVE}' not found in directory '$(pwd)'." >&2 266 exit 31 267 fi 268 if [ ! -f "${TESTS_ARCHIVE}" ]; then 269 echo "Downloaded file '${TESTS_ARCHIVE}' not found in directory '$(pwd)'." >&2 270 exit 32 271 fi 272 # Unpack the browser and move xpcshell in place for updating the preload list. 273 echo "INFO: Unpacking resources..." 274 ${UNPACK_CMD} "${BROWSER_ARCHIVE}" 275 mkdir -p tests 276 cd tests 277 ${UNPACK_CMD} "../${TESTS_ARCHIVE}" 278 cd "${BASEDIR}" 279 cp tests/bin/xpcshell "${PRODUCT}" 280 } 281 282 # Downloads the current in-tree HSTS (HTTP Strict Transport Security) files. 283 # Runs a simple xpcshell script to generate up-to-date HSTS information. 284 # Compares the new HSTS output with the old to determine whether we need to update. 285 function compare_hsts_files { 286 cd "${BASEDIR}" 287 288 HSTS_PRELOAD_INC_HG="${HGREPO}/raw-file/default/security/manager/ssl/$(basename "${HSTS_PRELOAD_INC_OLD}")" 289 290 echo "INFO: Downloading existing include file..." 291 rm -rf "${HSTS_PRELOAD_ERRORS}" "${HSTS_PRELOAD_INC_OLD}" 292 echo "INFO: ${WGET} ${HSTS_PRELOAD_INC_HG}" 293 ${WGET} -O "${HSTS_PRELOAD_INC_OLD}" "${HSTS_PRELOAD_INC_HG}" 294 295 if [ ! -f "${HSTS_PRELOAD_INC_OLD}" ]; then 296 echo "Downloaded file '${HSTS_PRELOAD_INC_OLD}' not found in directory '$(pwd)' - this should have been downloaded above from ${HSTS_PRELOAD_INC_HG}." >&2 297 exit 41 298 fi 299 300 # Run the script to get an updated preload list. 301 echo "INFO: Generating new HSTS preload list..." 302 cd "${BASEDIR}/${PRODUCT}" 303 if ! LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:. ./xpcshell "${HSTS_PRELOAD_SCRIPT}" "${HSTS_PRELOAD_INC_OLD}"; then 304 echo "HSTS preload list generation failed" >&2 305 exit 43 306 fi 307 308 # The created files should be non-empty. 309 echo "INFO: Checking whether new HSTS preload list is valid..." 310 if [ ! -s "${HSTS_PRELOAD_INC_NEW}" ]; then 311 echo "New HSTS preload list ${HSTS_PRELOAD_INC_NEW} is empty. That's less good." >&2 312 exit 42 313 fi 314 cd "${BASEDIR}" 315 316 # Check for differences 317 echo "INFO: diffing old/new HSTS preload lists into ${HSTS_DIFF_ARTIFACT}" 318 ${DIFF} "${HSTS_PRELOAD_INC_OLD}" "${HSTS_PRELOAD_INC_NEW}" | tee "${HSTS_DIFF_ARTIFACT}" 319 if [ -s "${HSTS_DIFF_ARTIFACT}" ] 320 then 321 return 0 322 fi 323 return 1 324 } 325 326 # Downloads the current in-tree HPKP (HTTP public key pinning) files. 327 # Runs a simple xpcshell script to generate up-to-date HPKP information. 328 # Compares the new HPKP output with the old to determine whether we need to update. 329 function compare_hpkp_files { 330 cd "${BASEDIR}" 331 HPKP_PRELOAD_JSON_HG="${HGREPO}/raw-file/default/security/manager/tools/$(basename "${HPKP_PRELOAD_JSON}")" 332 333 HPKP_PRELOAD_OUTPUT_HG="${HGREPO}/raw-file/default/security/manager/ssl/${HPKP_PRELOAD_INC}" 334 335 rm -f "${HPKP_PRELOAD_OUTPUT}" 336 ${WGET} -O "${HPKP_PRELOAD_INPUT}" "${HPKP_PRELOAD_OUTPUT_HG}" 337 ${WGET} -O "${HPKP_PRELOAD_JSON}" "${HPKP_PRELOAD_JSON_HG}" 338 339 # Run the script to get an updated preload list. 340 echo "INFO: Generating new HPKP preload list..." 341 cd "${BASEDIR}/${PRODUCT}" 342 if ! LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:. ./xpcshell "${HPKP_PRELOAD_SCRIPT}" "${HPKP_PRELOAD_JSON}" "${HPKP_PRELOAD_OUTPUT}" > "${HPKP_PRELOAD_ERRORS}"; then 343 echo "HPKP preload list generation failed" >&2 344 exit 54 345 fi 346 347 # The created files should be non-empty. 348 echo "INFO: Checking whether new HPKP preload list is valid..." 349 350 if [ ! -s "${HPKP_PRELOAD_OUTPUT}" ]; then 351 echo "${HPKP_PRELOAD_OUTPUT} is empty. That's less good." >&2 352 exit 52 353 fi 354 if ! grep kPreloadPKPinsExpirationTime "${HPKP_PRELOAD_OUTPUT}"; then 355 echo "${HPKP_PRELOAD_OUTPUT} is missing an expiration time. Truncated?" >&2 356 exit 53 357 fi 358 cd "${BASEDIR}" 359 360 echo "INFO: diffing old/new HPKP preload lists..." 361 ${DIFF} "${HPKP_PRELOAD_INPUT}" "${HPKP_PRELOAD_OUTPUT}" | tee "${HPKP_DIFF_ARTIFACT}" 362 if [ -s "${HPKP_DIFF_ARTIFACT}" ] 363 then 364 return 0 365 fi 366 return 1 367 } 368 369 function is_valid_xml { 370 xmlfile=$1 371 XMLLINT=$(command -v xmllint 2>/dev/null | head -n1) 372 373 if [ ! -x "${XMLLINT}" ]; then 374 echo "ERROR: xmllint not found in PATH" 375 exit 60 376 fi 377 ${XMLLINT} --nonet --noout "${xmlfile}" 378 } 379 380 # Downloads the public suffix list 381 function compare_suffix_lists { 382 HG_SUFFIX_URL="${HGREPO}/raw-file/default/${HG_SUFFIX_PATH}" 383 cd "${BASEDIR}" 384 385 echo "INFO: ${WGET} -O ${PUBLIC_SUFFIX_LOCAL} ${PUBLIC_SUFFIX_URL}" 386 rm -f "${PUBLIC_SUFFIX_LOCAL}" 387 ${WGET} -O "${PUBLIC_SUFFIX_LOCAL}" "${PUBLIC_SUFFIX_URL}" 388 389 echo "INFO: ${WGET} -O ${HG_SUFFIX_LOCAL} ${HG_SUFFIX_URL}" 390 rm -f "${HG_SUFFIX_LOCAL}" 391 ${WGET} -O "${HG_SUFFIX_LOCAL}" "${HG_SUFFIX_URL}" 392 393 echo "INFO: diffing in-tree suffix list against the suffix list from publicsuffix.org" 394 ${DIFF} ${PUBLIC_SUFFIX_LOCAL} ${HG_SUFFIX_LOCAL} | tee "${SUFFIX_LIST_DIFF_ARTIFACT}" 395 if [ -s "${SUFFIX_LIST_DIFF_ARTIFACT}" ] 396 then 397 return 0 398 fi 399 return 1 400 } 401 402 function compare_remote_settings_files { 403 # cd "${TOPSRCDIR}" 404 405 REMOTE_SETTINGS_SERVER="https://firefox.settings.services.mozilla.com/v1" 406 407 # 1. List remote settings collections from server. 408 echo "INFO: fetch remote settings list from server" 409 ${WGET} -qO- "${REMOTE_SETTINGS_SERVER}/buckets/monitor/collections/changes/records" |\ 410 ${JQ} -r '.data[] | .bucket+"/"+.collection+"/"+(.last_modified|tostring)' |\ 411 # 2. For each entry ${bucket, collection, last_modified} 412 while IFS="/" read -r bucket collection last_modified; do 413 414 # 3. Check to see if the collection exists in the dump directory of the repository, 415 # if it does not then we aren't keeping the dump, and so we skip it. 416 local_dump_file="${REMOTE_SETTINGS_DIR}/${bucket}/${collection}.json" 417 if [ ! -r "${local_dump_file}" ]; then 418 continue 419 fi 420 421 # 4. Download server version into REMOTE_SETTINGS_DIR folder 422 remote_records_url="$REMOTE_SETTINGS_SERVER/buckets/${bucket}/collections/${collection}/changeset?_expected=${last_modified}" 423 local_location_output="$REMOTE_SETTINGS_DIR/${bucket}/${collection}.json" 424 425 # We sort both the keys and the records in search-config-v2 to make it 426 # easier to read and to experiment with making changes via the dump file. 427 if [ "${collection}" = "search-config-v2" ]; then 428 ${WGET} -qO- "$remote_records_url" | ${JQ} --sort-keys '{"data": .changes | sort_by(.recordType, .identifier), "timestamp": .timestamp}' > "${local_location_output}" 429 else 430 ${WGET} -qO- "$remote_records_url" | ${JQ} '{"data": .changes, "timestamp": .timestamp}' > "${local_location_output}" 431 fi 432 433 # 5. Download attachments if needed. 434 if [ "${bucket}" = "blocklists" ] && [ "${collection}" = "addons-bloomfilters" ]; then 435 # Find the attachment with the most recent generation_time, like _updateMLBF in Blocklist.sys.mjs. 436 # The server should return one "bloomfilter-base" record, but in case it returns multiple, 437 # return the most recent one. The server may send multiple entries if we ever decide to use 438 # the "filter_expression" feature of Remote Settings to send different records to specific 439 # channels. In that case this code should be updated to recognize the filter expression, 440 # but until we do, simply select the most recent record - can't go wrong with that. 441 # Note that "attachment_type" and "generation_time" are specific to addons-bloomfilters. 442 update_remote_settings_attachment "${bucket}" "${collection}" addons-mlbf.bin \ 443 'map(select(.attachment_type == "bloomfilter-base")) | sort_by(.generation_time) | last' 444 update_remote_settings_attachment "${bucket}" "${collection}" softblocks-addons-mlbf.bin \ 445 'map(select(.attachment_type == "softblocks-bloomfilter-base")) | sort_by(.generation_time) | last' 446 fi 447 # TODO: Bug 1873448. This cannot handle new/removed files currently, due to the 448 # build system making it difficult. 449 if [ "${bucket}" = "main" ] && [ "${collection}" = "search-config-icons" ]; then 450 ${JQ} -r '.data[] | .id' < "${local_location_output}" |\ 451 while read -r id; do 452 # We do not want quotes around ${id} 453 # shellcheck disable=SC2086 454 update_remote_settings_attachment "${bucket}" "${collection}" ${id} ".[] | select(.id == \"${id}\")" 455 done 456 fi 457 # NOTE: The downloaded data is not validated. xpcshell should be used for that. 458 459 # bug 1959683: remote settings update can add untracked search-config-icons 460 # It is not safe to take these (see https://bugzilla.mozilla.org/show_bug.cgi?id=1873448) 461 # If they are around as untracked files when `arc diff` runs, that command will fail. 462 # (We explicitly don't want to use `arc diff --allow-untracked` to avoid accidentally 463 # missing files from other updates - we'd rather the job fail.) 464 if [ "${USE_GIT}" == "true" ]; then 465 ${GIT} -C "${TOPSRCDIR}" clean -f -d services/settings/dumps/main/search-config-icons 466 else 467 ${HG} --cwd "${TOPSRCDIR}" purge services/settings/dumps/main/search-config-icons 468 fi 469 done 470 471 echo "INFO: diffing old/new remote settings dumps..." 472 create_repo_diff "${REMOTE_SETTINGS_DIR}" "${REMOTE_SETTINGS_DIFF_ARTIFACT}" 473 474 # cd "${BASEDIR}" 475 if [ -s "${REMOTE_SETTINGS_DIFF_ARTIFACT}" ] 476 then 477 return 0 478 fi 479 return 1 480 } 481 482 # Helper for compare_remote_settings_files to download attachments from remote settings. 483 # The format and location is documented at: 484 # https://firefox-source-docs.mozilla.org/services/settings/index.html#services-packaging-attachments 485 function update_remote_settings_attachment() { 486 local bucket=$1 487 local collection=$2 488 local attachment_id=$3 489 # $4 is a jq filter on the arrays that should return one record with the attachment 490 local jq_attachment_selector=".data | map(select(.attachment)) | $4" 491 492 # These paths match _readAttachmentDump in services/settings/Attachments.sys.mjs. 493 local path_to_attachment="${bucket}/${collection}/${attachment_id}" 494 local path_to_meta="${bucket}/${collection}/${attachment_id}.meta.json" 495 local meta_file="${REMOTE_SETTINGS_DIR}/${path_to_meta}" 496 497 # Those files should have been created by compare_remote_settings_files before the function call. 498 local source_collection_location="${REMOTE_SETTINGS_DIR}/${bucket}/${collection}.json" 499 500 # Exact the metadata for this attachment from the already downloaded collection, 501 # and compare with our current metadata to see if the attachment has changed or not. 502 # Uses cmp for fast compare (rather than repository tools). 503 if ${JQ} -cj "${jq_attachment_selector}" < "${source_collection_location}" | cmp --silent - "${meta_file}"; then 504 # Metadata not changed, don't bother downloading the attachments themselves. 505 return 506 fi 507 # Metadata changed. Download attachments. 508 509 # Save the metadata. 510 ${JQ} -cj <"${source_collection_location}" "${jq_attachment_selector}" > "${meta_file}" 511 512 echo "INFO: Downloading updated remote settings dump: ${bucket}/${collection}/${attachment_id}" 513 514 if [ -z "${ATTACHMENT_BASE_URL}" ] ; then 515 ATTACHMENT_BASE_URL=$(${WGET} -qO- "${REMOTE_SETTINGS_SERVER}" | ${JQ} -r .capabilities.attachments.base_url) 516 fi 517 attachment_path_from_meta=$(${JQ} -r < "${meta_file}" .attachment.location) 518 ${WGET} -qO "${REMOTE_SETTINGS_DIR}/${path_to_attachment}" "${ATTACHMENT_BASE_URL}${attachment_path_from_meta}" 519 } 520 521 function compare_mobile_experiments() { 522 echo "INFO ${WGET} ${EXPERIMENTER_URL}" 523 ${WGET} -O experiments.json "${EXPERIMENTER_URL}" 524 ${WGET} -O fenix-experiments-old.json "${HGREPO}/raw-file/default/${FENIX_INITIAL_EXPERIMENTS}" 525 ${WGET} -O focus-experiments-old.json "${HGREPO}/raw-file/default/${FOCUS_INITIAL_EXPERIMENTS}" 526 527 # shellcheck disable=SC2016 528 ${JQ} --arg APP_NAME fenix '{"data":map(select(.appName == $APP_NAME))}' < experiments.json > fenix-experiments-new.json 529 # shellcheck disable=SC2016 530 ${JQ} --arg APP_NAME focus_android '{"data":map(select(.appName == $APP_NAME))}' < experiments.json > focus-experiments-new.json 531 532 ( ${DIFF} fenix-experiments-old.json fenix-experiments-new.json; ${DIFF} focus-experiments-old.json focus-experiments-new.json ) > "${EXPERIMENTER_DIFF_ARTIFACT}" 533 if [ -s "${EXPERIMENTER_DIFF_ARTIFACT}" ]; then 534 return 0 535 else 536 # no change 537 return 1 538 fi 539 } 540 541 function update_ct_logs() { 542 echo "INFO: Updating CT logs..." 543 "${TOPSRCDIR}"/mach python "${CT_LOG_UPDATE_SCRIPT}" 544 } 545 546 # Clones an hg repo 547 function clone_repo { 548 cd "${BASEDIR}" 549 if [ ! -d "${TOPSRCDIR}" ]; then 550 ${HG} robustcheckout --sharebase /tmp/hg-store -b default "${HGREPO}" "${TOPSRCDIR}" 551 fi 552 553 ${HG} -R "${TOPSRCDIR}" pull 554 ${HG} -R "${TOPSRCDIR}" update -C default 555 } 556 557 # Copies new HSTS files in place, and commits them. 558 function stage_hsts_files { 559 cd "${BASEDIR}" 560 cp -f "${HSTS_PRELOAD_INC_NEW}" "${TOPSRCDIR}/security/manager/ssl/" 561 } 562 563 function stage_hpkp_files { 564 cd "${BASEDIR}" 565 cp -f "${HPKP_PRELOAD_OUTPUT}" "${TOPSRCDIR}/security/manager/ssl/${HPKP_PRELOAD_INC}" 566 } 567 568 function stage_tld_suffix_files { 569 cd "${BASEDIR}" 570 cp -a "${PUBLIC_SUFFIX_LOCAL}" "${TOPSRCDIR}/${HG_SUFFIX_PATH}" 571 } 572 573 function stage_mobile_experiments_files { 574 cd "${BASEDIR}" 575 576 cp fenix-experiments-new.json "${TOPSRCDIR}/${FENIX_INITIAL_EXPERIMENTS}" 577 cp focus-experiments-new.json "${TOPSRCDIR}/${FOCUS_INITIAL_EXPERIMENTS}" 578 } 579 580 # Push all pending commits to Phabricator 581 function push_repo { 582 if [ "${SKIP_PUSH}" == "true" ]; then 583 echo "Skipping push due to --skip-push" 584 return 0 585 fi 586 587 cd "${TOPSRCDIR}" 588 if [ ! -r "${HOME}/.arcrc" ] 589 then 590 return 1 591 fi 592 if ! ARC=$(command -v arc) && ! ARC=$(command -v arcanist) 593 then 594 return 1 595 fi 596 if [ -z "${REVIEWERS}" ] 597 then 598 return 1 599 fi 600 # Clean up older review requests 601 # Turn Needs Review D624: No bug, Automated HSTS ... 602 # into D624 603 for diff in $($ARC list | grep "Needs Review" | grep -E "${BRANCH} repo-update" | awk 'match($0, /D[0-9]+[^: ]/) { print substr($0, RSTART, RLENGTH) }') 604 do 605 echo "Removing old request $diff" 606 # There is no 'arc abandon', see bug 1452082 607 echo '{"transactions": [{"type":"abandon", "value": true}], "objectIdentifier": "'"${diff}"'"}' | $ARC call-conduit -- differential.revision.edit 608 done 609 610 # bug 1959683: using /dev/null as stdin causes arcanist to fail quickly 611 # instead of hang if user input is requested. 612 $ARC diff --verbatim --reviewers "${REVIEWERS}" < /dev/null 613 } 614 615 616 617 # Main 618 619 preflight_cleanup 620 621 mkdir -p "${DATADIR}" 622 623 # Clone the repository here as some sections will use it for source data, and 624 # we'll need it later anyway. 625 if [ "${CLONE_REPO}" == "true" ] 626 then 627 clone_repo 628 fi 629 630 if [ "${DO_HSTS}" == "true" ] || [ "${DO_HPKP}" == "true" ] || [ "${DO_PRELOAD_PINSET}" == "true" ] 631 then 632 if [ "${USE_TC}" == "true" ]; then 633 download_shared_artifacts_from_tc 634 else 635 download_shared_artifacts_from_ftp 636 fi 637 unpack_artifacts 638 fi 639 640 if [ "${DO_HSTS}" == "true" ]; then 641 if compare_hsts_files 642 then 643 HSTS_UPDATED=true 644 fi 645 fi 646 if [ "${DO_HPKP}" == "true" ]; then 647 if compare_hpkp_files 648 then 649 HPKP_UPDATED=true 650 fi 651 fi 652 if [ "${DO_REMOTE_SETTINGS}" == "true" ]; then 653 if compare_remote_settings_files 654 then 655 REMOTE_SETTINGS_UPDATED=true 656 fi 657 fi 658 if [ "${DO_SUFFIX_LIST}" == "true" ]; then 659 if compare_suffix_lists 660 then 661 SUFFIX_LIST_UPDATED=true 662 fi 663 fi 664 if [ "${DO_MOBILE_EXPERIMENTS}" == "true" ]; then 665 if compare_mobile_experiments 666 then 667 MOBILE_EXPERIMENTS_UPDATED=true 668 fi 669 fi 670 if [ "${DO_CT_LOGS}" == "true" ]; then 671 update_ct_logs 672 fi 673 674 675 if [ "${HSTS_UPDATED}" == "false" ] && [ "${HPKP_UPDATED}" == "false" ] && [ "${REMOTE_SETTINGS_UPDATED}" == "false" ] && [ "${SUFFIX_LIST_UPDATED}" == "false" ] && [ "${MOBILE_EXPERIMENTS_UPDATED}" == "false" ] && [ "${DO_CT_LOGS}" == "false" ]; then 676 echo "INFO: no updates required. Exiting." 677 exit 0 678 else 679 if [ "${DRY_RUN}" == "true" ]; then 680 echo "INFO: Updates are available, not updating hg in dry-run mode." 681 exit 2 682 fi 683 fi 684 685 COMMIT_MESSAGE="No Bug, ${BRANCH} repo-update" 686 if [ "${HSTS_UPDATED}" == "true" ] 687 then 688 stage_hsts_files 689 COMMIT_MESSAGE="${COMMIT_MESSAGE} HSTS" 690 fi 691 692 if [ "${HPKP_UPDATED}" == "true" ] 693 then 694 stage_hpkp_files 695 COMMIT_MESSAGE="${COMMIT_MESSAGE} HPKP" 696 fi 697 698 if [ "${REMOTE_SETTINGS_UPDATED}" == "true" ] 699 then 700 COMMIT_MESSAGE="${COMMIT_MESSAGE} remote-settings" 701 fi 702 703 if [ "${SUFFIX_LIST_UPDATED}" == "true" ] 704 then 705 stage_tld_suffix_files 706 COMMIT_MESSAGE="${COMMIT_MESSAGE} tld-suffixes" 707 fi 708 709 if [ "${MOBILE_EXPERIMENTS_UPDATED}" == "true" ] 710 then 711 stage_mobile_experiments_files 712 COMMIT_MESSAGE="${COMMIT_MESSAGE} mobile-experiments" 713 fi 714 715 if [ "${DO_CT_LOGS}" == "true" ] 716 then 717 # CT log files are already updated in-place in the tree, so 718 # there's no need to stage them. 719 COMMIT_MESSAGE="${COMMIT_MESSAGE} ct-logs" 720 fi 721 722 if [ ${DONTBUILD} == true ]; then 723 COMMIT_MESSAGE="${COMMIT_MESSAGE} - (DONTBUILD)" 724 fi 725 if [ ${CLOSED_TREE} == true ]; then 726 COMMIT_MESSAGE="${COMMIT_MESSAGE} - CLOSED TREE" 727 fi 728 if [ ${APPROVAL} == true ]; then 729 COMMIT_MESSAGE="${COMMIT_MESSAGE} - a=repo-update" 730 fi 731 732 if [ "${USE_GIT}" == "true" ]; then 733 if ${GIT} -C "${TOPSRCDIR}" commit -a --author "${COMMIT_AUTHOR}" -m "${COMMIT_MESSAGE}" 734 then 735 push_repo 736 fi 737 else 738 if ${HG} -R "${TOPSRCDIR}" commit -u "${COMMIT_AUTHOR}" -m "${COMMIT_MESSAGE}" 739 then 740 push_repo 741 fi 742 fi 743 744 echo "All done"