.taskcluster.yml (27789B)
1 # yamllint disable rule:line-length 2 # This file is rendered via JSON-e by 3 # - hg-push - https://github.com/mozilla-releng/fxci-config/blob/main/build-decision/src/build_decision/hg_push.py 4 # { 5 # tasks_for: 'hg-push', 6 # push: {owner, comment, pushlog_id, pushdate}, 7 # repository: {url, project, level}, 8 # now, 9 # as_slugid: // function 10 # ownTaskId: // taskId of the task that will be created 11 # } 12 # 13 # - cron tasks - https://github.com/mozilla-releng/fxci-config/blob/main/build-decision/src/build_decision/cron/decision.py 14 # { 15 # tasks_for: 'cron', 16 # push: {revision, pushlog_id, pushdate, owner} 17 # repository: {url, project, level}, 18 # cron: {task_id, job_name, job_symbol, quoted_args}, 19 # now, 20 # ownTaskId: // taskId of the task that will be created 21 # } 22 # 23 # - action tasks - See: 24 # * taskcluster/gecko_taskgraph/actions/registry.py, 25 # * https://docs.taskcluster.net/docs/manual/design/conventions/actions/spec 26 # * https://github.com/mozilla-releng/fxci-config/blob/main/src/ciadmin/generate/in_tree_actions.py 27 # 28 # The registry generates the hookPayload that appears in actions.json, and 29 # contains data from the decision task as well as JSON-e code to combine that 30 # with data supplied as part of the action spec. When the hook is fired, the 31 # hookPayload is rendered with JSON-e to produce a payload for the hook task 32 # template. 33 # 34 # The ci-admin code wraps the content of this file (.taskcluster.yml) with a 35 # JSON-e $let statement that produces the context described below, and 36 # installs that as the hook task template. 37 # 38 # { 39 # tasks_for: 'action', 40 # push: {owner, pushlog_id, revision, base_revision}, 41 # repository: {url, project, level}, 42 # input, 43 # taskId, // targetted taskId 44 # taskGroupId, // targetted taskGroupId 45 # action: {name, title, description, taskGroupId, symbol, repo_scope, cb_name} 46 # ownTaskId: // taskId of the task that will be created 47 # clientId: // clientId that triggered this hook 48 # } 49 --- 50 version: 1 51 reporting: checks-v1 52 policy: 53 pullRequests: collaborators 54 tasks: 55 # NOTE: support for actions in ci-admin requires that the `tasks` property be an array *before* JSON-e rendering 56 # takes place. 57 - $if: 'tasks_for in ["hg-push", "action", "cron"]' 58 then: 59 $let: 60 # sometimes the push user is just `ffxbld` or the like, but we want an email-like field.. 61 ownerEmail: {$if: '"@" in push.owner', then: '${push.owner}', else: '${push.owner}@noreply.mozilla.org'} 62 # ensure there's no trailing `/` on the repo URL 63 repoUrl: {$if: 'repository.url[-1] == "/"', then: {$eval: 'repository.url[:-1]'}, else: {$eval: 'repository.url'}} 64 # expire try earlier than other branches 65 expires: 66 $if: 'repository.project == "try"' 67 then: {$fromNow: '28 days'} 68 else: {$fromNow: '1 year'} 69 trustDomain: gecko 70 treeherder_link: '[Treeherder job](https://treeherder.mozilla.org/#/jobs?repo=${repository.project}&revision=${push.revision}&selectedTaskRun=${ownTaskId})' 71 in: 72 taskId: {$if: 'tasks_for != "action"', then: '${ownTaskId}'} 73 taskGroupId: 74 $if: 'tasks_for == "action"' 75 then: 76 '${action.taskGroupId}' 77 else: 78 '${ownTaskId}' # same as taskId; this is how automation identifies a decision task 79 schedulerId: '${trustDomain}-level-${repository.level}' 80 81 created: {$fromNow: ''} 82 deadline: {$fromNow: '1 day'} 83 expires: {$eval: 'expires'} 84 metadata: 85 $merge: 86 - owner: "${ownerEmail}" 87 source: "${repoUrl}/raw-file/${push.revision}/.taskcluster.yml" 88 - $if: 'tasks_for == "hg-push"' 89 then: 90 name: "Gecko Decision Task" 91 description: 'The task that creates all of the other tasks in the task graph (${treeherder_link})' 92 else: 93 $if: 'tasks_for == "action"' 94 then: 95 name: "Action: ${action.title}" 96 description: | 97 ${action.description} 98 99 ${treeherder_link} 100 101 Action triggered by clientID `${clientId}` 102 else: 103 name: "Decision Task for cron job ${cron.job_name}" 104 description: 'Created by a [cron task](https://firefox-ci-tc.services.mozilla.com/tasks/${cron.task_id}) (${treeherder_link})' 105 106 provisionerId: "${trustDomain}-${repository.level}" 107 workerType: "decision" 108 109 tags: 110 $if: 'tasks_for == "hg-push"' 111 then: 112 createdForUser: "${ownerEmail}" 113 kind: decision-task 114 else: 115 $if: 'tasks_for == "action"' 116 then: 117 createdForUser: '${ownerEmail}' 118 kind: 'action-callback' 119 else: 120 $if: 'tasks_for == "cron"' 121 then: 122 kind: cron-task 123 124 routes: 125 $flattenDeep: 126 - "tc-treeherder.v2.${repository.project}.${push.revision}" 127 - $if: 'tasks_for == "hg-push"' 128 then: 129 - "index.${trustDomain}.v2.${repository.project}.latest.taskgraph.decision" 130 - "index.${trustDomain}.v2.${repository.project}.revision.${push.revision}.taskgraph.decision" 131 - "index.${trustDomain}.v2.${repository.project}.pushlog-id.${push.pushlog_id}.decision" 132 - "notify.email.${ownerEmail}.on-failed" 133 - "notify.email.${ownerEmail}.on-exception" 134 # Send a notification email if the push comes from try 135 - $if: 'repository.project == "try"' 136 then: 137 - "notify.email.${ownerEmail}.on-completed" 138 - $if: 'repository.project == "mozilla-central"' 139 then: 140 # Notify #thunderbird-ci 141 - "notify.matrix-room.!TWztIhgqLawNpRBZTC:mozilla.org.on-completed" 142 else: 143 $if: 'tasks_for == "action"' 144 then: 145 - "index.${trustDomain}.v2.${repository.project}.revision.${push.revision}.taskgraph.actions.${ownTaskId}" 146 - "index.${trustDomain}.v2.${repository.project}.pushlog-id.${push.pushlog_id}.actions.${ownTaskId}" 147 else: # cron 148 - "index.${trustDomain}.v2.${repository.project}.latest.taskgraph.decision-${cron.job_name}" 149 - "index.${trustDomain}.v2.${repository.project}.revision.${push.revision}.taskgraph.decision-${cron.job_name}" 150 - "index.${trustDomain}.v2.${repository.project}.pushlog-id.${push.pushlog_id}.decision-${cron.job_name}" 151 # list each cron task on this revision, so actions can find them 152 - 'index.${trustDomain}.v2.${repository.project}.revision.${push.revision}.cron.${ownTaskId}' 153 # BUG 1500166 Notify ciduty by email if a nightly hook fails 154 - $if: 'repository.project != "try"' 155 then: 156 - "notify.email.ciduty+failedcron@mozilla.com.on-failed" 157 - "notify.email.ciduty+exceptioncron@mozilla.com.on-exception" 158 - "notify.email.sheriffs+failedcron@mozilla.org.on-failed" 159 - "notify.email.sheriffs+exceptioncron@mozilla.org.on-exception" 160 161 scopes: 162 $if: 'tasks_for == "hg-push"' 163 then: 164 - 'assume:repo:${repoUrl[8:]}:branch:default' 165 - 'queue:route:notify.email.${ownerEmail}.*' 166 - 'in-tree:hook-action:project-${trustDomain}/in-tree-action-${repository.level}-*' 167 - 'index:insert-task:${trustDomain}.v2.${repository.project}.*' 168 else: 169 $if: 'tasks_for == "action"' 170 then: 171 # when all actions are hooks, we can calculate this directly rather than using a variable 172 - '${action.repo_scope}' 173 else: 174 - 'assume:repo:${repoUrl[8:]}:cron:${cron.job_name}' 175 176 dependencies: [] 177 requires: all-completed 178 179 priority: 180 # Most times, there is plenty of worker capacity so everything runs 181 # quickly, but sometimes a storm of action tasks lands. Then we 182 # want, from highest to lowest: 183 # - cron tasks (time-sensitive) (low) 184 # - action tasks (avoid interfering with the other two) (very-low) 185 # - decision tasks (minimize user-visible delay) (lowest) 186 # SCM levels all use different workerTypes, so there is no need for priority 187 # between levels; "low" is the highest priority available at all levels, and 188 # nothing runs at any higher priority on these workerTypes. 189 $if: "tasks_for == 'cron'" 190 then: low 191 else: 192 $if: "tasks_for == 'action'" 193 then: very-low 194 else: lowest # tasks_for == 'hg-push' 195 retries: 196 $if: "tasks_for == 'hg-push' && repository.level != '1'" 197 then: 0 198 else: 5 199 200 payload: 201 env: 202 # run-task uses these to check out the source; the inputs 203 # to `mach taskgraph decision` are all on the command line. 204 $merge: 205 - GECKO_BASE_REPOSITORY: 'https://hg.mozilla.org/mozilla-unified' 206 GECKO_BASE_REV: '${push.base_revision}' 207 GECKO_HEAD_REPOSITORY: '${repoUrl}' 208 GECKO_HEAD_REF: '${push.revision}' 209 GECKO_HEAD_REV: '${push.revision}' 210 HG_STORE_PATH: /builds/worker/checkouts/hg-store 211 TASKCLUSTER_CACHES: /builds/worker/checkouts 212 TASKCLUSTER_VOLUMES: /builds/worker/artifacts 213 MOZ_UPLOAD_DIR: /builds/worker/artifacts 214 MOZ_AUTOMATION: '1' 215 # mach generates pyc files when reading `mach_commands.py` 216 # This causes cached_task digest generation to be random for 217 # some tasks. Disable bytecode generation to work around that. 218 PYTHONDONTWRITEBYTECODE: '1' 219 MACH_BUILD_PYTHON_NATIVE_PACKAGE_SOURCE: 'system' 220 - $if: 'tasks_for == "action"' 221 then: 222 ACTION_TASK_GROUP_ID: '${action.taskGroupId}' # taskGroupId of the target task 223 ACTION_TASK_ID: {$json: {$eval: 'taskId'}} # taskId of the target task (JSON-encoded) 224 ACTION_INPUT: {$json: {$eval: 'input'}} 225 ACTION_CALLBACK: '${action.cb_name}' 226 227 cache: 228 "${trustDomain}-level-${repository.level}-checkouts-sparse-v4": /builds/worker/checkouts 229 230 features: 231 taskclusterProxy: true 232 chainOfTrust: true 233 234 # Note: This task is built server side without the context or tooling that 235 # exist in tree so we must hard code the hash 236 image: 'mozillareleases/gecko_decision:5.1.0@sha256:8879ecb7f859e1001b0aa67de440a18c3a896446bfd0e2e87688bb367e590667' 237 238 maxRunTime: 3600 239 240 command: 241 - /builds/worker/bin/run-task-hg 242 - '--gecko-checkout=/builds/worker/checkouts/gecko' 243 - '--gecko-sparse-profile=build/sparse-profiles/taskgraph' 244 - '--' 245 - bash 246 - -cx 247 - $let: 248 extraArgs: 249 $if: 'tasks_for == "cron"' 250 then: '${cron.quoted_args}' 251 else: 252 $if: 'repository.project in ["autoland", "try"]' 253 then: '--no-verify' 254 else: '' 255 in: 256 $if: 'tasks_for == "action"' 257 then: > 258 cd /builds/worker/checkouts/gecko && 259 ln -s /builds/worker/artifacts artifacts && 260 ./mach --log-no-times taskgraph action-callback 261 else: > 262 cd /builds/worker/checkouts/gecko && 263 ln -s /builds/worker/artifacts artifacts && 264 ./mach --log-no-times taskgraph decision 265 --pushlog-id='${push.pushlog_id}' 266 --pushdate='${push.pushdate}' 267 --project='${repository.project}' 268 --owner='${ownerEmail}' 269 --level='${repository.level}' 270 --tasks-for='${tasks_for}' 271 --repository-type=hg 272 --base-repository="$GECKO_BASE_REPOSITORY" 273 --base-rev="$GECKO_BASE_REV" 274 --head-repository="$GECKO_HEAD_REPOSITORY" 275 --head-ref="$GECKO_HEAD_REF" 276 --head-rev="$GECKO_HEAD_REV" 277 ${extraArgs} 278 279 artifacts: 280 'public': 281 type: 'directory' 282 path: '/builds/worker/artifacts' 283 expires: {$eval: expires} 284 'public/docker-contexts': 285 type: 'directory' 286 path: '/builds/worker/checkouts/gecko/docker-contexts' 287 # This needs to be at least the deadline of the 288 # decision task + the docker-image task deadlines. 289 # It is set to a week to allow for some time for 290 # debugging, but they are not useful long-term. 291 expires: {$fromNow: '7 day'} 292 293 extra: 294 $merge: 295 - treeherder: 296 $merge: 297 - machine: 298 platform: gecko-decision 299 - $if: 'tasks_for == "hg-push"' 300 then: 301 symbol: D 302 else: 303 $if: 'tasks_for == "action"' 304 then: 305 groupName: 'action-callback' 306 groupSymbol: AC 307 symbol: "${action.symbol}" 308 else: 309 groupSymbol: cron 310 symbol: "${cron.job_symbol}" 311 - $if: 'tasks_for == "action"' 312 then: 313 parent: '${action.taskGroupId}' 314 action: 315 name: '${action.name}' 316 context: 317 taskGroupId: '${action.taskGroupId}' 318 taskId: {$eval: 'taskId'} 319 input: {$eval: 'input'} 320 clientId: {$eval: 'clientId'} 321 - $if: 'tasks_for == "cron"' 322 then: 323 cron: {$json: {$eval: 'cron'}} 324 - tasks_for: '${tasks_for}' 325 # Email for all pushes should link to treeherder 326 - $if: 'tasks_for == "hg-push"' 327 then: 328 notify: 329 $merge: 330 - email: 331 $merge: 332 - link: 333 text: "Treeherder Jobs" 334 href: "https://treeherder.mozilla.org/#/jobs?repo=${repository.project}&revision=${push.revision}" 335 - $if: 'repository.project == "try"' 336 then: 337 subject: "Thank you for your try submission of ${push.revision}. It's the best!" 338 content: "Your try push has been submitted. It's the best! Use the link to view the status of your jobs." 339 - $if: 'repository.project == "mozilla-central"' 340 then: 341 matrixBody: "${repository.project} push notification: https://treeherder.mozilla.org/#/jobs?repo=${repository.project}&revision=${push.revision}" 342 # Decision task for events originating from Github 343 - $if: 'tasks_for[:6] == "github"' 344 then: 345 $let: 346 $merge: 347 - trustDomain: gecko 348 ownTaskId: {$eval: as_slugid("decision_task")} 349 isPullRequest: false 350 eventType: '${tasks_for[7:]}' # strip out 'github-' 351 eventAction: '${event["action"]}' # empty string if 'action' doesn't exist 352 - $switch: 353 'tasks_for == "github-push"': 354 ownerEmail: '${event.pusher.email}' 355 baseRepoUrl: '${event.repository.html_url}' 356 repoUrl: '${event.repository.html_url}' 357 project: '${event.repository.name}' 358 ref: '${event.ref}' 359 baseRev: '${event.before}' 360 headRev: '${event.after}' 361 'tasks_for[:19] == "github-pull-request"': 362 ownerEmail: '${event.pull_request.user.login}@users.noreply.github.com' 363 baseRepoUrl: '${event.pull_request.base.repo.html_url}' 364 repoUrl: '${event.pull_request.head.repo.html_url}' 365 project: '${event.pull_request.base.repo.name}' 366 # Normalize 'ref' to include 'refs/heads' as pull-requests must be associated 367 # with a head ref. The benefit of doing this is that `run-task` won't attempt 368 # to fetch all tags from the repo just on the off-chance it is a tag. 369 ref: 'refs/heads/${event.pull_request.head.ref}' 370 baseRev: '${event.pull_request.base.sha}' 371 headRev: '${event.pull_request.head.sha}' 372 isPullRequest: true 373 in: 374 $let: 375 shortRef: 376 $if: 'ref[:11] == "refs/heads/"' 377 then: {$eval: 'ref[11:]'} 378 else: ${ref} 379 in: 380 $if: > 381 (eventType == "push" && shortRef == "main") 382 || (isPullRequest && eventAction in ["opened", "reopened", "synchronize"]) 383 then: 384 $let: 385 level: 386 $if: 'eventType == "push" && repoUrl == "https://github.com/mozilla-firefox/firefox" && shortRef == "main"' 387 then: 3 388 else: 1 389 in: 390 taskId: '${ownTaskId}' 391 taskGroupId: '${ownTaskId}' 392 schedulerId: '${trustDomain}-level-${level}' 393 394 created: {$fromNow: ''} 395 deadline: {$fromNow: '1 day'} 396 expires: {$fromNow: '1 year 1 second'} # 1 second so artifacts expire first 397 metadata: 398 owner: "${ownerEmail}" 399 source: "${repoUrl}/raw/${headRev}/.taskcluster.yml" 400 name: "Decision Task (${eventType})" 401 description: 'The task that creates all of the other tasks in the task graph' 402 403 provisionerId: "${trustDomain}-${level}" 404 workerType: "decision" 405 406 tags: 407 createdForUser: "${ownerEmail}" 408 kind: decision-task 409 410 routes: 411 $flattenDeep: 412 - checks 413 - $if: 'eventType == "push"' 414 then: 415 - "tc-treeherder.v2.${project}.${headRev}" 416 - "index.${trustDomain}.v2.${project}.latest.taskgraph.decision" 417 - "index.${trustDomain}.v2.${project}.revision.${headRev}.taskgraph.decision" 418 scopes: 419 $switch: 420 isPullRequest: 421 - 'assume:repo:${baseRepoUrl[8:]}:${eventType}' 422 'eventType == "push"': 423 - 'assume:repo:${repoUrl[8:]}:branch:${shortRef}' 424 dependencies: [] 425 requires: all-completed 426 priority: very-low 427 retries: 5 428 429 payload: 430 env: 431 GECKO_BASE_REPOSITORY: '${baseRepoUrl}' 432 GECKO_BASE_REV: '${baseRev}' 433 GECKO_HEAD_REPOSITORY: '${repoUrl}' 434 GECKO_HEAD_REF: '${ref}' 435 GECKO_HEAD_REV: '${headRev}' 436 GECKO_REPOSITORY_TYPE: git 437 REPOSITORIES: {$json: {gecko: "Mozilla Firefox"}} 438 TASKCLUSTER_CACHES: /builds/worker/checkouts 439 TASKCLUSTER_VOLUMES: /builds/worker/artifacts 440 MOZ_UPLOAD_DIR: /builds/worker/artifacts 441 MOZ_AUTOMATION: '1' 442 # mach generates pyc files when reading `mach_commands.py` 443 # This causes cached_task digest generation to be random for 444 # some tasks. Disable bytecode generation to work around that. 445 PYTHONDONTWRITEBYTECODE: '1' 446 MACH_BUILD_PYTHON_NATIVE_PACKAGE_SOURCE: 'system' 447 cache: 448 "${trustDomain}-level-${level}-${project}-checkouts-git-shallow-v1": /builds/worker/checkouts 449 450 features: 451 taskclusterProxy: true 452 chainOfTrust: true 453 454 image: mozillareleases/taskgraph:run-task-latest 455 maxRunTime: 1800 456 command: 457 - run-task 458 - '--gecko-checkout=/builds/worker/checkouts/gecko' 459 - '--gecko-shallow-clone' 460 - '--' 461 - bash 462 - -cx 463 - > 464 cd /builds/worker/checkouts/gecko && 465 ln -s /builds/worker/artifacts artifacts && 466 ./mach --log-no-times taskgraph decision \ 467 --pushlog-id='0' \ 468 --pushdate='0' \ 469 --project='${project}' \ 470 --owner='${ownerEmail}' \ 471 --level='${level}' \ 472 --repository-type=git \ 473 --tasks-for='${tasks_for}' \ 474 --base-repository='${baseRepoUrl}' \ 475 --base-rev='${baseRev}' \ 476 --head-repository='${repoUrl}' \ 477 --head-ref='${ref}' \ 478 --head-rev='${headRev}' 479 480 artifacts: 481 'public': 482 type: 'directory' 483 path: '/builds/worker/artifacts' 484 expires: {$fromNow: '1 year'} 485 'public/docker-contexts': 486 type: 'directory' 487 path: '/builds/worker/checkouts/gecko/docker-contexts' 488 # This needs to be at least the deadline of the 489 # decision task + the docker-image task deadlines. 490 # It is set to a week to allow for some time for 491 # debugging, but they are not useful long-term. 492 expires: {$fromNow: '7 day'} 493 494 extra: 495 $merge: 496 - treeherder: 497 machine: 498 platform: gecko-decision 499 symbol: D 500 - tasks_for: '${tasks_for}'