Skip to content

Production release train

Ships a release across regions with canaries, agent checks, a risk-based approval path, a customer-facing go signal and cleanup.

Source: scripts/dev/review-runbooks/release-train.yaml. CI checks it with runbook spec validate; teams, people, skills and called runbooks it names resolve against your server when you publish it.

apiVersion: therunbook.app/v1
kind: Runbook
metadata:
id: release-train
title: Production release train
description: Ships a release across regions with canaries, agent checks, a risk-based approval path, a customer-facing go signal and cleanup.
owners: [team:oncall]
tags: [deploy, production, release, agentic]
spec:
execution:
title: "Release ${{ inputs.version }} (${{ inputs.risk }} risk)"
timeout: 6h
concurrency: { key: release-train, limit: 1, on_conflict: queue }
inputs:
version: { type: string, required: true }
risk: { type: enum, options: [low, medium, high], default: medium }
regions: { type: list, items: { type: string }, default: [us-east, us-west, eu-west] }
fields:
outcome: { type: enum, options: [pending, shipped, rolled_back, halted], default: pending, indexed: true }
vars: { decision: undecided }
defaults: { actor: agent, timeout: 20m, on_failure: handoff }
steps:
- id: preflight
title: Preflight
steps:
- id: checks
title: Preflight checks
parallel:
- id: changelog
title: Changelog and release notes are ready
check: true
instructions: Confirm the changelog for ${{ inputs.version }} lists every merged change and links the release notes.
- id: migrations
title: Database migrations are backward compatible
check: true
instructions: Dry-run pending migrations against a replica and check every one is additive.
outputs:
pending: { type: integer, required: true }
- id: freeze
title: No change freeze is active
check: true
instructions: Check the change calendar for a freeze or an open major incident.
- id: build
title: Build and sign the release artifacts
instructions: Build images for ${{ inputs.version }}, sign them and record the digest.
outputs:
digest: { type: string, required: true }
catch:
- id: preflight_failed
title: Tell the release channel preflight failed
instructions: Post which preflight check failed and stop the train.
- id: halt_preflight
set: { fields: { outcome: halted } }
- id: stop_after_preflight
end: { status: failed, message: Preflight failed }
- id: risk_gate
title: Risk gate
switch:
cases:
- when: "${{ inputs.risk == 'high' }}"
steps:
- id: cab_review
title: Change advisory review
approval:
prompt: Approve the high-risk release ${{ inputs.version }}?
options: [ship, hold]
approvers: { teams: [oncall] }
timeout: 4h
on_timeout: escalate
escalate_to: [team:customer-success]
escalation_timeout: 1h
- id: hold_if_rejected
if: "${{ steps.cab_review.outputs.decision == 'hold' }}"
end: { status: cancelled, message: Held by the change advisory review }
- when: "${{ inputs.risk == 'medium' }}"
steps:
- id: lead_signoff
title: Release lead signs off
approval:
prompt: Ship ${{ inputs.version }}?
options: [ship, hold]
approvers: { teams: [oncall] }
timeout: 1h
default:
steps:
- id: auto_ok
set: { vars: { decision: auto } }
- id: canary
title: Canary
join: any
parallel:
- id: canary_us
title: Canary in us-east
check: true
instructions: Deploy to 5% of us-east and compare error rate and p95 latency with the baseline for 15 minutes.
- id: canary_eu
title: Canary in eu-west
check: true
instructions: Deploy to 5% of eu-west and compare error rate and p95 latency with the baseline for 15 minutes.
- id: rollout
title: Roll out per region
foreach:
items: "${{ inputs.regions }}"
as: region
concurrency: 1
max_items: 6
steps:
- id: deploy_region
title: "Deploy to ${{ region }}"
destructive: true
instructions: Roll ${{ inputs.version }} out to every instance in ${{ region }}.
- id: settle
title: "Wait for ${{ region }} to settle"
loop:
until: "${{ steps.region_health.outputs.error_rate <= 0.01 }}"
max_iterations: 4
delay: 20s
steps:
- id: region_health
title: "Error rate in ${{ region }}"
check: true
instructions: Read the 5xx rate in ${{ region }} over the last 5 minutes.
outputs:
error_rate: { type: number, required: true }
- id: verify
title: Post-deploy checks
call:
runbook: post-deploy-checks
with:
service: api
version: "${{ inputs.version }}"
regions: "${{ inputs.regions }}"
- id: announce
title: Announce
parallel:
- id: status_page
title: Update the status page
actor: human
assignee: team:oncall
instructions: Mark the maintenance window complete on the public status page.
checklist: [Maintenance closed, Release notes linked]
- id: customer_go
title: Customer success confirms the go-live email went out
wait: { signal: release_email_sent, timeout: 1d }
- id: shipped
set: { fields: { outcome: shipped }, vars: { decision: shipped } }
outputs:
decision: "${{ vars.decision }}"
finally:
- id: retro_notes
title: Write the release notes summary
instructions: Summarize what shipped, any rollbacks and follow-ups, with the run link.

Publish it to a project:

Terminal window
runbook push release-train.yaml --project <project> --draft
runbook publish release-train --project <project>