release.yml

name: Build, Publish & Deploy

# Single workflow (not three separate ones triggered off each other via
# workflow_run) so every job below shares one checkout of one commit SHA,
# and `needs:` gives real ordering guarantees instead of hoping two
# independently-triggered workflows happen to finish in the right order.
#
# Previously: "Publish images" and "Publish Helm charts" each had their own
# `on: push` trigger, and "Deploy" ran off `workflow_run` of "Publish images"
# only -- so Deploy could apply/restart before the charts workflow had
# finished (or even started), and a workflow_run-triggered job's default
# `actions/checkout` checks out the *current tip of main* rather than the
# commit that triggered the run it's reacting to. Back-to-back pushes could
# therefore deploy a different revision's charts/values than the one whose
# images were just built. Using `needs:` across jobs in one workflow instead
# means everything below is pinned to the same `github.sha`, and `deploy`
# can't start until `publish-charts` has actually finished.

on:
  push:
    branches: [main]

permissions:
  contents: read
  packages: write

jobs:
  changes:
    # Cheap, network-light job: figures out which images actually need a
    # rebuild, and whether charts/ changed, so downstream jobs only do work
    # when relevant. Runs on a GitHub-hosted runner since it doesn't touch
    # the internal registry or cluster.
    runs-on: ubuntu-latest
    outputs:
      matrix: ${{ steps.matrix.outputs.result }}
      charts: ${{ steps.filter.outputs.charts }}
    steps:
      - uses: actions/checkout@v4

      - uses: dorny/paths-filter@v3
        id: filter
        with:
          filters: |
            workflow:
              - '.github/workflows/release.yml'
            agent-orchestrator:
              - 'package.json'
              - 'package-lock.json'
              - 'packages/messaging/**'
              - 'apps/agent-orchestrator/**'
            opencode-swe-agent:
              - 'package.json'
              - 'package-lock.json'
              - 'packages/messaging/**'
              - 'packages/agent-runtime/**'
              - 'packages/github-app-auth/**'
              - 'apps/opencode-swe-agent/**'
            claude-code-swe-agent:
              - 'package.json'
              - 'package-lock.json'
              - 'packages/messaging/**'
              - 'packages/agent-runtime/**'
              - 'packages/github-app-auth/**'
              - 'apps/claude-code-swe-agent/**'
            integration-gateway:
              - 'package.json'
              - 'package-lock.json'
              - 'packages/github-app-auth/**'
              - 'apps/integration-gateway/**'
            connection-broker:
              - 'package.json'
              - 'package-lock.json'
              - 'apps/connection-broker/**'
            mcp-broker:
              - 'package.json'
              - 'package-lock.json'
              - 'apps/mcp-broker/**'
            recipe-scraper:
              - 'package.json'
              - 'package-lock.json'
              - 'packages/messaging/**'
              - 'tools/recipe-scraper/**'
            recipe-publisher:
              - 'package.json'
              - 'package-lock.json'
              - 'packages/messaging/**'
              - 'tools/recipe-publisher/**'
            web-search:
              - 'package.json'
              - 'package-lock.json'
              - 'packages/messaging/**'
              - 'tools/web-search/**'
            web-fetch:
              - 'package.json'
              - 'package-lock.json'
              - 'packages/messaging/**'
              - 'tools/web-fetch/**'
            image-gen:
              - 'package.json'
              - 'package-lock.json'
              - 'packages/messaging/**'
              - 'tools/image-gen/**'
            kubectl-readonly:
              - 'package.json'
              - 'package-lock.json'
              - 'packages/messaging/**'
              - 'tools/kubectl-readonly/**'
            signoz-query:
              - 'package.json'
              - 'package-lock.json'
              - 'packages/messaging/**'
              - 'tools/signoz-query/**'
            github-tool:
              - 'package.json'
              - 'package-lock.json'
              - 'packages/messaging/**'
              - 'packages/github-app-auth/**'
              - 'tools/github/**'
            glyph:
              - 'package.json'
              - 'package-lock.json'
              - 'packages/messaging/**'
              - 'tools/glyph/**'
            ssh-tool:
              - 'package.json'
              - 'package-lock.json'
              - 'packages/messaging/**'
              - 'tools/ssh/**'
            core-controller:
              - 'controllers/core-controller/**'
            temporal-engine:
              - 'engines/temporal/**'
            localtool-executor:
              - 'sidecars/localtool-executor/**'
            charts:
              - 'charts/**'

      - name: Compute image matrix
        id: matrix
        # Which images to build/push comes from .github/release-images.json --
        # one entry per image naming the `filter` (a key in the block above)
        # that governs it, plus its Dockerfile/context/build_args. To add a
        # new image: add a filter entry above for its paths (or reuse an
        # existing one), then one entry to release-images.json. Nothing here
        # needs to change -- `changes` is dorny/paths-filter's own JSON array
        # of every filter name that matched, so there is no per-image env var
        # or jq flag to keep in sync with the filters block by hand.
        env:
          WORKFLOW_CHANGED: ${{ steps.filter.outputs.workflow }}
          CHANGED_FILTERS: ${{ steps.filter.outputs.changes }}
        run: |
          if [ "$WORKFLOW_CHANGED" = "true" ]; then
            # The workflow itself changed -- rebuild everything so a broken
            # matrix/cache config surfaces immediately, not on the next
            # unrelated image change.
            MATRIX=$(jq -c '.' .github/release-images.json)
          else
            MATRIX=$(jq -c --argjson changed "$CHANGED_FILTERS" \
              'map(select(.filter as $f | $changed | index($f) != null))' \
              .github/release-images.json)
          fi

          echo "Building: $(echo "$MATRIX" | jq -c '[.[].image]')"
          echo "result=$MATRIX" >> "$GITHUB_OUTPUT"

  build-images:
    # Multi-arch (linux/amd64 + linux/arm64), built NATIVELY: every image x
    # architecture pair is its own job, amd64 on GitHub's x86 runners and arm64
    # on GitHub's arm64 runners (free for public repos). That beats a single
    # `platforms: linux/amd64,linux/arm64` build two ways: no QEMU emulation
    # (several times faster for npm/go/apt-heavy stages), and maximum
    # parallelism -- there is deliberately no `max-parallel`, so every pair runs
    # at once, bounded only by the account's concurrent-job limit.
    #
    # GitHub-hosted, not the self-hosted ARC runner: building and pushing needs
    # only GHCR (via GITHUB_TOKEN), never the cluster. `deploy` below stays on
    # ARC because it is the only runner holding cluster credentials.
    #
    # Each job pushes its single-arch image BY DIGEST (untagged) and hands the
    # digest to `publish-images`, which stitches them into one multi-arch
    # manifest per image and applies the real tags. So `:latest`/`:<sha>` only
    # ever move once BOTH architectures exist -- never a half-published image.
    needs: changes
    # push-to-main only (collaborators), but keep a fork from pushing to our
    # GHCR namespace if this workflow is ever re-targeted.
    if: github.repository == 'imaustink/agent-controller' && needs.changes.outputs.matrix != '[]'
    runs-on: ${{ matrix.arch == 'arm64' && 'ubuntu-24.04-arm' || 'ubuntu-24.04' }}
    timeout-minutes: 60
    strategy:
      fail-fast: false
      matrix:
        img: ${{ fromJson(needs.changes.outputs.matrix) }}
        arch: [amd64, arm64]
    name: build ${{ matrix.img.image }} (${{ matrix.arch }})
    steps:
      - uses: actions/checkout@v4

      - uses: docker/setup-buildx-action@v3

      # Public images on GHCR, one package per image under a shared
      # `agent-controller/` path: ghcr.io/imaustink/agent-controller/<image>.
      # A package is created on its first push, so adding an image is still
      # just an entry in release-images.json. GITHUB_TOKEN can push because
      # the workflow grants `packages: write`.
      - name: Log in to GHCR
        run: |
          echo "$GITHUB_TOKEN" | docker login ghcr.io \
            -u "${{ github.actor }}" --password-stdin
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

      - name: Build & push ${{ matrix.img.image }} (linux/${{ matrix.arch }}) by digest
        id: build
        uses: docker/build-push-action@v6
        with:
          context: ${{ matrix.img.context }}
          file: ${{ matrix.img.dockerfile }}
          build-args: ${{ matrix.img.build_args }}
          platforms: linux/${{ matrix.arch }}
          # Links each package to this repo on GHCR, so it shows up on the
          # repo page and can inherit its access settings.
          labels: |
            org.opencontainers.image.source=https://github.com/${{ github.repository }}
          # Untagged, by digest: tags are applied to the multi-arch manifest in
          # `publish-images`, so a single arch never lands under :latest alone.
          outputs: type=image,name=ghcr.io/imaustink/agent-controller/${{ matrix.img.image }},push-by-digest=true,name-canonical=true,push=true
          # Registry cache, per image AND arch, in the image's own GHCR package.
          # Not the GHA cache backend: ~24 images x 2 arches with `mode=max`
          # (which also caches the npm ci / go mod download / apt-get build
          # stages) would blow through the repo's 10GB Actions cache and evict
          # itself every run. `ignore-error` so a cache-export hiccup can never
          # fail a publish; `image-manifest`/`oci-mediatypes` for GHCR.
          cache-from: type=registry,ref=ghcr.io/imaustink/agent-controller/${{ matrix.img.image }}:buildcache-${{ matrix.arch }}
          cache-to: type=registry,ref=ghcr.io/imaustink/agent-controller/${{ matrix.img.image }}:buildcache-${{ matrix.arch }},mode=max,image-manifest=true,oci-mediatypes=true,ignore-error=true

      - name: Export digest
        run: |
          mkdir -p "$RUNNER_TEMP/digests"
          digest="${{ steps.build.outputs.digest }}"
          touch "$RUNNER_TEMP/digests/${digest#sha256:}"

      # `__` delimiters so one image's pattern can never match another image
      # that merely shares a prefix (e.g. `github` vs a future `github-x`).
      - uses: actions/upload-artifact@v4
        with:
          name: digests__${{ matrix.img.image }}__${{ matrix.arch }}
          path: ${{ runner.temp }}/digests/*
          if-no-files-found: error
          retention-days: 1

  publish-images:
    # One job per image: combine its per-arch digests into a single multi-arch
    # manifest list and tag it. `:latest` tracks main; `:<sha>` is immutable,
    # for pinning. Runs only when EVERY build-images job succeeded (default
    # `needs` semantics), so tags never point at a partial set of arches.
    needs: [changes, build-images]
    if: github.repository == 'imaustink/agent-controller' && needs.changes.outputs.matrix != '[]'
    runs-on: ubuntu-latest
    strategy:
      fail-fast: false
      matrix:
        img: ${{ fromJson(needs.changes.outputs.matrix) }}
    name: publish ${{ matrix.img.image }}
    steps:
      - uses: actions/download-artifact@v4
        with:
          path: ${{ runner.temp }}/digests
          pattern: digests__${{ matrix.img.image }}__*
          merge-multiple: true

      - uses: docker/setup-buildx-action@v3

      - name: Log in to GHCR
        run: |
          echo "$GITHUB_TOKEN" | docker login ghcr.io \
            -u "${{ github.actor }}" --password-stdin
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

      - name: Create & push multi-arch manifest
        working-directory: ${{ runner.temp }}/digests
        env:
          IMAGE: ghcr.io/imaustink/agent-controller/${{ matrix.img.image }}
        run: |
          # The index-level source annotation is what GHCR reads to link a
          # multi-arch package to this repo (the per-arch label alone isn't).
          refs=()
          for digest in *; do refs+=("$IMAGE@sha256:$digest"); done
          docker buildx imagetools create \
            --annotation "index:org.opencontainers.image.source=https://github.com/${GITHUB_REPOSITORY}" \
            -t "$IMAGE:latest" \
            -t "$IMAGE:$GITHUB_SHA" \
            "${refs[@]}"

      - name: Verify both architectures are published
        env:
          IMAGE: ghcr.io/imaustink/agent-controller/${{ matrix.img.image }}
        run: |
          # Ignore attestation entries (platform unknown/unknown) that buildx
          # adds for provenance; assert the real platforms are exactly the two.
          platforms=$(docker buildx imagetools inspect --raw "$IMAGE:$GITHUB_SHA" \
            | jq -r '[.manifests[].platform | select(.os != "unknown") | "\(.os)/\(.architecture)"] | sort | unique | join(",")')
          echo "$IMAGE:$GITHUB_SHA -> $platforms"
          if [ "$platforms" != "linux/amd64,linux/arm64" ]; then
            echo "::error::$IMAGE:$GITHUB_SHA has platforms '$platforms', expected linux/amd64,linux/arm64"
            exit 1
          fi

  publish-charts:
    # Publishes the two Helm charts (charts/agent-controller,
    # charts/community-components) as OCI artifacts to ghcr.io whenever
    # something under charts/ changes on main. Runs on a GitHub-hosted
    # runner, like the image jobs above -- it only needs outbound internet to
    # ghcr.io and a GITHUB_TOKEN, not access to the internal cluster (only
    # `deploy` needs that, which is why it alone stays self-hosted).
    #
    # OCI chart tags are MUTABLE: pushing the same Chart.yaml `version` again
    # overwrites that tag, and Argo (pinned to it) then caches the first render
    # and never re-applies later template changes -- the stale-ClusterRole
    # crashloop from PR #257. To make a reused tag impossible, this job bumps
    # each chart's patch version, commits it back to main (the version stays the
    # source of truth, in git, as clean sequential semver), and publishes THAT.
    #
    # Install with:
    #   helm install agent-controller oci://ghcr.io/imaustink/charts/agent-controller --version <version>
    #   helm install community-components oci://ghcr.io/imaustink/charts/community-components --version <version>
    needs: changes
    runs-on: ubuntu-latest
    # Guard so a fork that somehow re-targets this job cannot publish under our package namespace.
    if: github.repository == 'imaustink/agent-controller' && needs.changes.outputs.charts == 'true'
    # contents: write to push the version bump back to main; packages: write to
    # push the chart to GHCR (this job's override replaces the workflow default).
    permissions:
      contents: write
      packages: write
    # Serialize release runs so two can't read the same version and both bump to
    # it. Queue rather than cancel, so no release is dropped.
    concurrency:
      group: chart-release
      cancel-in-progress: false
    steps:
      # ref: main (not the triggering SHA) + full history so the bump is computed
      # from, and pushed onto, the current branch tip -- a fast-forward under the
      # concurrency serialization above.
      - uses: actions/checkout@v4
        with:
          ref: main
          fetch-depth: 0

      - name: Bump chart patch versions
        # Edit only here; the commit + push happens after packaging has
        # validated the build (below), and before the OCI push.
        run: |
          set -euo pipefail
          for chart in charts/agent-controller charts/community-components; do
            f="$chart/Chart.yaml"
            base=$(awk '/^version:/{print $2; exit}' "$f")
            major=${base%%.*}; rest=${base#*.}; minor=${rest%%.*}; patch=${rest#*.}
            case "$patch" in
              *[!0-9]* | "") echo "::error::$f version '$base' is not X.Y.Z"; exit 1 ;;
            esac
            new="${major}.${minor}.$((patch + 1))"
            sed -i "s/^version:.*/version: ${new}/" "$f"
            echo "${chart#charts/}: ${base} -> ${new}"
          done

      - name: Sync core-controller CRDs into the Helm chart
        # charts/agent-controller/charts/core-controller/crds/*.yaml is
        # generated (not committed, see its README.md) -- Helm's crds/
        # convention needs the files physically present before `helm
        # dependency update`/`helm package` will pick them up.
        run: cp controllers/core-controller/config/crd/bases/*.yaml charts/agent-controller/charts/core-controller/crds/

      - name: Set up Helm
        uses: azure/setup-helm@v4
        with:
          version: v3.18.4

      - name: Helm login to GHCR
        run: |
          echo "${{ secrets.GITHUB_TOKEN }}" | helm registry login ghcr.io \
            -u "${{ github.actor }}" --password-stdin

      - name: Fetch agent-controller chart dependencies
        run: |
          # Helm doesn't recurse into file:// subchart dependencies, so the
          # agent-orchestrator subchart's own qdrant dependency needs its own
          # `dependency update` before the parent chart's.
          helm dependency update charts/agent-controller/charts/agent-orchestrator
          helm dependency update charts/agent-controller

      - name: Package charts
        # Uses the bumped versions now in each Chart.yaml. Done before the git
        # push so a chart that fails to build aborts the job without having
        # committed a version to main.
        run: |
          mkdir -p /tmp/charts
          helm package charts/agent-controller -d /tmp/charts
          helm package charts/community-components -d /tmp/charts

      - name: Commit the version bump back to main
        # git BEFORE the OCI push: the version is claimed in git first, so a
        # push that loses a race fails the job before any tag is written — a
        # reused/overwritten OCI tag (the bug this prevents) is impossible. A
        # failure AFTER this leaves an unpublished version (a harmless gap in the
        # sequence), never an overwrite.
        #
        # [skip ci] so this commit — which only touches Chart.yaml under charts/
        # — does not re-trigger this workflow and loop.
        run: |
          set -euo pipefail
          git config user.name "github-actions[bot]"
          git config user.email "41898282+github-actions[bot]@users.noreply.github.com"
          git add charts/agent-controller/Chart.yaml charts/community-components/Chart.yaml
          git commit -m "chore(release): bump chart versions [skip ci]"
          git push origin HEAD:main

      - name: Push charts to GHCR
        run: |
          for tgz in /tmp/charts/*.tgz; do
            helm push "$tgz" oci://ghcr.io/imaustink/charts
          done

  deploy:
    # build-images is listed explicitly, not just reached through
    # publish-images: when a build fails, publish-images is SKIPPED (not
    # failed), and the `always()` guard below only rejects failure/cancellation
    # among DIRECT needs -- so without it a broken build would deploy anyway.
    # `changes` is listed for the same reason: if it fails, every downstream
    # job is skipped, and deploy would otherwise run on the unchanged :latest.
    needs: [changes, build-images, publish-images, publish-charts]
    # Self-hosted ARC runner, deliberately NOT GitHub-hosted like the build jobs
    # above: it is the only runner with credentials for the cluster. Only ever
    # reachable via push to main (collaborators only).
    runs-on: arc-runner-set-agent-controller
    # Guard so a fork that somehow re-targets this job cannot run it on our infra.
    #
    # `always()` because build-images/publish-images/publish-charts are legitimately
    # skipped when nothing relevant changed (e.g. a docs-only push) -- a
    # skip must not block deploy the way a failure should. Explicitly reject
    # failure/cancellation instead of just checking for 'success' so deploy
    # doesn't run (and restart images) on top of a build it can't trust.
    if: |
      always() &&
      github.repository == 'imaustink/agent-controller' &&
      !contains(needs.*.result, 'failure') &&
      !contains(needs.*.result, 'cancelled')
    # Override the workflow-level permissions (contents: read, packages: write):
    # this job never pushes packages, but it does need `issues: write` so the
    # rollback-on-failure step can open the ai-triage ticket.
    permissions:
      contents: read
      issues: write
    steps:
      - uses: actions/checkout@v4

      - name: Set up Helm
        uses: azure/setup-helm@v4
        with:
          version: v3.18.4

      - uses: azure/setup-kubectl@v4

      - name: Sync core-controller CRDs into the Helm chart
        # charts/agent-controller/charts/core-controller/crds/*.yaml is
        # generated (not committed, see its README.md) -- Helm's crds/
        # convention needs the files physically present before `helm
        # dependency update`/`helm upgrade` will pick them up. Same step
        # publish-charts runs before packaging.
        #
        # This only covers a FIRST install. See the next step for why.
        run: cp controllers/core-controller/config/crd/bases/*.yaml charts/agent-controller/charts/core-controller/crds/

      - name: Apply CRDs
        # Helm's crds/ directory is INSTALL-ONLY: `helm upgrade` never creates a
        # newly-added CRD and never updates an existing one
        # (charts/agent-controller/charts/core-controller/crds/README.md). On a
        # cluster whose release predates a CRD, that CRD simply never arrives,
        # and a CRD that gained a field keeps serving the old schema -- so the
        # API server silently PRUNES the new field out of every CR written to
        # it. A missing field is not a visible failure; it is a resource that
        # applies cleanly and then behaves as though half its spec were never
        # set.
        #
        # Until now the only thing closing that gap was a human remembering to
        # run kubectl by hand. values-production.yaml carried FOUR separate
        # "applied manually once" comments to that effect -- for a
        # since-retired routing CRD (twice: the CRD, then a field added to it)
        # and for Agent.initContainers. Every one of those was a deploy that
        # looked green and shipped a cluster the charts did not describe.
        #
        # --server-side: the target cluster's CRDs have INCONSISTENT provenance,
        #   which is precisely why a single client-side apply cannot be trusted
        #   to reconcile them. Field managers observed on the live cluster:
        #     - tools/skills/toolruns/localtools -> "helm" only, no
        #       last-applied-configuration annotation. A client-side apply here
        #       diffs against an empty baseline: able to add and change fields,
        #       but unable to REMOVE one dropped from the CRD, so a deleted
        #       property would linger in the served schema indefinitely.
        #     - agentruns/agents -> "helm" AND "kubectl-client-side-apply":
        #       Helm-installed, then hand-patched later.
        #     - the since-retired routing CRD -> "kubectl-client-side-apply"
        #       ONLY, with no helm manager at all. It was never installed by
        #       Helm on this cluster; it arrived entirely by hand.
        #   Those last two rows are the four "applied manually once" comments in
        #   values-production.yaml, still legible in the cluster's own field
        #   ownership. Server-side apply diffs against real field ownership
        #   rather than an annotation that only some objects carry. It is also
        #   what e2e/scripts/up.sh already uses, so dev and CD converge.
        # --force-conflicts: consequently those fields are owned by other
        #   managers (helm, and on three of them kubectl-client-side-apply too),
        #   and server-side apply refuses to take them over without this. Safe
        #   here because every one of those managers wrote the same generated
        #   artifact -- the chart's copy and this one both come from the same
        #   *_types.go source of truth -- so there is no third version of the
        #   truth to lose. Verified by applying all 7 as the deploy runner's own
        #   ServiceAccount against the live cluster (server dry-run): all
        #   succeed, with a non-fatal one-time annotation-migration warning on
        #   the hand-patched ones, which resolves itself on the next apply.
        run: |
          # Preflight the one permission this step needs, before touching
          # anything. CRDs are CLUSTER-scoped; every other kubectl/helm call in
          # this job is namespaced, so this is the first thing here to require
          # cluster-scoped rights and the runner's ServiceAccount may well not
          # have them. Without this check the failure is a raw RBAC error from
          # whichever CRD happened to be applied first -- accurate, but it names
          # a resource rather than the missing grant. The ServiceAccount is the
          # ARC runner pod's own (there is no kubeconfig step in this job) and
          # is cluster infrastructure, not something this repo can grant itself.
          # `-A` only to keep the check quiet: without it kubectl warns that
          # customresourcedefinitions is not namespace-scoped (the job's context
          # carries a namespace). stderr is deliberately NOT suppressed, so a
          # connectivity failure here is still visible rather than being
          # misreported as a permissions problem.
          if ! kubectl auth can-i create customresourcedefinitions.apiextensions.k8s.io -A >/dev/null; then
            echo "::error::The deploy runner's ServiceAccount cannot write CRDs, so this step cannot apply them."
            echo "CRDs are cluster-scoped; the rest of this job is namespaced, which is why nothing caught this earlier."
            echo "Grant the ARC runner's ServiceAccount a ClusterRole with:"
            echo "  apiGroups: [\"apiextensions.k8s.io\"]"
            echo "  resources: [\"customresourcedefinitions\"]"
            echo "  verbs: [\"get\", \"list\", \"create\", \"update\", \"patch\"]"
            echo "Until then every CRD change must be applied by hand, which is the gap this step exists to close."
            exit 1
          fi

          kubectl apply --server-side --force-conflicts \
            -f controllers/core-controller/config/crd/bases/

          # An established CRD is the precondition for community-components'
          # Tool/Skill/Agent/EventSource/EventTrigger CRs below. Without the wait, a
          # freshly-created CRD can still be un-established when Helm applies
          # the first CR against it, which surfaces as a "no matches for kind"
          # failure that succeeds on a re-run -- the shape of flake that gets
          # dismissed as infrastructure noise.
          for crd in controllers/core-controller/config/crd/bases/*.yaml; do
            name=$(kubectl get -f "$crd" -o jsonpath='{.metadata.name}')
            kubectl wait --for=condition=established --timeout=60s "crd/$name"
          done

      - name: Fetch agent-controller chart dependencies
        run: |
          # Helm doesn't recurse into file:// subchart dependencies, so the
          # agent-orchestrator subchart's own qdrant dependency needs its own
          # `dependency update` before the parent chart's.
          helm dependency update charts/agent-controller/charts/agent-orchestrator
          helm dependency update charts/agent-controller

      # Pins the Deployments' `:latest` to the digest it names right now, in
      # the release itself, so `helm rollback` restores the image as well as
      # the chart (see the script's header for the incident behind this).
      - name: Pin image digests
        run: ./scripts/pin-image-digests.sh "$RUNNER_TEMP/image-pins.yaml"

      # --create-namespace on both: `-n controller-agent` does not create the
      # namespace, so on a cluster that has never had this stack the very first
      # deploy fails on a missing namespace rather than bootstrapping it. A
      # no-op once it exists.
      - name: Helm upgrade agent-controller
        id: upgrade_agent
        run: |
          helm upgrade --install agent-controller charts/agent-controller \
            -f charts/agent-controller/values-production.yaml \
            -f "$RUNNER_TEMP/image-pins.yaml" \
            -n controller-agent --create-namespace --timeout 5m

      - name: Helm upgrade community-components
        id: upgrade_community
        run: |
          helm upgrade --install community-components charts/community-components \
            -f charts/community-components/values-production.yaml \
            -n controller-agent --create-namespace --timeout 5m

      # ADR 0048 retired the IntegrationRoute kind. "Apply CRDs" above only ever
      # adds/updates, so the old CRD would linger on the cluster forever. It
      # must go AFTER the community-components upgrade, never before: Helm
      # rebuilds the previous release's manifest to diff it, and a kind the API
      # server no longer serves fails that upgrade with "resource mapping not
      # found". Idempotent; a no-op once the CRD is gone. Non-fatal: the runner's
      # CRD ClusterRole (see "Apply CRDs") is not guaranteed to include
      # `delete`, and a leftover CRD with no controller is inert -- not worth
      # failing (and rolling back) an otherwise good deploy over.
      - name: Remove retired CRDs
        run: |
          if kubectl auth can-i delete customresourcedefinitions.apiextensions.k8s.io -A >/dev/null; then
            kubectl delete crd integrationroutes.core.controller-agent.dev --ignore-not-found
          else
            echo "::warning::Runner cannot delete CRDs; remove integrationroutes.core.controller-agent.dev by hand."
          fi

      # Tool/Agent Jobs (recipe-scraper, web-search, web-fetch,
      # recipe-publisher, opencode-swe-agent) are launched fresh per-invocation
      # by core-controller and always pull `:latest` -- no restart needed.
      # The long-running Deployments are pinned by digest ("Pin image
      # digests"), so a new image already rolled them; the restart still
      # picks up Secrets/ConfigMaps that changed without a template change.
      - name: Rollout restart
        id: rollout
        run: |
          kubectl rollout restart deploy/agent-orchestrator deploy/core-controller deploy/agent-controller-integration-gateway -n controller-agent
          kubectl rollout status deploy/agent-orchestrator deploy/core-controller deploy/agent-controller-integration-gateway -n controller-agent --timeout=5m

      # If any deploy step above failed, roll the affected workloads back to
      # their previous working revision and open an `ai-triage` ticket with the
      # full failure diagnostics. `if: failure()` so this only runs on a bad
      # rollout; the job still ends red (a prior step failed) so the failure is
      # never silently swallowed -- the ticket is the durable, actionable record.
      - name: Rollback & open ai-triage ticket on failure
        if: failure()
        env:
          UPGRADE_AGENT_OUTCOME: ${{ steps.upgrade_agent.outcome }}
          UPGRADE_COMMUNITY_OUTCOME: ${{ steps.upgrade_community.outcome }}
          ROLLOUT_OUTCOME: ${{ steps.rollout.outcome }}
          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        run: ./scripts/rollback-deploy.sh