|
| 1 | +name: Model compatibility |
| 2 | + |
| 3 | +# Does every model the picker offers still answer through the clients OpenBot ships? The picker |
| 4 | +# lists each provider's models live, but every harness image carries its model client at a fixed |
| 5 | +# version (the Claude Agent SDK harness bundles a Claude Code CLI, the LangGraph harness sends the |
| 6 | +# ChatGPT plan's backend a fixed Codex client version, the others carry the provider SDKs they were |
| 7 | +# built with), and a provider can refuse a new model to an older client. scripts/model-compat.ts |
| 8 | +# starts each harness image, one job per harness, and sends it one request per listed model on |
| 9 | +# every provider that has a key here. See docs/releasing.md, "Model compatibility". |
| 10 | +# |
| 11 | +# Keys are repository secrets, one per provider. A provider without one is reported as skipped, |
| 12 | +# not failed. On any failure the run fails and opens (or updates) one issue titled "Model |
| 13 | +# compatibility check failing"; a passing run closes it. |
| 14 | +# |
| 15 | +# What answered on the `source` image is the model lock (shared/model-lock.json), the models the |
| 16 | +# picker may offer for each harness. When a model newly answers or stops answering, the run opens |
| 17 | +# or updates one pull request that refreshes the lock. See docs/releasing.md, "Model lock". |
| 18 | +# |
| 19 | +# Runs only on a schedule and by hand, on main: never on a pull request, so no fork's code ever |
| 20 | +# runs with these keys. |
| 21 | +on: |
| 22 | + schedule: |
| 23 | + - cron: "41 6 * * *" |
| 24 | + workflow_dispatch: |
| 25 | + inputs: |
| 26 | + tags: |
| 27 | + description: Harness images to check, comma-separated. `latest` is the published release people run; `source` is built from this commit, which is what the next release ships. |
| 28 | + type: string |
| 29 | + default: latest,source |
| 30 | + required: false |
| 31 | + |
| 32 | +permissions: {} |
| 33 | + |
| 34 | +concurrency: |
| 35 | + group: model-compat |
| 36 | + cancel-in-progress: false |
| 37 | + |
| 38 | +jobs: |
| 39 | + # The published harnesses, and every engine the model lock names whether it is published yet or |
| 40 | + # not (an engine new Bots run on is locked, and checked, from before its first release). |
| 41 | + harnesses: |
| 42 | + name: List the harnesses to check |
| 43 | + if: github.repository == 'CopilotKit/OpenBot' && github.ref == 'refs/heads/main' |
| 44 | + runs-on: ubuntu-latest |
| 45 | + timeout-minutes: 5 |
| 46 | + permissions: |
| 47 | + contents: read |
| 48 | + outputs: |
| 49 | + images: ${{ steps.list.outputs.images }} |
| 50 | + steps: |
| 51 | + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 |
| 52 | + with: |
| 53 | + persist-credentials: false |
| 54 | + - uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2.2.0 |
| 55 | + with: |
| 56 | + bun-version: 1.4.3 |
| 57 | + - id: list |
| 58 | + run: echo "images=$(bun scripts/model-compat.ts harnesses)" >> "$GITHUB_OUTPUT" |
| 59 | + |
| 60 | + check: |
| 61 | + name: ${{ matrix.harness }} |
| 62 | + needs: harnesses |
| 63 | + runs-on: ubuntu-latest |
| 64 | + timeout-minutes: 90 |
| 65 | + permissions: |
| 66 | + contents: read |
| 67 | + strategy: |
| 68 | + fail-fast: false |
| 69 | + matrix: |
| 70 | + harness: ${{ fromJSON(needs.harnesses.outputs.images) }} |
| 71 | + steps: |
| 72 | + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 |
| 73 | + with: |
| 74 | + persist-credentials: false |
| 75 | + - uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2.2.0 |
| 76 | + with: |
| 77 | + bun-version: 1.4.3 |
| 78 | + # Writes its results and exits 0 whatever they say: the report job below decides. |
| 79 | + - name: One request per model through this harness |
| 80 | + env: |
| 81 | + HARNESS: ${{ matrix.harness }} |
| 82 | + TAGS: ${{ inputs.tags || 'latest,source' }} |
| 83 | + ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} |
| 84 | + OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} |
| 85 | + GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }} |
| 86 | + XAI_API_KEY: ${{ secrets.XAI_API_KEY }} |
| 87 | + run: >- |
| 88 | + bun scripts/model-compat.ts --harness "$HARNESS" --tags "$TAGS" --exit-zero |
| 89 | + --out "$RUNNER_TEMP/model-compat-$HARNESS.json" |
| 90 | + - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 |
| 91 | + with: |
| 92 | + name: model-compat-${{ matrix.harness }} |
| 93 | + path: ${{ runner.temp }}/model-compat-${{ matrix.harness }}.json |
| 94 | + if-no-files-found: error |
| 95 | + retention-days: 14 |
| 96 | + |
| 97 | + report: |
| 98 | + name: Report |
| 99 | + needs: [harnesses, check] |
| 100 | + if: always() && needs.harnesses.result == 'success' |
| 101 | + runs-on: ubuntu-latest |
| 102 | + timeout-minutes: 10 |
| 103 | + permissions: |
| 104 | + contents: read |
| 105 | + issues: write # open, update or close the one "Model compatibility check failing" issue |
| 106 | + env: |
| 107 | + GH_TOKEN: ${{ github.token }} |
| 108 | + ISSUE_TITLE: Model compatibility check failing |
| 109 | + RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} |
| 110 | + steps: |
| 111 | + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 |
| 112 | + with: |
| 113 | + persist-credentials: false |
| 114 | + - uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2.2.0 |
| 115 | + with: |
| 116 | + bun-version: 1.4.3 |
| 117 | + - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 |
| 118 | + with: |
| 119 | + pattern: model-compat-* |
| 120 | + path: ${{ runner.temp }}/results |
| 121 | + merge-multiple: true |
| 122 | + - name: Summarize |
| 123 | + id: summary |
| 124 | + run: | |
| 125 | + set +e |
| 126 | + shopt -s nullglob |
| 127 | + bun scripts/model-compat.ts report --summary "$RUNNER_TEMP/summary.md" \ |
| 128 | + --issue "$RUNNER_TEMP/issue-table.md" "$RUNNER_TEMP"/results/*.json |
| 129 | + code=$? |
| 130 | + echo "code=$code" >> "$GITHUB_OUTPUT" |
| 131 | + if [ -f "$RUNNER_TEMP/summary.md" ]; then cat "$RUNNER_TEMP/summary.md" >> "$GITHUB_STEP_SUMMARY"; fi |
| 132 | + exit 0 |
| 133 | + - name: Open or update the issue |
| 134 | + if: steps.summary.outputs.code != '0' |
| 135 | + run: | |
| 136 | + set -euo pipefail |
| 137 | + body="$RUNNER_TEMP/issue.md" |
| 138 | + { |
| 139 | + echo "The model compatibility check failed on $(date -u +%Y-%m-%d): $RUN_URL" |
| 140 | + echo |
| 141 | + echo "A model refused by a pinned client means that client needs bumping: see docs/releasing.md, \"Model compatibility\". This issue is updated by every failing run and closed by the first passing one." |
| 142 | + echo |
| 143 | + if [ -f "$RUNNER_TEMP/issue-table.md" ]; then |
| 144 | + cat "$RUNNER_TEMP/issue-table.md" |
| 145 | + else |
| 146 | + echo "No harness job wrote results. See the run." |
| 147 | + fi |
| 148 | + } > "$body" |
| 149 | + number=$(gh issue list --repo "$GITHUB_REPOSITORY" --state open --search "\"$ISSUE_TITLE\" in:title" \ |
| 150 | + --json number,title --jq "map(select(.title == env.ISSUE_TITLE)) | .[0].number // empty") |
| 151 | + if [ -n "$number" ]; then |
| 152 | + gh issue edit "$number" --repo "$GITHUB_REPOSITORY" --body-file "$body" |
| 153 | + else |
| 154 | + gh issue create --repo "$GITHUB_REPOSITORY" --title "$ISSUE_TITLE" --body-file "$body" |
| 155 | + fi |
| 156 | + - name: Close the issue when everything passes |
| 157 | + if: steps.summary.outputs.code == '0' |
| 158 | + run: | |
| 159 | + set -euo pipefail |
| 160 | + number=$(gh issue list --repo "$GITHUB_REPOSITORY" --state open --search "\"$ISSUE_TITLE\" in:title" \ |
| 161 | + --json number,title --jq "map(select(.title == env.ISSUE_TITLE)) | .[0].number // empty") |
| 162 | + if [ -n "$number" ]; then |
| 163 | + gh issue close "$number" --repo "$GITHUB_REPOSITORY" --comment "Passing again: $RUN_URL" |
| 164 | + fi |
| 165 | + - name: Fail when a model failed |
| 166 | + if: steps.summary.outputs.code != '0' |
| 167 | + run: | |
| 168 | + echo "::error::Model compatibility check failing. See the summary above and the issue titled \"$ISSUE_TITLE\"." |
| 169 | + exit 1 |
| 170 | +
|
| 171 | + lock: |
| 172 | + name: Refresh the model lock |
| 173 | + needs: [harnesses, check] |
| 174 | + if: always() && needs.harnesses.result == 'success' |
| 175 | + runs-on: ubuntu-latest |
| 176 | + timeout-minutes: 10 |
| 177 | + permissions: |
| 178 | + contents: write # push the one refresh branch |
| 179 | + pull-requests: write # open or update its pull request |
| 180 | + env: |
| 181 | + BRANCH: model-lock/refresh |
| 182 | + RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} |
| 183 | + steps: |
| 184 | + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 |
| 185 | + with: |
| 186 | + persist-credentials: false |
| 187 | + - uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2.2.0 |
| 188 | + with: |
| 189 | + bun-version: 1.4.3 |
| 190 | + - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 |
| 191 | + with: |
| 192 | + pattern: model-compat-* |
| 193 | + path: ${{ runner.temp }}/results |
| 194 | + merge-multiple: true |
| 195 | + - name: Write the lock from the source images' results |
| 196 | + id: lock |
| 197 | + run: | |
| 198 | + set -euo pipefail |
| 199 | + shopt -s nullglob |
| 200 | + files=("$RUNNER_TEMP"/results/*.json) |
| 201 | + if [ "${#files[@]}" -eq 0 ]; then |
| 202 | + echo "No harness job wrote results; the lock stays as it is." |
| 203 | + echo "changed=false" >> "$GITHUB_OUTPUT" |
| 204 | + exit 0 |
| 205 | + fi |
| 206 | + bun scripts/model-compat.ts lock --changes "$RUNNER_TEMP/lock-changes.md" "${files[@]}" |
| 207 | + if git diff --quiet -- shared/model-lock.json; then |
| 208 | + echo "changed=false" >> "$GITHUB_OUTPUT" |
| 209 | + else |
| 210 | + echo "changed=true" >> "$GITHUB_OUTPUT" |
| 211 | + { echo "## Model lock changes"; echo; cat "$RUNNER_TEMP/lock-changes.md"; } >> "$GITHUB_STEP_SUMMARY" |
| 212 | + fi |
| 213 | + # GITHUB_TOKEN's pull requests get CI runs that wait for a maintainer to approve them. A |
| 214 | + # GitHub App installation token, when one is configured, has no such limit. |
| 215 | + - id: app-token |
| 216 | + if: steps.lock.outputs.changed == 'true' && vars.OPENBOT_UPGRADE_APP_CLIENT_ID != '' |
| 217 | + uses: actions/create-github-app-token@64ab729a6294000f5aa4a4bc0b73d1f32bc45510 # v3.3.0 |
| 218 | + with: |
| 219 | + client-id: ${{ vars.OPENBOT_UPGRADE_APP_CLIENT_ID }} |
| 220 | + private-key: ${{ secrets.OPENBOT_UPGRADE_APP_PRIVATE_KEY }} |
| 221 | + permission-contents: write |
| 222 | + permission-pull-requests: write |
| 223 | + - name: Open or update the refresh pull request |
| 224 | + if: steps.lock.outputs.changed == 'true' |
| 225 | + env: |
| 226 | + GH_TOKEN: ${{ steps.app-token.outputs.token || secrets.GITHUB_TOKEN }} |
| 227 | + TOKEN_KIND: ${{ steps.app-token.outputs.token && 'app' || 'github-token' }} |
| 228 | + # Credentials for this one push, through gh, and no hook can run. |
| 229 | + GIT_CONFIG_COUNT: "3" |
| 230 | + GIT_CONFIG_KEY_0: credential.helper |
| 231 | + GIT_CONFIG_VALUE_0: "" |
| 232 | + GIT_CONFIG_KEY_1: credential.helper |
| 233 | + GIT_CONFIG_VALUE_1: "!gh auth git-credential" |
| 234 | + GIT_CONFIG_KEY_2: core.hooksPath |
| 235 | + GIT_CONFIG_VALUE_2: /dev/null |
| 236 | + run: | |
| 237 | + set -euo pipefail |
| 238 | + git switch -C "$BRANCH" |
| 239 | + git -c user.name="github-actions[bot]" \ |
| 240 | + -c user.email="41898282+github-actions[bot]@users.noreply.github.com" \ |
| 241 | + commit -m "Refresh the model lock from the daily model check" -- shared/model-lock.json |
| 242 | + git push --force "https://github.com/$GITHUB_REPOSITORY.git" "HEAD:refs/heads/$BRANCH" |
| 243 | + body="$RUNNER_TEMP/lock-pr.md" |
| 244 | + { |
| 245 | + echo "The daily model check found models that started or stopped answering through the clients the harnesses ship. This refreshes \`shared/model-lock.json\`, the models the picker may offer for each harness." |
| 246 | + echo |
| 247 | + cat "$RUNNER_TEMP/lock-changes.md" |
| 248 | + echo |
| 249 | + echo "From $RUN_URL. Every run that finds a change updates this pull request. See docs/releasing.md, \"Model lock\"." |
| 250 | + if [ "$TOKEN_KIND" != app ]; then |
| 251 | + echo |
| 252 | + echo "**CI must be started by hand.** This pull request was opened with \`GITHUB_TOKEN\`, so its CI runs wait for a maintainer to approve them." |
| 253 | + fi |
| 254 | + } > "$body" |
| 255 | + number=$(gh pr list --repo "$GITHUB_REPOSITORY" --head "$BRANCH" --state open --json number --jq ".[0].number // empty") |
| 256 | + if [ -n "$number" ]; then |
| 257 | + gh pr edit "$number" --repo "$GITHUB_REPOSITORY" --body-file "$body" |
| 258 | + else |
| 259 | + gh pr create --repo "$GITHUB_REPOSITORY" --base main --head "$BRANCH" \ |
| 260 | + --title "Refresh the model lock" --body-file "$body" |
| 261 | + fi |
0 commit comments