Skip to content

Commit 3c4d673

Browse files
authored
Offer a model only when the model check has proven it on the Bot's engine (#1779)
The model picker offers a model only when the model check has proven it answers on the Bot's engine. shared/model-lock.json records, per engine and provider, which models are verified, and which are refused and why. It is regenerated from a CI run of every listed model through the client each engine ships. A conversation saved on a model that is no longer offered shows a notice and falls back to the engine's default. Guards, each failing before and passing after: - the lock must match the engine pins and package-lock.json, so a client upgrade without a re-check fails CI; - the default engine must have at least one verified model. A model the provider has retired ("no longer available to new users") counts as refused, and a failing model is asked once more before it counts. Gemma 4 is refused wherever its reasoning leaks into the answer. Browser check on this head, Chromium and WebKit, light and dark: 46 of 46. The Claude Agent SDK engine offers Opus 5.5, Fable 5.2, Sonnet 5.5 and Haiku 5.5. A Google Bot hides the gemini-omni, Gemma 4 and gemini-2.5-flash-lite models.
1 parent 9d669e8 commit 3c4d673

22 files changed

Lines changed: 7025 additions & 2 deletions

‎.github/workflows/model-compat.yml‎

Lines changed: 261 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,261 @@
1+
name: Model compatibility
2+
3+
# Does every model the picker offers still answer through the clients OpenBot ships? The picker
4+
# lists each provider's models live, but every harness image carries its model client at a fixed
5+
# version (the Claude Agent SDK harness bundles a Claude Code CLI, the LangGraph harness sends the
6+
# ChatGPT plan's backend a fixed Codex client version, the others carry the provider SDKs they were
7+
# built with), and a provider can refuse a new model to an older client. scripts/model-compat.ts
8+
# starts each harness image, one job per harness, and sends it one request per listed model on
9+
# every provider that has a key here. See docs/releasing.md, "Model compatibility".
10+
#
11+
# Keys are repository secrets, one per provider. A provider without one is reported as skipped,
12+
# not failed. On any failure the run fails and opens (or updates) one issue titled "Model
13+
# compatibility check failing"; a passing run closes it.
14+
#
15+
# What answered on the `source` image is the model lock (shared/model-lock.json), the models the
16+
# picker may offer for each harness. When a model newly answers or stops answering, the run opens
17+
# or updates one pull request that refreshes the lock. See docs/releasing.md, "Model lock".
18+
#
19+
# Runs only on a schedule and by hand, on main: never on a pull request, so no fork's code ever
20+
# runs with these keys.
21+
on:
22+
schedule:
23+
- cron: "41 6 * * *"
24+
workflow_dispatch:
25+
inputs:
26+
tags:
27+
description: Harness images to check, comma-separated. `latest` is the published release people run; `source` is built from this commit, which is what the next release ships.
28+
type: string
29+
default: latest,source
30+
required: false
31+
32+
permissions: {}
33+
34+
concurrency:
35+
group: model-compat
36+
cancel-in-progress: false
37+
38+
jobs:
39+
# The published harnesses, and every engine the model lock names whether it is published yet or
40+
# not (an engine new Bots run on is locked, and checked, from before its first release).
41+
harnesses:
42+
name: List the harnesses to check
43+
if: github.repository == 'CopilotKit/OpenBot' && github.ref == 'refs/heads/main'
44+
runs-on: ubuntu-latest
45+
timeout-minutes: 5
46+
permissions:
47+
contents: read
48+
outputs:
49+
images: ${{ steps.list.outputs.images }}
50+
steps:
51+
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
52+
with:
53+
persist-credentials: false
54+
- uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2.2.0
55+
with:
56+
bun-version: 1.4.3
57+
- id: list
58+
run: echo "images=$(bun scripts/model-compat.ts harnesses)" >> "$GITHUB_OUTPUT"
59+
60+
check:
61+
name: ${{ matrix.harness }}
62+
needs: harnesses
63+
runs-on: ubuntu-latest
64+
timeout-minutes: 90
65+
permissions:
66+
contents: read
67+
strategy:
68+
fail-fast: false
69+
matrix:
70+
harness: ${{ fromJSON(needs.harnesses.outputs.images) }}
71+
steps:
72+
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
73+
with:
74+
persist-credentials: false
75+
- uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2.2.0
76+
with:
77+
bun-version: 1.4.3
78+
# Writes its results and exits 0 whatever they say: the report job below decides.
79+
- name: One request per model through this harness
80+
env:
81+
HARNESS: ${{ matrix.harness }}
82+
TAGS: ${{ inputs.tags || 'latest,source' }}
83+
ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
84+
OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
85+
GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }}
86+
XAI_API_KEY: ${{ secrets.XAI_API_KEY }}
87+
run: >-
88+
bun scripts/model-compat.ts --harness "$HARNESS" --tags "$TAGS" --exit-zero
89+
--out "$RUNNER_TEMP/model-compat-$HARNESS.json"
90+
- uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
91+
with:
92+
name: model-compat-${{ matrix.harness }}
93+
path: ${{ runner.temp }}/model-compat-${{ matrix.harness }}.json
94+
if-no-files-found: error
95+
retention-days: 14
96+
97+
report:
98+
name: Report
99+
needs: [harnesses, check]
100+
if: always() && needs.harnesses.result == 'success'
101+
runs-on: ubuntu-latest
102+
timeout-minutes: 10
103+
permissions:
104+
contents: read
105+
issues: write # open, update or close the one "Model compatibility check failing" issue
106+
env:
107+
GH_TOKEN: ${{ github.token }}
108+
ISSUE_TITLE: Model compatibility check failing
109+
RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
110+
steps:
111+
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
112+
with:
113+
persist-credentials: false
114+
- uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2.2.0
115+
with:
116+
bun-version: 1.4.3
117+
- uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2
118+
with:
119+
pattern: model-compat-*
120+
path: ${{ runner.temp }}/results
121+
merge-multiple: true
122+
- name: Summarize
123+
id: summary
124+
run: |
125+
set +e
126+
shopt -s nullglob
127+
bun scripts/model-compat.ts report --summary "$RUNNER_TEMP/summary.md" \
128+
--issue "$RUNNER_TEMP/issue-table.md" "$RUNNER_TEMP"/results/*.json
129+
code=$?
130+
echo "code=$code" >> "$GITHUB_OUTPUT"
131+
if [ -f "$RUNNER_TEMP/summary.md" ]; then cat "$RUNNER_TEMP/summary.md" >> "$GITHUB_STEP_SUMMARY"; fi
132+
exit 0
133+
- name: Open or update the issue
134+
if: steps.summary.outputs.code != '0'
135+
run: |
136+
set -euo pipefail
137+
body="$RUNNER_TEMP/issue.md"
138+
{
139+
echo "The model compatibility check failed on $(date -u +%Y-%m-%d): $RUN_URL"
140+
echo
141+
echo "A model refused by a pinned client means that client needs bumping: see docs/releasing.md, \"Model compatibility\". This issue is updated by every failing run and closed by the first passing one."
142+
echo
143+
if [ -f "$RUNNER_TEMP/issue-table.md" ]; then
144+
cat "$RUNNER_TEMP/issue-table.md"
145+
else
146+
echo "No harness job wrote results. See the run."
147+
fi
148+
} > "$body"
149+
number=$(gh issue list --repo "$GITHUB_REPOSITORY" --state open --search "\"$ISSUE_TITLE\" in:title" \
150+
--json number,title --jq "map(select(.title == env.ISSUE_TITLE)) | .[0].number // empty")
151+
if [ -n "$number" ]; then
152+
gh issue edit "$number" --repo "$GITHUB_REPOSITORY" --body-file "$body"
153+
else
154+
gh issue create --repo "$GITHUB_REPOSITORY" --title "$ISSUE_TITLE" --body-file "$body"
155+
fi
156+
- name: Close the issue when everything passes
157+
if: steps.summary.outputs.code == '0'
158+
run: |
159+
set -euo pipefail
160+
number=$(gh issue list --repo "$GITHUB_REPOSITORY" --state open --search "\"$ISSUE_TITLE\" in:title" \
161+
--json number,title --jq "map(select(.title == env.ISSUE_TITLE)) | .[0].number // empty")
162+
if [ -n "$number" ]; then
163+
gh issue close "$number" --repo "$GITHUB_REPOSITORY" --comment "Passing again: $RUN_URL"
164+
fi
165+
- name: Fail when a model failed
166+
if: steps.summary.outputs.code != '0'
167+
run: |
168+
echo "::error::Model compatibility check failing. See the summary above and the issue titled \"$ISSUE_TITLE\"."
169+
exit 1
170+
171+
lock:
172+
name: Refresh the model lock
173+
needs: [harnesses, check]
174+
if: always() && needs.harnesses.result == 'success'
175+
runs-on: ubuntu-latest
176+
timeout-minutes: 10
177+
permissions:
178+
contents: write # push the one refresh branch
179+
pull-requests: write # open or update its pull request
180+
env:
181+
BRANCH: model-lock/refresh
182+
RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
183+
steps:
184+
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
185+
with:
186+
persist-credentials: false
187+
- uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2.2.0
188+
with:
189+
bun-version: 1.4.3
190+
- uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2
191+
with:
192+
pattern: model-compat-*
193+
path: ${{ runner.temp }}/results
194+
merge-multiple: true
195+
- name: Write the lock from the source images' results
196+
id: lock
197+
run: |
198+
set -euo pipefail
199+
shopt -s nullglob
200+
files=("$RUNNER_TEMP"/results/*.json)
201+
if [ "${#files[@]}" -eq 0 ]; then
202+
echo "No harness job wrote results; the lock stays as it is."
203+
echo "changed=false" >> "$GITHUB_OUTPUT"
204+
exit 0
205+
fi
206+
bun scripts/model-compat.ts lock --changes "$RUNNER_TEMP/lock-changes.md" "${files[@]}"
207+
if git diff --quiet -- shared/model-lock.json; then
208+
echo "changed=false" >> "$GITHUB_OUTPUT"
209+
else
210+
echo "changed=true" >> "$GITHUB_OUTPUT"
211+
{ echo "## Model lock changes"; echo; cat "$RUNNER_TEMP/lock-changes.md"; } >> "$GITHUB_STEP_SUMMARY"
212+
fi
213+
# GITHUB_TOKEN's pull requests get CI runs that wait for a maintainer to approve them. A
214+
# GitHub App installation token, when one is configured, has no such limit.
215+
- id: app-token
216+
if: steps.lock.outputs.changed == 'true' && vars.OPENBOT_UPGRADE_APP_CLIENT_ID != ''
217+
uses: actions/create-github-app-token@64ab729a6294000f5aa4a4bc0b73d1f32bc45510 # v3.3.0
218+
with:
219+
client-id: ${{ vars.OPENBOT_UPGRADE_APP_CLIENT_ID }}
220+
private-key: ${{ secrets.OPENBOT_UPGRADE_APP_PRIVATE_KEY }}
221+
permission-contents: write
222+
permission-pull-requests: write
223+
- name: Open or update the refresh pull request
224+
if: steps.lock.outputs.changed == 'true'
225+
env:
226+
GH_TOKEN: ${{ steps.app-token.outputs.token || secrets.GITHUB_TOKEN }}
227+
TOKEN_KIND: ${{ steps.app-token.outputs.token && 'app' || 'github-token' }}
228+
# Credentials for this one push, through gh, and no hook can run.
229+
GIT_CONFIG_COUNT: "3"
230+
GIT_CONFIG_KEY_0: credential.helper
231+
GIT_CONFIG_VALUE_0: ""
232+
GIT_CONFIG_KEY_1: credential.helper
233+
GIT_CONFIG_VALUE_1: "!gh auth git-credential"
234+
GIT_CONFIG_KEY_2: core.hooksPath
235+
GIT_CONFIG_VALUE_2: /dev/null
236+
run: |
237+
set -euo pipefail
238+
git switch -C "$BRANCH"
239+
git -c user.name="github-actions[bot]" \
240+
-c user.email="41898282+github-actions[bot]@users.noreply.github.com" \
241+
commit -m "Refresh the model lock from the daily model check" -- shared/model-lock.json
242+
git push --force "https://github.com/$GITHUB_REPOSITORY.git" "HEAD:refs/heads/$BRANCH"
243+
body="$RUNNER_TEMP/lock-pr.md"
244+
{
245+
echo "The daily model check found models that started or stopped answering through the clients the harnesses ship. This refreshes \`shared/model-lock.json\`, the models the picker may offer for each harness."
246+
echo
247+
cat "$RUNNER_TEMP/lock-changes.md"
248+
echo
249+
echo "From $RUN_URL. Every run that finds a change updates this pull request. See docs/releasing.md, \"Model lock\"."
250+
if [ "$TOKEN_KIND" != app ]; then
251+
echo
252+
echo "**CI must be started by hand.** This pull request was opened with \`GITHUB_TOKEN\`, so its CI runs wait for a maintainer to approve them."
253+
fi
254+
} > "$body"
255+
number=$(gh pr list --repo "$GITHUB_REPOSITORY" --head "$BRANCH" --state open --json number --jq ".[0].number // empty")
256+
if [ -n "$number" ]; then
257+
gh pr edit "$number" --repo "$GITHUB_REPOSITORY" --body-file "$body"
258+
else
259+
gh pr create --repo "$GITHUB_REPOSITORY" --base main --head "$BRANCH" \
260+
--title "Refresh the model lock" --body-file "$body"
261+
fi

‎.github/workflows/model-lock.yml‎

Lines changed: 121 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,121 @@
1+
name: Model lock
2+
3+
# A pull request that changes a harness's model client (its requirements, its package.json or
4+
# lockfile, its Dockerfile, or the ChatGPT plan's Codex client version) must not break a model the
5+
# model lock (shared/model-lock.json) says that harness runs: a person set to it would get the
6+
# provider's refusal on every message. This builds the changed harnesses from the pull request,
7+
# sends each listed model one request through them (scripts/model-compat.ts), and fails when a
8+
# model the lock verifies no longer answers. See docs/releasing.md, "Model lock".
9+
#
10+
# Only for pull requests from this repository: a fork's pull request is never given the provider
11+
# keys, so it is skipped and the daily check covers it once it merges.
12+
on:
13+
pull_request:
14+
paths:
15+
- "agent-*/requirements.txt"
16+
- "agent-*/Dockerfile"
17+
- "agent-*/package.json"
18+
- "agent-*/bun.lock"
19+
- "agent-*/package-lock.json"
20+
- "agent-langgraph-agui/src/chatgpt_models.py"
21+
- "shared/model-lock.json"
22+
- "scripts/model-compat.ts"
23+
- ".github/workflows/model-lock.yml"
24+
25+
permissions: {}
26+
27+
concurrency:
28+
group: model-lock-${{ github.event.pull_request.number }}
29+
cancel-in-progress: true
30+
31+
jobs:
32+
changed:
33+
name: Harnesses whose client changed
34+
if: github.event.pull_request.head.repo.full_name == github.repository
35+
runs-on: ubuntu-latest
36+
timeout-minutes: 5
37+
permissions:
38+
contents: read
39+
outputs:
40+
harnesses: ${{ steps.list.outputs.harnesses }}
41+
steps:
42+
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
43+
with:
44+
persist-credentials: false
45+
fetch-depth: 0
46+
- uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2.2.0
47+
with:
48+
bun-version: 1.4.3
49+
- id: list
50+
env:
51+
BASE_SHA: ${{ github.event.pull_request.base.sha }}
52+
HEAD_SHA: ${{ github.event.pull_request.head.sha }}
53+
run: |
54+
set -euo pipefail
55+
changed=$(git diff --name-only "$BASE_SHA...$HEAD_SHA")
56+
# The harnesses the check runs: the published ones, and every engine the lock names,
57+
# published yet or not, that has a row in shared/harnesses.json.
58+
checked=$(bun scripts/model-compat.ts harnesses)
59+
# The check itself, or the lock, changed: every harness the lock names is checked.
60+
if grep -qxE 'shared/model-lock.json|scripts/model-compat.ts|\.github/workflows/model-lock.yml' <<<"$changed"; then
61+
harnesses=$(jq -c --argjson checked "$checked" \
62+
'[.engines | keys[] | select(. as $h | $checked | index($h))]' shared/model-lock.json)
63+
else
64+
harnesses=$(grep -oE '^agent-[a-z0-9-]+/' <<<"$changed" | tr -d / | sort -u \
65+
| jq -R . | jq -sc --argjson checked "$checked" \
66+
'map(select(. as $h | $checked | index($h)))')
67+
fi
68+
echo "harnesses=$harnesses" >> "$GITHUB_OUTPUT"
69+
echo "Harnesses to check: $harnesses"
70+
71+
check:
72+
name: ${{ matrix.harness }}
73+
needs: changed
74+
if: needs.changed.outputs.harnesses != '[]'
75+
runs-on: ubuntu-latest
76+
timeout-minutes: 60
77+
permissions:
78+
contents: read
79+
strategy:
80+
fail-fast: false
81+
matrix:
82+
harness: ${{ fromJSON(needs.changed.outputs.harnesses) }}
83+
steps:
84+
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
85+
with:
86+
persist-credentials: false
87+
- uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2.2.0
88+
with:
89+
bun-version: 1.4.3
90+
- name: One request per model through this pull request's harness
91+
env:
92+
HARNESS: ${{ matrix.harness }}
93+
ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
94+
OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
95+
GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }}
96+
XAI_API_KEY: ${{ secrets.XAI_API_KEY }}
97+
# The published image too, with the same keys in the same run: a verified model that fails
98+
# on both is the provider's or the key's today, not this change's (`brokenVerified`).
99+
run: >-
100+
bun scripts/model-compat.ts --harness "$HARNESS" --tags latest,source --exit-zero
101+
--out "$RUNNER_TEMP/model-compat.json"
102+
# The run's results, with the repository's own keys: `bun scripts/model-compat.ts lock` writes
103+
# the lock from them when a pin moved (docs/releasing.md, "Model lock").
104+
- uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
105+
with:
106+
name: model-lock-${{ matrix.harness }}
107+
path: ${{ runner.temp }}/model-compat.json
108+
if-no-files-found: error
109+
retention-days: 14
110+
- name: Every model the lock verifies still answers
111+
env:
112+
HARNESS: ${{ matrix.harness }}
113+
run: |
114+
set +e
115+
bun scripts/model-compat.ts guard --summary "$RUNNER_TEMP/guard.md" "$RUNNER_TEMP/model-compat.json"
116+
code=$?
117+
cat "$RUNNER_TEMP/guard.md" >> "$GITHUB_STEP_SUMMARY"
118+
if [ "$code" -ne 0 ]; then
119+
echo "::error::A model the lock verifies for $HARNESS no longer answers through this change's client. See the summary."
120+
fi
121+
exit "$code"

‎api/client-boundary.allow.ts‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -398,6 +398,12 @@ export const ALLOWED: Allowed[] = [
398398
rule: "third-party",
399399
why: "image registries, never OpenBot's server",
400400
},
401+
{
402+
file: "scripts/model-compat.ts",
403+
what: /^(fetch\(\)|\/health|\/api\/agents\/openbot\/generate)$/,
404+
rule: "third-party",
405+
why: "model providers' listings and the harness images the model check starts on this machine, never OpenBot's server",
406+
},
401407
// Scripts that run and stop a server on this machine.
402408
{
403409
file: "scripts/update-local.ts",

0 commit comments

Comments
 (0)