From 5a346d610c2c9f4d52e17e4a8aaf445b1a0faabc Mon Sep 17 00:00:00 2001 From: benw Date: Wed, 17 Jun 2026 11:08:55 +0300 Subject: [PATCH 1/5] PLU-276: Collapse /evals:eval command into the eval skill MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Skills are now slash-invocable (commands and skills were unified in Claude Code), so the separate commands/eval.md duplicated ~95% of the evaluator skill's prose and risked silent drift. Rename skills/evaluator/ -> skills/eval/ (name: eval) so /evals:eval is served directly by the skill — no command file, no shim, no indirection hop, single source of truth. Delete commands/eval.md and update the test comment to point at the new path. README /evals:eval references stay valid (same name, now from the skill). Co-Authored-By: Claude Opus 4.8 (1M context) --- .gitignore | 1 + commands/eval.md | 86 ----------------------------- skills/{evaluator => eval}/SKILL.md | 2 +- tests/test_tools.py | 4 +- 4 files changed, 4 insertions(+), 89 deletions(-) delete mode 100644 commands/eval.md rename skills/{evaluator => eval}/SKILL.md (99%) diff --git a/.gitignore b/.gitignore index 98453e2..3a21900 100644 --- a/.gitignore +++ b/.gitignore @@ -3,3 +3,4 @@ __pycache__/ *.log .DS_Store scripts/ +.omc/ diff --git a/commands/eval.md b/commands/eval.md deleted file mode 100644 index 4fe49b8..0000000 --- a/commands/eval.md +++ /dev/null @@ -1,86 +0,0 @@ ---- -description: "Create a fine-tuned LLM-as-a-judge evaluator on the Plurai platform" -argument-hint: "[task description or --data file.csv]" -allowed-tools: ["Bash", "Read", "Edit", "Write", "Glob", "Grep", "Agent"] ---- - -**Auth.** Two distinct cases — handle them differently. - -**Case A — `Plurai API key not set`** (no key configured yet, e.g. fresh install or after `auth logout`): - -1. Ask the user (in chat) to paste their Plurai API key. If they don't have one, point them to https://app.plurai.ai/settings?tab=api-keys → **Create new key**. -2. Run `uv run --project ${CLAUDE_PLUGIN_ROOT} python -m evals_mcp auth login --key ` with the pasted key. -3. On success (`Saved API key to .`), retry the failed tool call. On failure, relay the stderr message to the user. - -**Case B — `Plurai API key invalid or expired`** (server returned 401: the on-disk key was rejected): - -The key currently on disk is bad. If you have a key from earlier in this conversation, that IS the rejected key — do NOT call `auth login` with it. You MUST ask the user (in chat, this turn) to paste a NEW, freshly-generated key from https://app.plurai.ai/settings?tab=api-keys → **Create new key**. Only after the user supplies a new key in this turn, run `uv run --project ${CLAUDE_PLUGIN_ROOT} python -m evals_mcp auth login --key ` and retry. Never silently auto-renew with a remembered key — the user must see the prompt and supply a fresh key. - -**Case C — other errors (5xx, transport).** Any `{"error": ...}` envelope that's NOT an auth message will carry a `recovery_hint` field telling you exactly which tool to retry. Surface the error text to the user, ask whether to retry, and on yes call **the same tool that failed** with **the same arguments**. Do NOT escalate to `start_evaluator` unless that's the tool that failed — restarting from `start_evaluator` creates a new thread and re-fires the entire flow. If the envelope includes a `thread_id`, that thread is still alive on the platform and the retry will resume it. - -Call `search_evaluators` first as an optimization to check whether the user already has an evaluator in their Plurai workspace that fits this task. **If the list is empty, say nothing about it and proceed silently to create a new one** — a fresh user has no evaluators yet and should not be told something is missing. If one or more existing evaluators match, surface the full list to the user and use `ask_user` to ask whether to reuse one or create a new one. If reusing, jump straight to step 6 (integration snippet): the endpoint URL is the `endpoint_url` from the search result for the chosen evaluator, and the API key comes from `get_api_key` (the same Plurai API key the user configured at session start — no new key is created). - -If creating new, call `start_evaluator`. - -For `task_description`: 1-2 short sentences. Include the core task and desired label names **only if the user already mentioned them** — do NOT pre-ask the user for labels, scope, or criteria before this call (the agent's refinement round covers all of those, and pre-asking causes the user to see the labels question twice). Do NOT include examples, detailed criteria, or long explanations. - -**Platform constraint — the task definition is frozen.** The `task_description` passed to `start_evaluator` is permanent for that evaluator. Subsequent `send_message` calls only refine the *generated samples* (add/remove/edit examples), never the task itself (judging criteria, scope). Labels CAN still be changed within the same task. If at any point — including after the user sees the samples — they want to change the underlying task, you MUST tell them the task can't be edited, confirm they want to restart, then call `start_evaluator` again with a revised `task_description`. Do NOT try to amend the task via `send_message`; it will silently leave the underlying task wrong while mutating samples. - -**Important — input template**: If the evaluation involves multiple fields (e.g. context + response for grounding, or a conversation), you MUST specify the input template explicitly in the task description. The evaluator receives a SINGLE text input, so all fields must be combined into one message using a clear template. Examples: - -- Grounding: "Input format: '## Context:\n{context}\n\n## Response:\n{response}'" -- Conversation: "Input format: 'User: {msg}\nAI: {msg}\nUser: {msg}\nAI: {msg}'" -- QA: "Input format: '## Question:\n{question}\n\n## Answer:\n{answer}'" - -**Data path — if the user supplied a labeled data file** (e.g. `/eval --data path/to/file.csv` or they pasted a path): after `start_evaluator`, read the file, parse it into `{sample, label, reasoning?}` records, then call `upload_data` with the `example_set_id` from the start_evaluator response. Do NOT synthesize records. Continue with `ask_user` using the refinement questions from the start_evaluator `agent_response`. - -**Delegate or ask — applied per decision.** - -At every decision point in the flow (refinement questions, model choice, integration language), the rule is the same: if the user's intent already speaks to *this* decision, act on it; otherwise ask. - -- **Refinement questions** cover the OVERALL task (labels, scope, judging criteria) — never per-example. If the user handed you a spec source for them, answer yourself via `send_message` from that source; otherwise route them to the user via `ask_user`, rephrased as options. -- **Other decisions** (model choice, integration language) are independent. Delegation on one does not imply delegation on another — e.g. a user who pointed at a codebase but said nothing about model still gets the SLM/LLM ask. -- **Repo-state inference for the language ask only.** Treat a root-level manifest as an "intent" signal — if the working directory unambiguously identifies a language, act on it without asking. Model choice stays conversational-intent-only; it fires a server-side run. - -When ambiguous, ask. User-facing questions ALWAYS go through `ask_user` — never invent a plain-text Q&A turn. - -**Recommend a default when one option is clearly best.** Whenever you call `ask_user` and one of the options is the defensible default (the one most users in this situation should pick), append `(Recommended)` to that option's label and list it first. Apply this to refinement questions (labels, scope, judging criteria), the reuse-vs-create-new ask, and the Language ask **only when the ask is shown** (step 6 may skip it via inference). If the options are genuinely equivalent for this user (no clear winner), do not force one — list them in a natural order without a marker. - -**Skip handling (refinement only).** Treat the host's `AskUserQuestion` as a *partial* form: the user may submit having answered only some tabs, or escape/decline the whole thing. -- For any refinement question that **has a user answer in the response**: use that answer when composing `send_message`. -- For any refinement question with **no user answer** (omitted from `answers`, or the whole response is `"User declined to answer questions"`/interrupted): fill it yourself with the orchestrator's own pick using sensible defaults. - -Then send a single `send_message` composing the merged answers and continue the normal post-refinement flow. Do NOT retry the ask, do NOT re-prompt the user for the missing questions, do NOT surface the interruption text verbatim, do NOT stall. - -**Surfacing progress to the user.** After every `send_message`, show the user the `agent_response` text verbatim so they see what the platform is doing (e.g. "I've generated 16 synthetic examples..."). Whenever a response includes `url`, you MUST display it to the user as a clickable markdown link in that turn — never silently move on. - -After the refinement round (whether the user answered via `ask_user` or you self-answered via `send_message`): -1. If the user answered, compose their answers and call `send_message`. If you self-answered, you've already done this — proceed. -2. **Surface the UI experience link, then settle the model.** Once the response includes `url`, share it as a clickable markdown link whose link text is exactly `UI experience` — do NOT substitute any other label such as "Data Canvas", the evaluator name, or the thread title. Then tell the user they can review/edit the generated data and track progress in the UI experience. - - Apply the per-decision rule: if the user already specified a model preference upfront, use it (jump to step 3 with `Optimize [LLM]` or `Optimize [SLM]`). Otherwise, call `ask_user` with header `"Model Choice"` and question `"Which model would you like to generate?"`. Default options: - - label `"SLM - best for production scale (Recommended)"`, description `"Our fine-tuned small-language model with low inference cost, realtime latency, and high accuracy. Pro plan only. ~20 min."` - - label `"Optimized LLM - for dev iterations"`, description `"Our calibration on a large language model, best for local checks and quick validations. ~2 min."` - - **Whenever the SLM option is presented to the user (either as the recommended choice above or in the upgrade-gated variant below), you MUST first emit a one-line plain-text message containing the clickable markdown link `[Learn more about intent-calibrated SLMs](https://intercom.help/plurai/en/articles/14113048-intent-calibrated-slms-to-the-rescue)` BEFORE calling `ask_user`.** Do NOT embed the URL inside the `question` or any option `description` — those fields render as plain text in the picker and the link will not be clickable. Keep it in the conversational text that precedes the ask. - - **Gate on `slm_allowed` (from the response).** If `false`, swap the option list (do NOT drop to one option — `ask_user` rejects single-option questions). Use these two options, and prepend the upgrade prompt verbatim to the question: "SLM vibe-training requires a paid Plurai plan. Upgrade at https://app.plurai.ai/settings?tab=subscription-billing to unlock the fine-tuned small-language model." (Plain URL inside the question — render the clickable `[Plurai Settings](...)` and `[Learn more about intent-calibrated SLMs](...)` markdown links in the text turn that precedes the ask.) - - label `"Continue with Optimized LLM"`, description `"Our calibration on a large language model, best for local checks and quick validations. ~2 min."` - - label `"Wait — I'll upgrade my plan first"`, description `"Stop here for now. After upgrading at Plurai Settings, resume this flow and SLM will be available."` - - If the user picks "Wait", do NOT call `send_message` — acknowledge and end the turn. Otherwise proceed to step 3 with `Optimize [LLM]`. - - Do not add an explicit Other option, and do not add any extra confirmation question before this ask. - - **On Other / decline / ambiguous answer.** Don't fire optimization without an explicit pick — neither `Optimize [LLM]` nor `Optimize [SLM]`. If free text clearly maps to one of them, use that. On confusion, answer inline and re-ask once. Otherwise restate the two options in plain text and end the turn. -3. Call `send_message` with EXACTLY `Optimize [LLM]` or `Optimize [SLM]` based on user's choice. These are hardcoded strings — do not modify them. Only one call needed. Tell the user the run has started — for SLM say "vibe-training has started (~20 min)", for LLM say "optimization has started (~2 min)". The response carries `classifier_id` — record it for the wake-up polls below. The integration endpoint URL isn't available yet — it surfaces with the results in step 6. **If the response is the "still running in the background" envelope** (common under batch load), the run is in progress and the server enforces ONE run per thread — schedule a `ScheduleWakeup(delaySeconds=120)` and call `send_message(thread_id, '')` again on wake. The server is idempotent per thread: that resume re-awaits the existing run, it never restarts it. Never resend the message until the wake-up fires, and never change the message. -4. **Schedule a wake-up and end the turn.** Use `ScheduleWakeup` with `delaySeconds = 120` for LLM or `delaySeconds = 1200` for SLM. The MCP responses carry the wait/poll contract themselves; follow the `instructions` field they return. -5. **On wake-up:** call `get_results(classifier_id)`. Pass `classifier_id` every time — the MCP server is stateless across subprocess restarts. If still pending, the response's `instructions` field tells you the re-poll interval (60s LLM, 300s SLM); schedule another wake-up and end the turn. Repeat until results land. -6. **When results land:** show baseline vs trained metrics (accuracy, precision, recall) and the improvement delta. Frame the trained row as "vibe-trained" if the user picked SLM and as "optimized" if they picked LLM. Then settle the integration language by repo inference — `Glob` the working directory root only (no recursion; depth-mixed manifests are the polyglot case): - - **Python**: `pyproject.toml`, `setup.py`, `setup.cfg`, `requirements.txt`, `Pipfile`, `poetry.lock`, `uv.lock`. - - **JavaScript/TypeScript**: `package.json`, `tsconfig.json`, `pnpm-lock.yaml`, `yarn.lock`, `package-lock.json`. - - **Ambiguous** if no manifest, both groups present, or extra working directories disagree → fall through to the ask. - - **On unambiguous inference: emit the snippet directly — no `ask_user`, no picker, no Recommended marker.** Prefix it with one plain-text line: `Detected a Python project — emitting Python. Reply with JavaScript/TypeScript or cURL if you'd like a different language.` (substitute the detected language). On ambiguous / missing signal only, call `ask_user` with header `"Language"`, question `"Which language should I emit the integration snippet in?"`, options Python, JavaScript/TypeScript, cURL. - - Once the language is settled, call `get_api_key` to retrieve the user's existing Plurai API key (configured at session start — local read, **not** a new key creation) and embed it in the integration snippet. The integration code MUST format the input using the same template specified in the task description. For example, if the evaluator uses "## Context:\n{context}\n\n## Response:\n{response}", the code must combine the fields into a single string following that exact template before sending to the endpoint. The evaluator only accepts a single message — never multiple messages. The language ask (when shown) is the only post-results user question — do NOT ask whether to create a key or whether to integrate. diff --git a/skills/evaluator/SKILL.md b/skills/eval/SKILL.md similarity index 99% rename from skills/evaluator/SKILL.md rename to skills/eval/SKILL.md index 8699d66..35d0934 100644 --- a/skills/evaluator/SKILL.md +++ b/skills/eval/SKILL.md @@ -1,5 +1,5 @@ --- -name: evaluator +name: eval description: > Use when the user wants to build an LLM-as-a-judge, evaluator, classifier, or guardrail for their AI application. Triggers on: evaluating AI outputs, judging response quality, classifying text, building guardrails, diff --git a/tests/test_tools.py b/tests/test_tools.py index c8c384b..5cf89bb 100644 --- a/tests/test_tools.py +++ b/tests/test_tools.py @@ -1161,8 +1161,8 @@ async def test_ask_user_returns_ask_user_question_payload(ctx: Any) -> None: # Decline/escape of the host AskUserQuestion must not stall the flow: every # payload reminds the model to treat the "User declined to answer questions" # response (or any interruption) as a skip and fall back to the per-decision - # default in the skill/command. See the "Skip handling" section in - # skills/evaluator/SKILL.md and commands/eval.md. + # default in the skill. See the "Skip handling" section in + # skills/eval/SKILL.md. assert "declined" in out["instructions"].lower() assert "skip" in out["instructions"].lower() assert len(out["askUserQuestions"]) == 1 From 8d22535b883530c29907f8d3938865b92a7f8da2 Mon Sep 17 00:00:00 2001 From: benw Date: Wed, 17 Jun 2026 11:13:13 +0300 Subject: [PATCH 2/5] PLU-276: Sync marketplace version to 0.4.0; drop commands/ from docs - marketplace.json plugins[].version 0.3.0 -> 0.4.0 to match plugin.json - CLAUDE.md: remove commands/ from repo layout and intro (the plugin ships skills only); fold slash-command guidance into the Skills section (skills are slash-invocable and auto-activated) - Release checklist now calls out keeping both version fields in sync Co-Authored-By: Claude Opus 4.8 (1M context) --- .claude-plugin/marketplace.json | 2 +- CLAUDE.md | 17 ++++++++--------- 2 files changed, 9 insertions(+), 10 deletions(-) diff --git a/.claude-plugin/marketplace.json b/.claude-plugin/marketplace.json index 379ea68..9c14546 100644 --- a/.claude-plugin/marketplace.json +++ b/.claude-plugin/marketplace.json @@ -9,7 +9,7 @@ "name": "evals", "source": "./", "description": "Create fine-tuned LLM-as-a-judge evaluators on the Plurai platform", - "version": "0.3.0" + "version": "0.4.0" } ] } diff --git a/CLAUDE.md b/CLAUDE.md index f5cd1e1..d4b1d16 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -1,10 +1,9 @@ # Evals — Claude Code Plugin -A Claude Code plugin (`evals` in the `plurai-plugins` marketplace) that ships for creating evaluator agents on the Plurai platform. +A Claude Code plugin (`evals` in the `plurai-plugins` marketplace) that ships skills and an MCP server for creating evaluator agents on the Plurai platform. ## Repo layout - `.claude-plugin/plugin.json` — manifest (bump `version` for releases) -- `commands/` — slash commands shipped to users (one .md per command) - `skills//SKILL.md` — skills shipped to users (frontmatter required) - `src//` — Python MCP server (FastMCP, registered via `.mcp.json`) - `evals/` — MCP tool evaluations (realistic multi-step questions) @@ -41,18 +40,18 @@ A Claude Code plugin (`evals` in the `plurai-plugins` marketplace) that ships :foo`) -- Frontmatter: `description`, `argument-hint` -- Use gerund forms in skill names (`git-pushing`, not `git-push`) - ### Skills (in `skills//SKILL.md`) -- Frontmatter `description` is critical — it's what triggers activation +- This plugin ships skills only — no `commands/` folder. Skills are both + slash-invocable (`skills//SKILL.md` → `/:`) and + auto-activated, so they fully replace standalone commands. +- Frontmatter `description` is critical — it's what triggers auto-activation; + add `argument-hint` for skills meant to be invoked with `/: ` - Process steps go in `SKILL.md`; reference material in `skills//reference/*.md` - Don't dump background context into `SKILL.md` — keep it procedural ## Release checklist -1. Bump `version` in `.claude-plugin/plugin.json` +1. Bump `version` in `.claude-plugin/plugin.json` **and** keep the matching + `plugins[].version` in `.claude-plugin/marketplace.json` in sync 2. Run `uv run pytest && uv run python evals/run.py` 3. Update `README.md` and `CHANGELOG.md` 4. Tag and push; marketplace consumers pull from the tag From 60a6021b972a7091374b60534b86eac034fef529 Mon Sep 17 00:00:00 2001 From: benw Date: Wed, 17 Jun 2026 11:16:39 +0300 Subject: [PATCH 3/5] PLU-276: Stop tracking dev-only artifacts in the public repo CLAUDE.md, .claude/, and .vscode/ are local development files (internal conventions, editor and plugin settings) that should not ship in the public marketplace repo. Untrack them and add to .gitignore; they remain on disk for local development. Co-Authored-By: Claude Opus 4.8 (1M context) --- .claude/settings.json | 5 ---- .gitignore | 3 +++ .vscode/settings.json | 3 --- CLAUDE.md | 62 ------------------------------------------- 4 files changed, 3 insertions(+), 70 deletions(-) delete mode 100644 .claude/settings.json delete mode 100644 .vscode/settings.json delete mode 100644 CLAUDE.md diff --git a/.claude/settings.json b/.claude/settings.json deleted file mode 100644 index 845fe3a..0000000 --- a/.claude/settings.json +++ /dev/null @@ -1,5 +0,0 @@ -{ - "enabledPlugins": { - "mcp-server-dev@claude-plugins-official": true - } -} diff --git a/.gitignore b/.gitignore index 3a21900..d56cd66 100644 --- a/.gitignore +++ b/.gitignore @@ -4,3 +4,6 @@ __pycache__/ .DS_Store scripts/ .omc/ +CLAUDE.md +.claude/ +.vscode/ diff --git a/.vscode/settings.json b/.vscode/settings.json deleted file mode 100644 index 86c5692..0000000 --- a/.vscode/settings.json +++ /dev/null @@ -1,3 +0,0 @@ -{ - "python.defaultInterpreterPath": "${workspaceFolder}/.venv/bin/python", -} diff --git a/CLAUDE.md b/CLAUDE.md deleted file mode 100644 index d4b1d16..0000000 --- a/CLAUDE.md +++ /dev/null @@ -1,62 +0,0 @@ -# Evals — Claude Code Plugin - -A Claude Code plugin (`evals` in the `plurai-plugins` marketplace) that ships skills and an MCP server for creating evaluator agents on the Plurai platform. - -## Repo layout -- `.claude-plugin/plugin.json` — manifest (bump `version` for releases) -- `skills//SKILL.md` — skills shipped to users (frontmatter required) -- `src//` — Python MCP server (FastMCP, registered via `.mcp.json`) -- `evals/` — MCP tool evaluations (realistic multi-step questions) -- `tests/` — pytest unit + integration tests - -## Authoring guidance (on-demand skills) -- For MCP server design (tool naming, annotations, transports, pagination, - Pydantic patterns, FastMCP usage), use the `mcp-builder` skill. -- For plugin packaging (plugin.json schema, command frontmatter, skill - frontmatter, hooks, marketplace.json), use the `plugin-development` skill - if installed; otherwise consult docs at code.claude.com/docs/en/plugins. - -## Stack -- Python 3.11+, `uv` for env/deps -- FastMCP (MCP Python SDK), Pydantic v2 - (`ConfigDict(extra='forbid', str_strip_whitespace=True)`) -- `httpx.AsyncClient` for outbound HTTP -- `ruff` + `ruff format`, `pyright` (strict mode) -- `pytest` + `pytest-asyncio` - -## Commands -- Install dev deps: `uv sync` -- Lint: `uv run ruff check . && uv run ruff format --check .` -- Type-check: `uv run pyright` -- Tests: `uv run pytest` -- Eval suite: `uv run python evals/run.py` -- Local plugin test: `claude --plugin-dir .` then `/reload-plugins` after edits - -## House conventions -### MCP tools (in `src//tools/`) -- All tool names prefixed `_` (e.g. `acme_search_users`) -- Every `@mcp.tool` sets `readOnlyHint`, `destructiveHint`, `idempotentHint`, `openWorldHint` -- Pydantic input models with strict `ConfigDict` -- Default response format Markdown; accept `response_format: "json" | "markdown"` for data-heavy tools -- Always paginate list endpoints - -### Skills (in `skills//SKILL.md`) -- This plugin ships skills only — no `commands/` folder. Skills are both - slash-invocable (`skills//SKILL.md` → `/:`) and - auto-activated, so they fully replace standalone commands. -- Frontmatter `description` is critical — it's what triggers auto-activation; - add `argument-hint` for skills meant to be invoked with `/: ` -- Process steps go in `SKILL.md`; reference material in `skills//reference/*.md` -- Don't dump background context into `SKILL.md` — keep it procedural - -## Release checklist -1. Bump `version` in `.claude-plugin/plugin.json` **and** keep the matching - `plugins[].version` in `.claude-plugin/marketplace.json` in sync -2. Run `uv run pytest && uv run python evals/run.py` -3. Update `README.md` and `CHANGELOG.md` -4. Tag and push; marketplace consumers pull from the tag - -## Don't -- Don't put `commands/`, `skills/`, `agents/`, or `hooks/` inside `.claude-plugin/` — only `plugin.json` lives there -- Don't run the MCP server with `python -m ` outside stdio context — it hangs -- Don't ship CLAUDE.md content to users — it's not loaded from installed plugins \ No newline at end of file From 8161d231339257e9a5d2831e7be8e3057f349b36 Mon Sep 17 00:00:00 2001 From: benw Date: Wed, 17 Jun 2026 11:18:45 +0300 Subject: [PATCH 4/5] PLU-276: Remove Linear-issue PR check workflow from public repo The check-linear-issue workflow exposed the internal Linear/PLU process and would fail CI for external contributors who have no Linear access. Remove it from the public marketplace repo (recoverable from git history if a private/internal copy is wanted). Co-Authored-By: Claude Opus 4.8 (1M context) --- .github/workflows/check-linear-issue.yml | 36 ------------------------ 1 file changed, 36 deletions(-) delete mode 100644 .github/workflows/check-linear-issue.yml diff --git a/.github/workflows/check-linear-issue.yml b/.github/workflows/check-linear-issue.yml deleted file mode 100644 index e5f3f87..0000000 --- a/.github/workflows/check-linear-issue.yml +++ /dev/null @@ -1,36 +0,0 @@ -name: Check for Linear Issue Link - -on: - pull_request: - types: [opened, edited, reopened, synchronize] - -jobs: - validate-linear-issue: - runs-on: ubuntu-latest - - steps: - - name: Check PR for Linear issue reference - run: | - echo "🔍 Checking PR for Linear issue reference..." - - TITLE=$(jq -r .pull_request.title "$GITHUB_EVENT_PATH") - BODY=$(jq -r .pull_request.body "$GITHUB_EVENT_PATH") - - # Check for Linear issue ID patterns in strict format (PLU-123: Title) - LINEAR_ID_TITLE=$(echo "$TITLE" | grep -o '^[A-Z]\{2,\}-[0-9]\+:' | sed 's/:$//' | head -n1) - LINEAR_ID_BODY=$(echo "$BODY" | grep -o '^[A-Z]\{2,\}-[0-9]\+:' | sed 's/:$//' | head -n1) - - # Check for Linear app URL format in body - LINEAR_URL=$(echo "$BODY" | grep -o 'https://linear\.app/[^/]*/issue/[A-Z]\{2,\}-[0-9]\+' | head -n1) - - if [ -n "$LINEAR_ID_TITLE" ]; then - echo "✅ Linear issue ID found in title: $LINEAR_ID_TITLE" - elif [ -n "$LINEAR_ID_BODY" ]; then - echo "✅ Linear issue ID found in body: $LINEAR_ID_BODY" - elif [ -n "$LINEAR_URL" ]; then - echo "✅ Linear issue URL found in body: $LINEAR_URL" - else - echo "❌ ERROR: No Linear issue reference found in PR title or description." - echo "Please include a Linear issue ID in strict format (e.g., 'PLU-123: Your title') in the title, or a Linear issue URL in the body." - exit 1 - fi From b35775063899e3f986ad801990bfa6ec2a118e69 Mon Sep 17 00:00:00 2001 From: benw Date: Wed, 17 Jun 2026 11:25:24 +0300 Subject: [PATCH 5/5] PLU-276: Fix stale "skill/command" wording after command removal MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The eval command was collapsed into the skill, so the orchestrator instruction strings and test comments that read "eval skill / command" are now inaccurate — only the skill exists. Drop the "/ command" half in 6 spots (classifiers.py, evaluator.py x3, test_tools.py x2). Behavior-neutral: string/comment wording only. ruff + pyright clean, 48 tests pass. Co-Authored-By: Claude Opus 4.8 (1M context) --- src/evals_mcp/tools/classifiers.py | 2 +- src/evals_mcp/tools/evaluator.py | 6 +++--- tests/test_tools.py | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/evals_mcp/tools/classifiers.py b/src/evals_mcp/tools/classifiers.py index c778118..c7d4977 100644 --- a/src/evals_mcp/tools/classifiers.py +++ b/src/evals_mcp/tools/classifiers.py @@ -343,7 +343,7 @@ async def _get_results(args: GetResultsArgs, ctx: Context[Any, Any, Any]) -> Any else ( "Results landed. Surface baseline vs optimized metrics to " "the user, then proceed to the integration-language step " - "per the eval skill/command." + "per the eval skill." ) ), } diff --git a/src/evals_mcp/tools/evaluator.py b/src/evals_mcp/tools/evaluator.py index 5a4b7e7..9b9c673 100644 --- a/src/evals_mcp/tools/evaluator.py +++ b/src/evals_mcp/tools/evaluator.py @@ -289,7 +289,7 @@ async def _handle_optimize( "literal). The user must explicitly pick a model via the " "Model Choice ask_user — never invent a payload from " "an 'Other' / declined answer. Re-ask via ask_user " - "with the two model options per the eval skill / command docs." + "with the two model options per the eval skill docs." ) } if normalized == "optimize [slm]" and not state.slm_allowed: @@ -527,7 +527,7 @@ async def _ask_user(args: AskUserArgs, ctx: Context[Any, Any, Any]) -> dict[str, " IF this is the model-choice question: after the user picks, call " "send_message ONCE with EXACTLY 'Optimize [LLM]' or " "'Optimize [SLM]' (hardcoded — do not modify). On Other / decline / " - "ambiguity, follow the Other policy in the eval skill / command — " + "ambiguity, follow the Other policy in the eval skill — " "never fire optimization without an explicit user pick." ) return { @@ -538,7 +538,7 @@ async def _ask_user(args: AskUserArgs, ctx: Context[Any, Any, Any]) -> dict[str, + extra + " If AskUserQuestion returns 'User declined to answer questions' (or is " "otherwise cancelled/interrupted/escaped), treat it as a SKIP and follow " - "the default action documented in your skill or command for this specific " + "the default action documented in your skill for this specific " "decision point. Do NOT retry the same ask, do NOT re-call ask_user " "with the same questions, do NOT surface a 'tool interrupted' or 'user " "declined' message verbatim to the user, and do NOT stall the flow." diff --git a/tests/test_tools.py b/tests/test_tools.py index 5cf89bb..b88d740 100644 --- a/tests/test_tools.py +++ b/tests/test_tools.py @@ -1260,7 +1260,7 @@ async def test_send_message_surfaces_url_when_commit_id_present( instructions = out["instructions"] # Must direct the orchestrator to surface the URL and ask the model # choice in the same turn — no separate review-confirmation gate. The - # gate policy itself lives in the eval skill / command docs, not here. + # gate policy itself lives in the eval skill docs, not here. assert "review/edit" in instructions assert "ask_user" in instructions assert "UI experience" in instructions @@ -1308,7 +1308,7 @@ async def test_send_message_post_commit_blocks_slm_when_not_entitled( # plus a "drop SLM when slm_allowed=false" directive — the orchestrator # branches on the boolean, not on per-state copy. # The structured signal carries the gate; the rule lives in the eval - # skill / command docs. The tool response just publishes slm_allowed + # skill docs. The tool response just publishes slm_allowed # and arms the optimize backstop via state. assert out["slm_allowed"] is False assert state.slm_allowed is False