Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion agent-claude-sdk/tests/fixtures/server-instructions.json

Large diffs are not rendered by default.

8 changes: 8 additions & 0 deletions agent-claude-sdk/tests/test_openbot_instructions.py
Original file line number Diff line number Diff line change
Expand Up @@ -116,6 +116,14 @@ async def scenario():
assert "take it in the same reply instead of saying you will" in system_prompt
assert "Nobody is watching this run" in system_prompt
assert "What I could not confirm" in system_prompt
# A finished job's report: a sentence or two, one more for what to check, shown by example.
# Claude Sonnet 5.5 here wrote 166 to 252 words for a three-app job, as it did on OpenBot's own
# runner, where GPT-5.6 Terra took 45 to 84.
assert "Add a sentence only for what the person must check" in system_prompt
assert "a whole report is: 'I saved this week's 4 invoices" in system_prompt
assert "then anything you could not confirm, a sentence each" not in system_prompt
# The claims rule is kept, and an inference is still called unconfirmed.
assert "call anything you could not check unconfirmed; an inference, however likely, is still unconfirmed." in system_prompt
# Told OpenBot asks before what it gives the Bot, so the Bot does not ask a second time.
assert "When you use the apps and tools OpenBot gives you, OpenBot asks" in system_prompt
# The server's instructions are the whole prompt, first: Claude Code's own is not added.
Expand Down
57 changes: 53 additions & 4 deletions shared/bot-prompt.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -359,15 +359,16 @@ describe("how a Bot asks, and how an unattended run ends", () => {
// tools could do all of it.
expect(said).not.toContain("smartlead");
expect(said).not.toContain("partly");
// A finished job: the result and its gaps, not a tour of what was built or an offer of more.
// A finished job: the result and what to check, not a tour of what was built or an offer of more.
expect(said).toContain(
"then anything you could not confirm, a sentence each",
"add a sentence only for what the person must check",
);
expect(said).toContain(
"do not describe what you made when the link shows it",
"do not list again what the link, a sheet, a message or a component already holds",
);
expect(said).toContain("or offer more work");
expect(said).toContain("recap your steps, repeat a summary");
expect(said).toContain("recap your steps");
expect(said).toContain("repeat a summary you already gave");
expect(said).toContain("do not announce or narrate your steps");
// No preamble either: on GPT-6.1 a "one short line before a long job" became a message of its own.
expect(said).not.toContain("one short line");
Expand All @@ -378,6 +379,54 @@ describe("how a Bot asks, and how an unattended run ends", () => {
);
});

test("a finished job is reported in a few sentences, with one for what to check, shown by example", () => {
// Claude Sonnet 5.5 reported a three-app job in 166 to 252 words, on OpenBot's runner and on the
// Claude Agent SDK alike: the rows again, "I read the sheet back", what it left out and why, and
// each inference as a bullet, where GPT-5.6 Terra took 45 to 84.
const said = flat(REPLY_GUIDANCE);
expect(said).not.toContain(
"then anything you could not confirm, a sentence each",
);
expect(said).toContain("say what you checked, read back or left out");
expect(said).toContain(
"a whole report is: 'i saved this week's 4 invoices to [invoices, week 46](link) and posted the list in #finance.",
);
// The example keeps the claims rule: an inference is said to be one.
expect(said).toContain(
"two vendor names come from their email addresses, so check those.'",
);
// On apps no test asks about.
for (const app of ["gmail", "sheets", "slack", "smartlead"])
expect(said).not.toContain(app);
});

test("an inference, however likely, is still called unconfirmed, on every model", () => {
// GPT-5.6 Terra took company names from email domains and reported them "reliably inferred, so
// none remain unconfirmed", which the claims grader fails.
for (const model of [
"claude-sonnet-5-5",
"gpt-5.6-terra",
"claude-opus-5-5",
undefined,
]) {
const howTo = flat(howToWorkGuidance(model));
expect(howTo).toContain(
"call anything you could not check unconfirmed; an inference, however likely, is still unconfirmed",
);
// The rest of the claims rule is as it was.
expect(howTo).toContain(
"report what you did, scoped to what you checked",
);
expect(howTo).toContain(
"only when a tool result in this conversation showed it",
);
expect(howTo).toContain("do not say there were no replies");
expect(howTo).toContain(
"end with what you could not confirm, if anything",
);
}
});

test("a spreadsheet gets live formulas, labelled inputs, units, formats, one tab and a read-back", () => {
const said = flat(DOCUMENT_GUIDANCE);
expect(said).toContain("live formulas");
Expand Down
32 changes: 26 additions & 6 deletions shared/bot-prompt.ts
Original file line number Diff line number Diff line change
Expand Up @@ -531,7 +531,14 @@ const HOW_TO_WORK_LINES = [
"thing keeps failing, stop and say briefly what failed and what you got done.",
];

/** Say a change is done only once a result showed it, with the read-back for a model that needs it. */
/**
* Say a change is done only once a result showed it, with the read-back for a model that needs it.
*
* AN INFERENCE IS STILL UNCONFIRMED. GPT-5.6 Terra, having taken company names from email domains,
* reported them "reliably inferred, so none remain unconfirmed", in 1 of 6 replays of one job, which
* the claims grader fails. Rewording the rest of this rule to keep checks out of the report (so
* Claude would stop writing "I read the sheet back") made no difference to length, and is not done.
*/
function claimsLines(readBack: boolean): string[] {
return [
"Say something is done, sent, saved or correct only when a tool result in this conversation",
Expand All @@ -544,7 +551,8 @@ function claimsLines(readBack: boolean): string[] {
]
: []),
"Report what you did, scoped to what you checked: say 'all' only if you checked every item, and",
"call anything you could not check unconfirmed. Leave out what you did not look at: do not say",
"call anything you could not check unconfirmed; an inference, however likely, is still unconfirmed.",
"Leave out what you did not look at: do not say",
"there were no replies, nothing else or nothing newer unless a result showed that. Deliver what",
"was asked even when part of it is missing, and end with what you could not confirm, if anything.",
];
Expand Down Expand Up @@ -603,6 +611,14 @@ export const HOW_TO_WORK_GUIDANCE = howToWorkGuidance();
* GPT-6.1 made that line a message of its own on every spreadsheet job. The last message still
* stands on its own, because it may be the only one read.
*
* A finished job is shown the same way, by a worked example on apps no test uses. Told "one or two
* sentences, then anything you could not confirm, a sentence each", Claude Sonnet 5.5 reported a
* three-app job in 166 to 252 words where GPT-5.6 Terra took 45 to 84: a line per item the sheet
* already held, the checks it ran, what it left out and why, and every inference as its own caveat.
* So the caveats are one sentence for what the person must check, and the example keeps one
* inference, said to be one, because the claims rule is unchanged. A delegated task is told the
* same after its brief (`BRIEF_CLOSE` in orchestration.ts), where Claude reads it last.
*
* It says it is about words. Generative UI is unchanged: a component the answer calls for is drawn.
*/
const REPLY_LINES = [
Expand All @@ -617,10 +633,14 @@ const REPLY_LINES = [
"about?' A request worded as a question, such as 'can you send me today's agenda?', is a request:",
"do it.",
"",
"When you finish a job, say what you did in one or two sentences, with the result or a link to it,",
"then anything you could not confirm, a sentence each. Do not describe what you made when the link",
"shows it, recap your steps, repeat a summary you already gave, add sections nobody asked for, or",
"offer more work.",
"When you finish a job, say what you did in one or two sentences, with the result or a link to it.",
"Do not list again what the link, a sheet, a message or a component already holds, recap your steps,",
"say what you checked, read back or left out, repeat a summary you already gave, add sections",
"nobody asked for, or offer more work. Add a sentence only for what the person must check: something",
"you inferred rather than read, or a part that did not go through. Asked to find this week's",
"invoices in email, save them to a Drive folder and tell the Finance channel, a whole report is: 'I",
"saved this week's 4 invoices to [Invoices, week 46](link) and posted the list in #finance. Two",
"vendor names come from their email addresses, so check those.'",
"While you work, do not announce or narrate your steps: the person sees each one as it runs, so",
"write when you have the result. Your last message should still make sense on its own and say",
"what is left, if anything.",
Expand Down
37 changes: 34 additions & 3 deletions shared/orchestration.ts
Original file line number Diff line number Diff line change
Expand Up @@ -76,6 +76,12 @@ export const REACTIONS = ["👍", "❤️", "🎉", "👀", "✅", "🙏", "😂
* search itself instead of handing it off, and told "stop the Figma one" it stopped the only task
* the conversation had, the venue's, in 2 of 3 runs. Stopping work the person still wants is worse
* than stopping nothing, so a stop that names no task of this conversation stops nothing.
*
* NOTHING HERE ON HOW THE TASK REPORTS. Claude's briefs often end "report back with ... anything you
* could not confirm", and the task's report is shaped after the brief instead (`BRIEF_CLOSE`). Told
* here to end a brief at what to check, Claude wrote shorter briefs that dropped the work's own
* checks: of 24 replayed briefs for "emails from the last day", 4 kept a rolling 24 hours (17 before),
* and the task then took a 30-hour-old email as the last day's.
*/
export const DELEGATING_GUIDANCE = [
`When you hold ${DELEGATE_TOOL}, you are the person's point of contact and the work they ask for runs in task threads of your own, so this conversation stays short. Each turn, do exactly one of these three things, never two:`,
Expand Down Expand Up @@ -299,6 +305,15 @@ export function readTaskReport(
*
* NAMES, NOT IDS. A task that posted in Slack quoted its message back with the mention as Slack
* stores it, `<@U0E2EALEX0>`.
*
* A REPORT OF WORK DONE IS SHAPED AFTER THE BRIEF (`BRIEF_CLOSE`). Claude Sonnet 5.5 reported a
* three-app job (read the inbox, build a sheet, post in #sales) in 166 to 252 words, on OpenBot's own
* runner and on the Claude Agent SDK alike, where GPT-5.6 Terra took 45 to 84: the sheet's rows
* again, "I read the sheet back", what it left out and why, and each inference as a bullet. Claude
* follows the brief, which comes last and often asks for "anything you could not confirm", over a
* rule before it. Replayed on the real model: rules reworded here, 145 to 190 words; "under 60
* words", 104 to 168; one paragraph of at most four sentences, no list, said after the brief, 81 to
* 114 on the Claude Agent SDK. A question's answer is left as it was.
*/
const BRIEF_FRAME = [
"This is a task from your conversation with the person, handed into a thread of its own so that conversation stays short. Do the work here with your tools, as you would there, and finish it.",
Expand Down Expand Up @@ -351,15 +366,31 @@ const BRIEF_PREFIXES = [
],
].map((frame) => [...frame, "", ""].join("\n"));

/**
* What follows the brief: the shape of a report of work done, whatever the brief asked it to hold.
* See `BRIEF_FRAME` on why it comes last. The example is on apps no test uses, and keeps one
* inference, said to be one, because the claims rule is unchanged.
*/
export const BRIEF_CLOSE =
"\n\nWhen the task above is to make, change or send something, your report is one paragraph of at most four sentences, with no list and no headings, whatever the task asks it to include: what you did, with its link, then at most one sentence on what the person should check, such as something you inferred rather than read. That sentence is where what you could not confirm goes, and nothing follows it. Leave out the items again, the checks you ran, what you read back and what you left out. Asked to find this week's invoices in email, save them to a Drive folder and tell the Finance channel, a whole report is: 'I saved this week's 4 invoices to [Invoices, week 46](link) and posted the list in #finance. Two vendor names come from their email addresses, so check those.'";

/** The brief, wrapped in the words that tell the turn it is a delegated task. */
export function frameBrief(brief: string): string {
return `${BRIEF_PREFIXES[0]}${brief}`;
return `${BRIEF_PREFIXES[0]}${brief}${BRIEF_CLOSE}`;
}

/** The brief back out of a framed message, or null if this was not one. See `readFiring`. */
/**
* The brief back out of a framed message, or null if this was not one. See `readFiring`. A thread
* opened before `BRIEF_CLOSE` was said has none to take off.
*/
export function readBrief(text: string): string | null {
for (const prefix of BRIEF_PREFIXES)
if (text.startsWith(prefix)) return text.slice(prefix.length);
if (text.startsWith(prefix)) {
const brief = text.slice(prefix.length);
return brief.endsWith(BRIEF_CLOSE)
? brief.slice(0, -BRIEF_CLOSE.length)
: brief;
}
return null;
}

Expand Down
53 changes: 53 additions & 0 deletions shared/tests/orchestration.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -77,6 +77,48 @@ describe("a delegated task's brief", () => {
expect(framed).toContain("never by an app's id or its markup for one");
});

test("a report of work done is shaped after the brief: a few sentences, no list, one for what to check", () => {
// Claude Sonnet 5.5 reported a three-app job in 166 to 252 words, where GPT-5.6 Terra took 45 to
// 84, answering its brief's "report back with ... anything you could not confirm" item by item.
// Said after the brief, where Claude reads it last, the report came to 81 to 114 words.
const brief =
"Put the last day's customer emails in a new sheet and post a short summary in #sales. Report back with the sheet link and anything you could not confirm.";
const framed = frameBrief(brief);
const after = framed.slice(framed.indexOf(brief) + brief.length);
expect(after).toContain(
"your report is one paragraph of at most four sentences, with no list and no headings, whatever the task asks it to include",
);
expect(after).toContain(
"then at most one sentence on what the person should check",
);
expect(after).toContain(
"That sentence is where what you could not confirm goes, and nothing follows it.",
);
expect(after).toContain(
"Leave out the items again, the checks you ran, what you read back and what you left out.",
);
// Only for work done: a question's answer is shaped by the frame before the brief, as it was.
expect(after).toContain(
"When the task above is to make, change or send something",
);
// The example keeps the claims rule (an inference is said to be one), on apps no test uses.
expect(after).toContain(
"a whole report is: 'I saved this week's 4 invoices to [Invoices, week 46](link) and posted the list in #finance. Two vendor names come from their email addresses, so check those.'",
);
for (const app of ["Gmail", "Sheets", "Slack", "Smartlead"])
expect(after).not.toContain(app);
// Read back, the brief is the brief alone.
expect(readBrief(framed)).toBe(brief);
});

test("a thread opened before a report of work done was shaped after the brief still reads as the Bot's brief", () => {
const before = `${frameBrief("Post the summary in #sales.").split("Post the summary in #sales.")[0]}Post the summary in #sales.`;
expect(before.endsWith("The task:\n\nPost the summary in #sales.")).toBe(
true,
);
expect(readBrief(before)).toBe("Post the summary in #sales.");
});

test("a thread opened before the report was asked to be the result alone still reads as the Bot's brief", () => {
const before =
"This is a task from your conversation with the person, handed into a thread of its own so that conversation stays short. Do the work here with your tools, as you would there, and finish it.\nAnswer the way you always do: when the answer has structure (figures, a comparison, a list of items, a plan, a record), draw it with a component, and say in words what it shows and what stands out. Never write out what a component already shows: no Markdown table or list of the same rows beside it.\nLead with the result and say plainly anything you could not confirm. Your answer and any components you draw are posted back in that conversation as they are, your words first; what you say while you work stays in this thread. Write it to the person: never mention this brief, and never say where a component sits, such as 'the chart above'.\nThe task:\n\nFind Dana's latest email.";
Expand Down Expand Up @@ -118,6 +160,17 @@ describe("how a Bot answers a short ask itself", () => {
* order itself, and wrote 👍 as a reply beside its 👍 reaction.
*/
describe("how a Bot that holds delegate is told to work", () => {
test("a brief is not told to stop short: the work's own checks stay in it", () => {
// Told to end a brief at what to check, Claude wrote shorter briefs: of 24 replayed briefs for
// "emails from the last day", 4 kept a rolling 24 hours (17 without the line), and the task then
// took a 30-hour-old email as the last day's. How the task reports is shaped after the brief.
expect(DELEGATING_GUIDANCE).not.toContain("end it there");
expect(DELEGATING_GUIDANCE).not.toContain("how to report");
expect(DELEGATING_GUIDANCE).toContain(
"every figure, name, link and constraint from this conversation the work needs",
);
});

test("names the work that goes to a task, so it is handed off without being asked", () => {
expect(DELEGATING_GUIDANCE).toContain("without waiting to be asked");
// On Claude Sonnet 5.5 a chart of four figures and a sheet with a total were done in the
Expand Down
Loading