From 0171fcc105987266abfa818eb9f239c14895bda0 Mon Sep 17 00:00:00 2001 From: David McKay Date: Mon, 16 Nov 2026 10:32:17 -0800 Subject: [PATCH 1/2] Draw a Bot's maths as formulas A Bot asked for a loan payment answered with LaTeX, and the chat drew it as source with the backslashes of \( and \[ eaten by markdown. Bot text now goes through Streamdown's maths plugin (remark-math and KaTeX) on every surface that draws it: replies, task threads, a task's result line in the conversation that asked, card text, interface markdown, group chats, tool results, run output and learned skills. The model's \( \) and \[ \] are rewritten to the $$ the plugin reads; code, existing $$ maths and single-dollar prices are left as written. The plugin and KaTeX's stylesheet and fonts load with the first formula, as the code plugin does. A set-apart formula scrolls sideways in its own block. --- .github/workflows/ci.yml | 5 +- app/package.json | 2 + .../components/channels/chat-transcript.tsx | 14 +- app/src/components/channels/group-chat.tsx | 6 +- app/src/components/channels/tool-activity.tsx | 11 +- app/src/components/gallery/markdown.tsx | 6 +- app/src/components/learning/skill-files.tsx | 7 +- .../components/responsibilities/goal-page.tsx | 7 +- app/src/lib/copilot/a2ui-markdown.tsx | 11 +- app/src/lib/markdown.tsx | 91 ++++- app/src/lib/math-plugin.ts | 18 + app/src/lib/maths.ts | 186 +++++++++++ app/src/styles.css | 26 ++ app/tests/maths-delimiters.test.ts | 76 +++++ app/tests/maths-rendering.fixture.tsx | 96 ++++++ app/tests/maths-rendering.test.tsx | 312 ++++++++++++++++++ bun.lock | 4 + 17 files changed, 846 insertions(+), 32 deletions(-) create mode 100644 app/src/lib/math-plugin.ts create mode 100644 app/src/lib/maths.ts create mode 100644 app/tests/maths-delimiters.test.ts create mode 100644 app/tests/maths-rendering.fixture.tsx create mode 100644 app/tests/maths-rendering.test.tsx diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a78e0f5cd2..f5e56cb822 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -197,7 +197,8 @@ jobs: # Publish dialog holding their layout wide, narrow and on a phone, and the Admin screens # where a rule or a grant names a team holding theirs wide and on a phone, an unticked # tickbox's edge standing 3:1 from what it is drawn on in light and dark, a Needs you row read - # whole at a phone's width and a chat's with its answers on a line of their own, which only a + # whole at a phone's width and a chat's with its answers on a line of their own, a Bot's maths + # drawn as formulas in KaTeX's font with a wide one scrolling in its own block, which only a # browser's layout, cascade # and paint resolve, and the app following the system's light or dark, which only a # browser's `prefers-color-scheme` reports, with what OpenBot Desktop's window is told of @@ -206,7 +207,7 @@ jobs: # test script's pretest does). - run: | bun run generate:app-config - bun test app/tests/tool-line-reduced-motion.test.tsx app/tests/onboarding-window-buttons.test.tsx app/tests/onboarding-orb-room.test.tsx app/tests/transcript-rail-clearance.test.tsx app/tests/a2ui-frame.test.tsx app/tests/update-banner-layout.test.tsx app/tests/composer-dictation-notice-layout.test.tsx app/tests/conversation-intro-halo.test.tsx app/tests/transcript-room.test.tsx app/tests/skins-window-buttons.test.tsx app/tests/opendots-rail-badge.test.tsx app/tests/account-version-layout.test.tsx app/tests/organization-menu-names.test.tsx app/tests/bot-orb-one-color.test.tsx app/tests/theme-follows-system.test.tsx app/tests/desktop-window-theme.test.tsx app/tests/bot-screen-record-layout.test.tsx app/tests/teams-screens-layout.test.tsx app/tests/team-grants-screens-layout.test.tsx app/tests/needs-you-board-layout.test.tsx app/tests/checkbox-contrast.test.tsx + bun test app/tests/tool-line-reduced-motion.test.tsx app/tests/onboarding-window-buttons.test.tsx app/tests/onboarding-orb-room.test.tsx app/tests/transcript-rail-clearance.test.tsx app/tests/a2ui-frame.test.tsx app/tests/update-banner-layout.test.tsx app/tests/composer-dictation-notice-layout.test.tsx app/tests/conversation-intro-halo.test.tsx app/tests/transcript-room.test.tsx app/tests/skins-window-buttons.test.tsx app/tests/opendots-rail-badge.test.tsx app/tests/account-version-layout.test.tsx app/tests/organization-menu-names.test.tsx app/tests/bot-orb-one-color.test.tsx app/tests/theme-follows-system.test.tsx app/tests/desktop-window-theme.test.tsx app/tests/bot-screen-record-layout.test.tsx app/tests/teams-screens-layout.test.tsx app/tests/team-grants-screens-layout.test.tsx app/tests/needs-you-board-layout.test.tsx app/tests/checkbox-contrast.test.tsx app/tests/maths-rendering.test.tsx env: OPENBOT_APP_BROWSER: '1' # The desktop setup window's, for the same reason: the AI step's form holding still while the diff --git a/app/package.json b/app/package.json index 1a473c2e59..31a4563c43 100644 --- a/app/package.json +++ b/app/package.json @@ -22,6 +22,7 @@ "@openbot/shared": "workspace:*", "@shadcn/react": "^0.3.0", "@streamdown/code": "2.0.0", + "@streamdown/math": "1.0.3", "@tabler/icons-react": "^3.36.1", "@tanstack/react-form": "^1.33.5", "@tanstack/react-hotkeys": "^0.10.0", @@ -36,6 +37,7 @@ "clsx": "^2.1.1", "diff": "9.0.0", "embla-carousel-react": "^8.6.0", + "katex": "0.16.47", "motion": "^13.1.0", "prompt-area": "^0.6.3", "react": "^19.2.0", diff --git a/app/src/components/channels/chat-transcript.tsx b/app/src/components/channels/chat-transcript.tsx index f01617a354..ba02bc7cfa 100644 --- a/app/src/components/channels/chat-transcript.tsx +++ b/app/src/components/channels/chat-transcript.tsx @@ -42,7 +42,6 @@ import { useRef, useState, } from "react"; -import { Streamdown } from "streamdown"; import { InlineApproval } from "@/components/approvals/inline-approval"; import { ArgumentsArrivingContext, @@ -104,6 +103,7 @@ import { folderStepSubject, } from "@/lib/folders/folder-steps"; import { + BotMarkdown, markdownComponents, markdownPlugins, replyRemarkPluginsFor, @@ -965,24 +965,24 @@ const TranscriptMessage = memo(function TranscriptMessage({ taskId={report.taskId} words={text} > - {revealed} - + ) : ( - {revealed} - + )} )} @@ -1902,9 +1902,9 @@ export function ServerToolLine({ running={running} > {shown ? ( - + {forDisplay(shown)} - + ) : null} ); diff --git a/app/src/components/channels/group-chat.tsx b/app/src/components/channels/group-chat.tsx index 22f353f896..950b6d646e 100644 --- a/app/src/components/channels/group-chat.tsx +++ b/app/src/components/channels/group-chat.tsx @@ -1,7 +1,6 @@ import { useMutation, useQuery, useQueryClient } from "@tanstack/react-query"; import { Link } from "@tanstack/react-router"; import { useEffect, useLayoutEffect, useRef } from "react"; -import { Streamdown } from "streamdown"; import { QuestionNotice, type WaitingQuestion, @@ -27,6 +26,7 @@ import { sendGroupMessageMutationOptions, } from "@/lib/groups"; import { + BotMarkdown, markdownComponents, markdownPlugins, replyRemarkPlugins, @@ -339,13 +339,13 @@ function GroupLine({ className="w-full" > - {message.text} - + )} diff --git a/app/src/components/channels/tool-activity.tsx b/app/src/components/channels/tool-activity.tsx index d9a1552f0e..98adcff5eb 100644 --- a/app/src/components/channels/tool-activity.tsx +++ b/app/src/components/channels/tool-activity.tsx @@ -32,7 +32,6 @@ import { useState, useSyncExternalStore, } from "react"; -import { Streamdown } from "streamdown"; import { PendingApprovals } from "@/components/approvals/inline-approval"; import { codingLook, foreignCodingCall } from "@/lib/copilot/coding-calls"; import { @@ -41,7 +40,11 @@ import { folderStepBody, folderStepSubject, } from "@/lib/folders/folder-steps"; -import { markdownComponents, markdownPlugins } from "@/lib/markdown"; +import { + BotMarkdown, + markdownComponents, + markdownPlugins, +} from "@/lib/markdown"; import { RESEARCH_RESULTS_LABELS, type ResearchSource, @@ -872,12 +875,12 @@ export function StepOutput({ ) : (
- {forDisplay(output)} - +
)} {long && !open ? ( diff --git a/app/src/components/gallery/markdown.tsx b/app/src/components/gallery/markdown.tsx index 23a85aa782..32c7a991d1 100644 --- a/app/src/components/gallery/markdown.tsx +++ b/app/src/components/gallery/markdown.tsx @@ -1,6 +1,6 @@ import { useContext, useState } from "react"; -import { Streamdown } from "streamdown"; import { + BotMarkdown, markdownComponents, markdownPlugins, replyRemarkPlugins, @@ -43,7 +43,7 @@ export function GalleryMarkdown({ ); if (!children) return null; return ( - {children} - + ); } diff --git a/app/src/components/learning/skill-files.tsx b/app/src/components/learning/skill-files.tsx index 86e72acb02..2773b94567 100644 --- a/app/src/components/learning/skill-files.tsx +++ b/app/src/components/learning/skill-files.tsx @@ -1,6 +1,5 @@ -import { Streamdown } from "streamdown"; import { readableSkillBody } from "@/lib/learning/display"; -import { markdownComponents } from "@/lib/markdown"; +import { BotMarkdown, markdownComponents } from "@/lib/markdown"; /** * A learned skill's files: SKILL.md rendered as the skill's own content, with no file name over it, and any other file @@ -20,13 +19,13 @@ export function SkillFiles({ return ( <> {skill?.text != null && ( - {readableSkillBody(skill.text)} - + )} {others.length > 0 && (
diff --git a/app/src/components/responsibilities/goal-page.tsx b/app/src/components/responsibilities/goal-page.tsx index ac9facd418..d3b4810cfc 100644 --- a/app/src/components/responsibilities/goal-page.tsx +++ b/app/src/components/responsibilities/goal-page.tsx @@ -13,7 +13,6 @@ import { import { useMutation, useQuery } from "@tanstack/react-query"; import { Link, type LinkProps, useNavigate } from "@tanstack/react-router"; import { Fragment, useId, useState } from "react"; -import { Streamdown } from "streamdown"; import { LoadingRows } from "@/components/layout/loading-rows"; import { PageCard, @@ -57,7 +56,7 @@ import { import { Separator } from "@/components/ui/separator"; import { Switch } from "@/components/ui/switch"; import { Textarea } from "@/components/ui/textarea"; -import { markdownComponents } from "@/lib/markdown"; +import { BotMarkdown, markdownComponents } from "@/lib/markdown"; import { deleteResponsibilityMutationKey, deleteResponsibilityMutationOptions, @@ -686,12 +685,12 @@ function Runs({ goalId, channelId }: { goalId: string; channelId: string }) { */ function RunMarkdown({ children }: { children: string }) { return ( - {children} - + ); } diff --git a/app/src/lib/copilot/a2ui-markdown.tsx b/app/src/lib/copilot/a2ui-markdown.tsx index 3ec1a51d3f..55ad5eba7d 100644 --- a/app/src/lib/copilot/a2ui-markdown.tsx +++ b/app/src/lib/copilot/a2ui-markdown.tsx @@ -1,5 +1,8 @@ -import { Streamdown } from "streamdown"; -import { markdownComponents, markdownPlugins } from "@/lib/markdown"; +import { + BotMarkdown, + markdownComponents, + markdownPlugins, +} from "@/lib/markdown"; /** * Markdown and code a Bot draws in an interface, drawn the way its reply draws them. @@ -31,7 +34,7 @@ function ReplyMarkdown({ }) { return (
- {children} - +
); } diff --git a/app/src/lib/markdown.tsx b/app/src/lib/markdown.tsx index 4724bc0719..8c39a65376 100644 --- a/app/src/lib/markdown.tsx +++ b/app/src/lib/markdown.tsx @@ -5,12 +5,21 @@ import { IconTable, } from "@tabler/icons-react"; import { useRouter } from "@tanstack/react-router"; -import type { ComponentProps } from "react"; +import { + type ComponentProps, + useEffect, + useMemo, + useSyncExternalStore, +} from "react"; import { type CodeHighlighterPlugin, defaultRemarkPlugins, + type MathPlugin, + type PluginConfig, + Streamdown, type StreamdownProps, } from "streamdown"; +import { hasMaths, withMathDelimiters } from "@/lib/maths"; import { NO_SLACK_NAMES, remarkSlackMentions, @@ -327,3 +336,83 @@ export function replyRemarkPluginsFor( /** A reply with no conversation's Slack names to hand: mentions still never show as ids. */ export const replyRemarkPlugins = replyRemarkPluginsFor(NO_SLACK_NAMES); + +let mathPlugin: MathPlugin | undefined; +let mathLoad: Promise | undefined; +const mathListeners = new Set<() => void>(); + +/** Streamdown's maths plugin and KaTeX's stylesheet, fetched the first time a formula is drawn. */ +function loadMathPlugin() { + mathLoad ??= import("@/lib/math-plugin") + .then(({ math }) => { + mathPlugin = math; + for (const listener of mathListeners) listener(); + }) + .catch((error: unknown) => { + // The formula stays drawn as its source; say why rather than leave it so for no reason. + console.error("Couldn't load maths rendering:", error); + mathLoad = undefined; + }); + return mathLoad; +} + +function onMathPlugin(listener: () => void) { + mathListeners.add(listener); + return () => { + mathListeners.delete(listener); + }; +} + +const currentMathPlugin = () => mathPlugin; + +/** One plugin set per set the caller passes, so Streamdown sees the same object on every render. */ +const withMath = new WeakMap(); +const NO_PLUGINS: PluginConfig = {}; + +function pluginsWithMath( + plugins: PluginConfig | undefined, + math: MathPlugin | undefined, +): PluginConfig | undefined { + if (!math) return plugins; + const base = plugins ?? NO_PLUGINS; + let all = withMath.get(base); + if (!all) { + all = { ...base, math }; + withMath.set(base, all); + } + return all; +} + +/** + * Streamdown, for a Bot's words wherever they are drawn, with its maths drawn as formulas. + * + * Takes everything Streamdown takes. The markdown has its `\( \)` and `\[ \]` written in the `$$` + * the maths plugin reads (`lib/maths.ts`), and the plugin is added to the caller's plugins. + * + * LOADED WITH THE FIRST FORMULA, NOT WITH THE CHAT, as the code plugin is. KaTeX and its stylesheet + * are most of the weight of drawing maths, and most conversations have none. The first formula is + * drawn as its source for the moment the plugin takes to arrive, then as a formula; every one after + * that draws as a formula from the start. Once in, the plugin stays with every surface, so a reply + * that grows a formula as it streams keeps the same plugins. + */ +export function BotMarkdown({ + children, + plugins, + ...props +}: Omit & { children: string }) { + const markdown = useMemo(() => withMathDelimiters(children), [children]); + const wantsMaths = hasMaths(markdown); + const math = useSyncExternalStore( + onMathPlugin, + currentMathPlugin, + currentMathPlugin, + ); + useEffect(() => { + if (wantsMaths && !math) void loadMathPlugin(); + }, [wantsMaths, math]); + return ( + + {markdown} + + ); +} diff --git a/app/src/lib/math-plugin.ts b/app/src/lib/math-plugin.ts new file mode 100644 index 0000000000..57d137d0a5 --- /dev/null +++ b/app/src/lib/math-plugin.ts @@ -0,0 +1,18 @@ +import { createMathPlugin } from "@streamdown/math"; +/* + * KaTeX's stylesheet and its fonts, which the stylesheet names by relative URL: the build emits the + * fonts beside the app's other assets and serves them from the app's own origin, the desktop's + * included, so a formula never reaches out to a CDN. Imported here, with the plugin, so the + * stylesheet arrives with the first formula and not with every conversation. + */ +import "katex/dist/katex.min.css"; + +/** + * Streamdown's maths plugin (remark-math and KaTeX). A single `$` stays a dollar sign, which is the + * plugin's default: "$25,000" in an answer is a price. A formula KaTeX can't read is drawn as its + * source in the muted text colour, as the plugin does, rather than as an error. + */ +export const math = createMathPlugin({ + singleDollarTextMath: false, + errorColor: "var(--muted-foreground)", +}); diff --git a/app/src/lib/maths.ts b/app/src/lib/maths.ts new file mode 100644 index 0000000000..d9799186b6 --- /dev/null +++ b/app/src/lib/maths.ts @@ -0,0 +1,186 @@ +/** + * Maths in a Bot's markdown, in the delimiters Streamdown's maths plugin reads. + * + * WHY. A model writing a formula in a chat writes LaTeX, and most write it the way LaTeX itself + * does: `\( ... \)` inside a sentence and `\[ ... \]` on lines of their own. Markdown reads `\(` + * and `\[` as an escaped bracket, so before this the reader saw "(P r / [1-(1+r)^{-n}])" with the + * backslashes gone and the formula as source. Streamdown's maths plugin (remark-math and KaTeX) + * reads `$$`: on lines of their own a formula set apart, inside a sentence a formula in the line. + * So `\[ ... \]` that stands on its own lines becomes a `$$` block, and `\( ... \)`, or a `\[ ... \]` + * in the middle of a sentence, becomes `$$ ... $$` in the line. Nothing else is rewritten. + * + * WHAT IT NEVER TOUCHES. Fenced code (``` or ~~~) and inline code keep every character, so code + * that escapes a bracket stays code. A `$$ ... $$` already written is left as it is. A single `$` is + * never maths (Streamdown's default), so "$25,000" and "$5 to $10" stay prices. An opener with no + * closer before the end of its paragraph stays as written: while a reply streams, a formula is + * drawn once its closer has arrived, and a stray `\(` in prose never swallows the rest of the + * message. A backslash that is itself escaped (`\\(`) is not an opener. + */ + +const FENCE = /^[ \t]*(?:>[ \t]*)*(`{3,}|~{3,})/; + +/** The length of the run of `char` starting at `index`. */ +function runAt(text: string, index: number, char: string): number { + let end = index; + while (text[end] === char) end += 1; + return end - index; +} + +/** + * Where the closer of a `\(` or `\[` opened just before `from` starts, or -1. The closer is a `\)` + * or `\]` whose backslash is not itself escaped, before the paragraph ends (a blank line). + */ +function closerFrom(text: string, from: number, close: ")" | "]"): number { + const paragraphEnd = text.slice(from).search(/\n[ \t]*\n/); + const end = paragraphEnd === -1 ? text.length : from + paragraphEnd; + let index = from; + while (index < end) { + if (text[index] !== "\\") { + index += 1; + continue; + } + const run = runAt(text, index, "\\"); + const after = index + run; + if (run % 2 === 1 && text[after] === close && after < end) return after - 1; + index = after; + } + return -1; +} + +/** The text from the start of the line `index` is on, up to `index`. */ +function lineBefore(text: string, index: number): string { + return text.slice(text.lastIndexOf("\n", index - 1) + 1, index); +} + +/** The text from `index` to the end of its line. */ +function lineAfter(text: string, index: number): string { + const end = text.indexOf("\n", index); + return text.slice(index, end === -1 ? text.length : end); +} + +/** + * A dollar sign inside a formula, as KaTeX reads one. Models write "\\(P=$25,000\\)" for a price + * in a formula; a bare `$` there would end the `$$` early or reach KaTeX as an error. + */ +function escapeDollars(formula: string): string { + return formula.replace(/(? line.trim()) + .filter(Boolean) + .map((line) => `${indent}${line}`); + return `$$\n${lines.join("\n")}\n${indent}$$`; +} + +/** A formula in the line. */ +function inlineFormula(formula: string): string { + return `$$${forKatex(formula.trim()).replace(/\s*\n\s*/g, " ")}$$`; +} + +/** Prose, with code spans and `$$` maths kept as written and `\( \)` and `\[ \]` rewritten. */ +function rewriteProse(text: string): string { + let out = ""; + let index = 0; + while (index < text.length) { + const char = text[index]; + if (char === "`") { + // A code span closes on a run of exactly as many backticks. + const run = runAt(text, index, "`"); + let close = text.indexOf("`".repeat(run), index + run); + while (close !== -1 && runAt(text, close, "`") !== run) + close = text.indexOf("`".repeat(run), close + runAt(text, close, "`")); + const end = close === -1 ? index + run : close + run; + out += text.slice(index, end); + index = end; + continue; + } + if (char === "$" && text[index + 1] === "$") { + const close = text.indexOf("$$", index + 2); + const end = close === -1 ? index + 2 : close + 2; + out += text.slice(index, end); + index = end; + continue; + } + if (char === "\\") { + const run = runAt(text, index, "\\"); + const opener = index + run - 1; + const kind = text[opener + 1]; + if (run % 2 === 1 && (kind === "(" || kind === "[")) { + const from = opener + 2; + const close = closerFrom(text, from, kind === "(" ? ")" : "]"); + const formula = close === -1 ? "" : text.slice(from, close); + if (formula.trim()) { + out += text.slice(index, opener); + const indent = lineBefore(text, opener); + const standsAlone = + kind === "[" && + /^[ \t]*$/.test(indent) && + /^[ \t]*$/.test(lineAfter(text, close + 2)); + out += standsAlone + ? displayBlock(formula, indent) + : inlineFormula(formula); + index = close + 2; + continue; + } + } + out += text.slice(index, index + run); + index += run; + continue; + } + out += char; + index += 1; + } + return out; +} + +/** `markdown` with `\( \)` and `\[ \]` maths written in the `$$` the maths plugin reads. */ +export function withMathDelimiters(markdown: string): string { + if (!markdown.includes("\\(") && !markdown.includes("\\[")) return markdown; + const lines = markdown.split(/(?<=\n)/); + let out = ""; + let prose = ""; + let fence: string | null = null; + for (const line of lines) { + const opened = FENCE.exec(line)?.[1]; + if (fence === null) { + if (opened) { + out += rewriteProse(prose) + line; + prose = ""; + fence = opened; + } else prose += line; + continue; + } + out += line; + // A fence closes on a run of the same character, at least as long, with nothing after it. + if ( + opened && + opened[0] === fence[0] && + opened.length >= fence.length && + line.trim().replace(/^(?:>[ \t]*)*/, "") === opened + ) + fence = null; + } + return out + rewriteProse(prose); +} + +/** Whether markdown holds maths for the maths plugin to draw. */ +export function hasMaths(markdown: string): boolean { + return markdown.includes("$$"); +} diff --git a/app/src/styles.css b/app/src/styles.css index e6d04cac78..d459140d9f 100644 --- a/app/src/styles.css +++ b/app/src/styles.css @@ -15,6 +15,32 @@ @source "../node_modules/streamdown/dist/*.js"; /* Its code plugin, which Streamdown's docs say to scan the same way. */ @source "../node_modules/@streamdown/code/dist/*.js"; +/* And its maths plugin, likewise. */ +@source "../node_modules/@streamdown/math/dist/*.js"; + +/* + * Maths a Bot writes, drawn by KaTeX (lib/math-plugin.ts). KaTeX's own fonts set the formula, as + * maths is set everywhere, and it takes the colour of the words around it, so it follows the theme + * with them. KaTeX draws a formula at 1.21 times the text round it; at the chat's size that stood a + * line taller than the prose beside it, so it is drawn at 1.1, which keeps a fraction readable + * without loosening the line. + * + * A formula set apart is centred on a block of its own, as KaTeX sets it, and scrolls sideways from + * its start when it is wider than the column, on a phone or in a card, and never the page. The scroll leaves a little room above + * and below so a fraction's top and a subscript's foot are not clipped by it. + */ +.katex { + font-size: 1.1em; +} + +.katex-display { + margin-block: 0.75rem; + max-width: 100%; + overflow-x: auto; + overflow-y: hidden; + padding-block: 0.25rem; + scrollbar-width: thin; +} /* * OpenBot Desktop on macOS has no separate title bar over this page: the window's buttons sit over diff --git a/app/tests/maths-delimiters.test.ts b/app/tests/maths-delimiters.test.ts new file mode 100644 index 0000000000..c35a1cc1ba --- /dev/null +++ b/app/tests/maths-delimiters.test.ts @@ -0,0 +1,76 @@ +import { describe, expect, test } from "bun:test"; +import { withMathDelimiters } from "@/lib/maths"; + +/** + * A Bot's LaTeX delimiters, written in the `$$` Streamdown's maths plugin reads, and nothing else + * touched. The answers are the shapes models write in a chat: the loan answer below is the one a + * General Assistant gave on 2026-11-16, which the chat drew as "(P r / [1-(1+r)^{-n}])". + * maths-rendering.test.tsx draws the same answers in a browser. + */ +describe("maths delimiters", () => { + test("a formula inside a sentence becomes inline maths, with a price in it kept a dollar", () => { + expect( + withMathDelimiters( + "The monthly payment is **$489.15**, calculated using \\(P r / [1-(1+r)^{-n}]\\), where \\(P=$25,000\\), \\(r=0.065/12\\), and \\(n=60\\).", + ), + ).toBe( + "The monthly payment is **$489.15**, calculated using $$P r / [1-(1+r)^{-n}]$$, where $$P=\\$25{,}000$$, $$r=0.065/12$$, and $$n=60$$.", + ); + }); + + test("a number's thousands separator stays a separator, and a list's comma stays a comma", () => { + expect(withMathDelimiters("\\(1,000,000 + f(1,2) + x_{1,2,3}\\)")).toBe( + "$$1{,}000{,}000 + f(1,2) + x_{1,2,3}$$", + ); + }); + + test("a formula on lines of its own becomes a block, at the indent it was written at", () => { + expect( + withMathDelimiters( + "Compound interest:\n\n\\[\nA=P\\left(1+\\frac{r}{n}\\right)^{nt}\n\\]\n\nHere, \\(A\\) is the final balance.", + ), + ).toBe( + "Compound interest:\n\n$$\nA=P\\left(1+\\frac{r}{n}\\right)^{nt}\n$$\n\nHere, $$A$$ is the final balance.", + ); + expect( + withMathDelimiters( + "1. The payment:\n \\[ M = P\\frac{r}{1-(1+r)^{-n}} \\]", + ), + ).toBe("1. The payment:\n $$\n M = P\\frac{r}{1-(1+r)^{-n}}\n $$"); + }); + + test("a set-apart formula in the middle of a sentence stays in the line", () => { + expect(withMathDelimiters("so \\[x^2\\] holds")).toBe("so $$x^2$$ holds"); + }); + + test("prices, code and an escaped backslash stay exactly as written", () => { + const untouched = [ + "The loan is $25,000 at 6.5%, about $489.15 a month, or $5 to $10 more with fees.", + "Match a call with `\\(\\d+\\)` in the pattern.", + '```python\nprice = f"$" + amount\npattern = r"\\(\\d+\\)"\n```', + "~~~\n\\[ not maths \\]\n~~~", + "A literal \\\\(not an opener\\\\) in prose.", + "Already maths: $$\\frac{1}{2}$$ and \\(", + "An opener with no closer \\( in this paragraph.\n\nAnd a closer \\) in the next.", + ]; + for (const markdown of untouched) + expect(withMathDelimiters(markdown)).toBe(markdown); + }); + + test("code beside a formula keeps its own text", () => { + expect( + withMathDelimiters( + "Use \\(x_1\\) here.\n\n```js\nconst re = /\\(\\d\\)/;\nconst cost = `$${n}`;\n```\n\nThen \\(x_2\\).", + ), + ).toBe( + "Use $$x_1$$ here.\n\n```js\nconst re = /\\(\\d\\)/;\nconst cost = `$${n}`;\n```\n\nThen $$x_2$$.", + ); + }); + + test("while a reply streams, a formula whose closer has not arrived stays as written", () => { + expect(withMathDelimiters("Using \\(P r / [1-(1+r)^")).toBe( + "Using \\(P r / [1-(1+r)^", + ); + expect(withMathDelimiters("\\[\nA=P\\left(1+")).toBe("\\[\nA=P\\left(1+"); + }); +}); diff --git a/app/tests/maths-rendering.fixture.tsx b/app/tests/maths-rendering.fixture.tsx new file mode 100644 index 0000000000..f6227e1d4a --- /dev/null +++ b/app/tests/maths-rendering.fixture.tsx @@ -0,0 +1,96 @@ +import type { Message } from "@ag-ui/core"; +import { CopilotKitProvider } from "@copilotkit/react-core/v2"; +import { taskReportMessageId } from "@openbot/shared/orchestration"; +import { QueryClient, QueryClientProvider } from "@tanstack/react-query"; +import { createRoot } from "react-dom/client"; +import { ChatTranscript } from "@/components/channels/chat-transcript"; +import { GalleryMarkdown } from "@/components/gallery/markdown"; +import { TooltipProvider } from "@/components/ui/tooltip"; + +/** + * One answer with maths in it, drawn where a Bot's words are drawn, as a page of its own for + * maths-rendering.test.tsx to bundle and load in a browser. + * + * ?surface=direct a reply in the conversation + * ?surface=thread the task's last message in its own thread + * ?surface=origin the task's result line in the conversation that asked for it + * ?surface=card the text of a card a Bot draws + * ?theme=dark the dark theme + * + * The answer is written as models write one: `\( \)` in a sentence, `\[ \]` on lines of its own and + * wider than a phone, `$$` in a sentence, prices with single dollars, and code with both in it. + */ +const params = new URLSearchParams(location.search); +if (params.get("theme") === "dark") + document.documentElement.classList.add("dark"); +const surface = params.get("surface") ?? "direct"; + +const ANSWER = [ + "The monthly payment is **$489.15**, calculated using \\(M = P\\frac{r}{1-(1+r)^{-n}}\\), where \\(P=$25,000\\) and \\(n=60\\).", + "", + "\\[", + "M = P\\cdot\\frac{r\\,(1+r)^{n}}{(1+r)^{n}-1} = 25000\\cdot\\frac{0.0054167\\,(1.0054167)^{60}}{(1.0054167)^{60}-1} \\approx 489.15", + "\\]", + "", + "The monthly rate is $$r = \\frac{0.065}{12}$$ for a 6.5% annual rate.", + "", + "The loan is $25,000 and fees run $5 to $10.", + "", + "```python", + 'price = "$25,000"', + 'pattern = r"\\(\\d+\\)"', + "```", +].join("\n"); + +const QUESTION: Message = { + id: "u1", + role: "user", + content: "What's the monthly payment on a $25,000 loan at 6.5% over 5 years?", +}; + +const messages: Record = { + direct: [QUESTION, { id: "a1", role: "assistant", content: ANSWER }], + thread: [ + { id: "u1", role: "user", content: "Calculate the monthly loan payment." }, + { id: "a1", role: "assistant", content: ANSWER }, + ], + origin: [ + QUESTION, + { + id: taskReportMessageId("result", "task-1", "run-1"), + role: "assistant", + content: ANSWER, + }, + ], +}; + +const queryClient = new QueryClient({ + defaultOptions: { queries: { retry: false } }, +}); + +const root = document.getElementById("root"); +if (root) { + createRoot(root).render( + + + +
+ {surface === "card" ? ( +
+ {ANSWER} +
+ ) : ( + + )} +
+
+
+
, + ); +} diff --git a/app/tests/maths-rendering.test.tsx b/app/tests/maths-rendering.test.tsx new file mode 100644 index 0000000000..aa18a9fc5e --- /dev/null +++ b/app/tests/maths-rendering.test.tsx @@ -0,0 +1,312 @@ +import { afterAll, beforeAll, describe, expect, test } from "bun:test"; +import { existsSync, mkdirSync, readFileSync } from "node:fs"; +import { basename, dirname, join } from "node:path"; +import type { Browser, Page } from "playwright-core"; +import { compile } from "tailwindcss"; + +/** + * Maths a Bot writes is drawn as formulas, everywhere its words are drawn: a reply, a task's thread, + * the task's result line in the conversation that asked for it, and a card's text. + * + * Seen on 2026-11-16: asked for a loan payment, a Bot answered "calculated using + * (P r / [1-(1+r)^{-n}])" and a compound-interest formula as "[ A=P\left(1+\frac{r}{n}\right)^{nt} ]". + * The chat drew LaTeX as source, and markdown had eaten the backslashes of `\(` and `\[`. + * + * The page is the real transcript and card text, bundled from maths-rendering.fixture.tsx with the + * app's stylesheet and KaTeX's, in a real browser. Each surface is checked for: every formula drawn + * by KaTeX in KaTeX's own font, the one set apart as a block; no delimiter left on screen; prices and + * the code block drawn as written; and on a phone, the wide formula scrolling sideways inside its + * own block while the page does not. + * + * OPENBOT_APP_BROWSER=1 bun test app/tests/maths-rendering.test.tsx + * + * Chromium by default, as CI has it. OPENBOT_APP_ENGINES=chromium,webkit adds WebKit. With + * OPENBOT_APP_SHOTS= each page is saved as maths----.png. + */ +const asked = process.env.OPENBOT_APP_BROWSER === "1"; +const shots = process.env.OPENBOT_APP_SHOTS; +const engines = (process.env.OPENBOT_APP_ENGINES ?? "chromium") + .split(",") + .map((engine) => engine.trim()) + .filter((engine) => engine === "chromium" || engine === "webkit") as ( + | "chromium" + | "webkit" +)[]; + +const APP = join(import.meta.dir, ".."); +const ORIGIN = "http://openbot.test"; +const SURFACES = ["direct", "thread", "origin", "card"] as const; +const THEMES = ["light", "dark"] as const; +const WIDTHS = [1440, 390] as const; + +/** The formulas the answer holds, as KaTeX keeps their source beside what it draws. */ +const FORMULAS = [ + "M = P\\frac{r}{1-(1+r)^{-n}}", + "P=\\$25{,}000", + "n=60", + "r = \\frac{0.065}{12}", +]; +const SET_APART = "\\approx 489.15"; + +/** The fixture and what it imports, KaTeX's stylesheet and fonts among them, built for a browser. */ +async function bundle(): Promise> { + const built = await Bun.build({ + entrypoints: [join(import.meta.dir, "maths-rendering.fixture.tsx")], + target: "browser", + naming: { entry: "fixture.[ext]", asset: "[name].[ext]" }, + define: { "process.env.NODE_ENV": JSON.stringify("production") }, + plugins: [ + { + name: "no-marks", + setup(build) { + build.onResolve({ filter: /^@\/lib\/marks$/ }, () => ({ + path: "marks", + namespace: "no-marks", + })); + build.onLoad({ filter: /.*/, namespace: "no-marks" }, () => ({ + contents: "export const markFor = () => null;", + loader: "js", + })); + }, + }, + ], + }); + if (!built.success) throw new AggregateError(built.logs, "bundle failed"); + const files = new Map(); + for (const output of built.outputs) + files.set(`/${basename(output.path)}`, { + type: output.type, + body: output, + }); + return files; +} + +/** Where `@import "[/]"` lands, by the package's `style` export as Vite reads it. */ +function packageStylesheet(id: string): string { + const parts = id.split("/"); + const name = id.startsWith("@") ? parts.slice(0, 2).join("/") : parts[0]; + const dir = [ + join(APP, "node_modules", name), + join(APP, "..", "node_modules", name), + ].find(existsSync); + if (!dir) throw new Error(`no package for ${id}`); + const pkg = JSON.parse(readFileSync(join(dir, "package.json"), "utf8")); + const subpath = `.${id.slice(name.length)}`; + const entry = pkg.exports?.[subpath]; + const file = + typeof entry === "string" + ? entry + : (entry?.style ?? entry?.default ?? pkg.main ?? "index.css"); + return join(dir, file); +} + +/** The app's stylesheet, built by Tailwind for these classes, as the app's build does. */ +async function stylesheet(classes: string[]): Promise { + const tailwind = dirname(Bun.resolveSync("tailwindcss/package.json", APP)); + const entry = join(APP, "src", "styles.css"); + const compiler = await compile(readFileSync(entry, "utf8"), { + base: dirname(entry), + loadStylesheet: async (id, base) => { + const path = + id === "tailwindcss" + ? join(tailwind, "index.css") + : id.startsWith(".") + ? join(base, id) + : packageStylesheet(id); + return { path, base: dirname(path), content: readFileSync(path, "utf8") }; + }, + }); + return compiler.build(classes); +} + +let files: Map; +const browsers = new Map(); +/** The app's stylesheet for every class the four surfaces draw, built once. */ +let css = ""; + +async function browserFor(name: "chromium" | "webkit"): Promise { + let browser = browsers.get(name); + if (!browser) { + const playwright = await import("playwright-core"); + browser = + name === "chromium" + ? // The CI job installs full Chromium without the headless shell. + await playwright.chromium.launch({ channel: "chromium" }) + : await playwright.webkit.launch(); + browsers.set(name, browser); + } + return browser; +} + +beforeAll(async () => { + if (!asked) return; + files = await bundle(); + const classes = new Set(); + for (const surface of SURFACES) { + const page = await open("chromium", "light", 1440, surface, false); + try { + for (const name of await page.evaluate(() => + [...document.querySelectorAll("[class]")].flatMap((element) => [ + ...element.classList, + ]), + )) + classes.add(name); + } finally { + await page.close(); + } + } + css = await stylesheet([...classes]); +}, 120_000); + +afterAll(async () => { + await Promise.all([...browsers.values()].map((browser) => browser.close())); +}); + +async function open( + engine: "chromium" | "webkit", + theme: (typeof THEMES)[number], + width: number, + surface: (typeof SURFACES)[number], + styled = true, +): Promise { + const browser = await browserFor(engine); + const page = await browser.newPage({ + viewport: { width, height: 900 }, + deviceScaleFactor: 2, + }); + page.on("pageerror", (error) => console.error(error)); + await page.route(`${ORIGIN}/**`, (route) => { + const path = new URL(route.request().url()).pathname; + if (path === "/") + return route.fulfill({ + contentType: "text/html", + body: `
`, + }); + const file = files.get(path); + if (file) + return file.body + .arrayBuffer() + .then((body) => + route.fulfill({ contentType: file.type, body: Buffer.from(body) }), + ); + return route.fulfill({ status: 404, body: "" }); + }); + await page.goto(`${ORIGIN}/?surface=${surface}&theme=${theme}`); + // The reply is revealed at a reading pace; its last paragraph is the code block. + await page + .getByText('pattern = r"\\(\\d+\\)"', { exact: false }) + .first() + .waitFor({ timeout: 20_000 }); + return page; +} + +/** What the surface shows, read from the page. */ +function read(page: Page) { + return page.evaluate(async (setApart) => { + await document.fonts.ready; + const surface = document.querySelector("[data-surface]"); + if (!surface) throw new Error("no surface drawn"); + const formulas = [...surface.querySelectorAll(".katex")]; + const source = (element: Element) => + element.querySelector('annotation[encoding="application/x-tex"]') + ?.textContent ?? ""; + const display = [...surface.querySelectorAll(".katex-display")]; + const wide = display.find((block) => source(block).includes(setApart)); + // The words a reader sees outside the formulas and the code block. + const prose = surface.cloneNode(true) as Element; + for (const element of prose.querySelectorAll( + ".katex, pre, code, [data-streamdown='code-block']", + )) + element.remove(); + const code = [...surface.querySelectorAll("pre, code")] + .map((element) => element.textContent ?? "") + .join("\n"); + return { + sources: formulas.map(source), + displaySources: display.map(source), + inCode: surface.querySelectorAll("pre .katex, code .katex").length, + prose: (prose.textContent ?? "").replace(/\s+/g, " "), + code, + katexFont: [...document.fonts].some( + (font) => + font.family.includes("KaTeX_Main") && font.status === "loaded", + ), + formulaFont: formulas[0] + ? getComputedStyle( + formulas[0].querySelector(".katex-html .mord") ?? formulas[0], + ).fontFamily + : "", + wide: wide + ? { scroll: wide.scrollWidth, client: wide.clientWidth } + : null, + pageScroll: document.documentElement.scrollWidth, + viewport: window.innerWidth, + }; + }, SET_APART); +} + +describe.skipIf(!asked)("maths a Bot writes", () => { + for (const engine of engines) { + for (const theme of THEMES) { + for (const width of WIDTHS) { + for (const surface of SURFACES) { + test(`${engine}, ${theme}, ${width}px, ${surface}: formulas drawn, prices and code as written`, async () => { + const page = await open(engine, theme, width, surface); + try { + // The first formula fetches the maths plugin; the checks below say if none came. + await page + .waitForSelector(".katex-display", { timeout: 10_000 }) + .catch(() => {}); + // The reply fades in as it arrives; read and shoot it once it has landed. + await page.evaluate(() => + Promise.all( + document + .getAnimations() + .map((animation) => animation.finished), + ), + ); + const seen = await read(page); + if (shots) { + mkdirSync(shots, { recursive: true }); + await page.screenshot({ + path: join( + shots, + `maths-${engine}-${theme}-${width}-${surface}.png`, + ), + fullPage: true, + }); + } + // Every formula, by KaTeX, in KaTeX's font. + for (const formula of FORMULAS) + expect(seen.sources).toContain(formula); + expect(seen.displaySources).toHaveLength(1); + expect(seen.displaySources[0]).toContain(SET_APART); + expect(seen.katexFont).toBe(true); + expect(seen.formulaFont).toContain("KaTeX"); + // No delimiter left on screen, and the prices are prices. + expect(seen.prose).not.toContain("\\("); + expect(seen.prose).not.toContain("\\["); + expect(seen.prose).not.toContain("$$"); + expect(seen.prose).not.toContain("frac"); + expect(seen.prose).toContain( + "The loan is $25,000 and fees run $5 to $10.", + ); + expect(seen.prose).toContain("$489.15"); + // The code block, character for character, with no formula in it. + expect(seen.inCode).toBe(0); + expect(seen.code).toContain('price = "$25,000"'); + expect(seen.code).toContain('pattern = r"\\(\\d+\\)"'); + // A wide formula scrolls in its own block; the page never scrolls sideways. + expect(seen.wide).not.toBeNull(); + if (width === 390 && seen.wide) + expect(seen.wide.scroll).toBeGreaterThan(seen.wide.client); + expect(seen.pageScroll).toBeLessThanOrEqual(seen.viewport); + } finally { + await page.close(); + } + }, 60_000); + } + } + } + } +}); diff --git a/bun.lock b/bun.lock index 098c126e35..ab0068dbd8 100644 --- a/bun.lock +++ b/bun.lock @@ -31,6 +31,7 @@ "@openbot/shared": "workspace:*", "@shadcn/react": "^0.3.0", "@streamdown/code": "2.0.0", + "@streamdown/math": "1.0.3", "@tabler/icons-react": "^3.36.1", "@tanstack/react-form": "^1.33.5", "@tanstack/react-hotkeys": "^0.10.0", @@ -45,6 +46,7 @@ "clsx": "^2.1.1", "diff": "9.0.0", "embla-carousel-react": "^8.6.0", + "katex": "0.16.47", "motion": "^13.1.0", "prompt-area": "^0.6.3", "react": "^19.2.0", @@ -716,6 +718,8 @@ "@streamdown/code": ["@streamdown/code@2.0.0", "", { "dependencies": { "shiki": "^4.4.3" }, "peerDependencies": { "react": "^18.0.0 || ^19.0.0" } }, "sha512-sSyX/9gOOcivbP7204+sfAam+gotxjUNV+T4EuN3aa6i5x1LaqtB6Ln/Mxzd1XGvm0vNApO4oKLZs+7gzFI1kQ=="], + "@streamdown/math": ["@streamdown/math@1.0.3", "", { "dependencies": { "katex": "^0.16.27", "rehype-katex": "^7.0.1", "remark-math": "^6.0.0" }, "peerDependencies": { "react": "^18.0.0 || ^19.0.0" } }, "sha512-TOTRlPYs4LBNT8GaIWNCEVrcGVQsVcfc8WZ/x/0IyO+Rvsu6W+nF+YCS4Pl3JTVWQK8QX7LbWUChSilwlyzPZw=="], + "@tabler/icons": ["@tabler/icons@3.46.0", "", {}, "sha512-f2RYFl3fzPwj5WO82x6en0dmkjefxEfOm16D1ByM6cj/McNiwOkL4VaPUoP9VVIrXAD9WnTSVFr70px703b//A=="], "@tabler/icons-react": ["@tabler/icons-react@3.46.0", "", { "dependencies": { "@tabler/icons": "3.46.0" }, "peerDependencies": { "react": ">= 16" } }, "sha512-CCm7xJWhDT2PH4ZIFkP6AgYKtVhq0gpYkjUN+GVh1AzmIQaa77OW0bQPBPQiTE0PsXMR9oSxFqA3qBglzPyrVQ=="], From 507695a9a233b8e77c34e870ab48101aa1dd2919 Mon Sep 17 00:00:00 2001 From: David McKay Date: Mon, 16 Nov 2026 10:45:10 -0800 Subject: [PATCH 2/2] Wait only for animations that end before reading the maths page --- app/tests/maths-rendering.test.tsx | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/app/tests/maths-rendering.test.tsx b/app/tests/maths-rendering.test.tsx index aa18a9fc5e..f23ada5d81 100644 --- a/app/tests/maths-rendering.test.tsx +++ b/app/tests/maths-rendering.test.tsx @@ -257,13 +257,22 @@ describe.skipIf(!asked)("maths a Bot writes", () => { await page .waitForSelector(".katex-display", { timeout: 10_000 }) .catch(() => {}); - // The reply fades in as it arrives; read and shoot it once it has landed. + // The reply fades in as it arrives; read and shoot it once it has landed. Only the + // animations that end are waited for: a running step's pulse never does. await page.evaluate(() => - Promise.all( - document - .getAnimations() - .map((animation) => animation.finished), - ), + Promise.race([ + Promise.all( + document + .getAnimations() + .filter( + (animation) => + animation.effect?.getComputedTiming().endTime !== + Number.POSITIVE_INFINITY, + ) + .map((animation) => animation.finished.catch(() => {})), + ), + new Promise((done) => setTimeout(done, 3000)), + ]), ); const seen = await read(page); if (shots) {