diff --git a/packages/pro-components/chat/chat-markdown/README.en-US.md b/packages/pro-components/chat/chat-markdown/README.en-US.md index 3053806aec..b9d520a138 100644 --- a/packages/pro-components/chat/chat-markdown/README.en-US.md +++ b/packages/pro-components/chat/chat-markdown/README.en-US.md @@ -10,10 +10,10 @@ style | Object | - | CSS(Cascading Style Sheets) | N custom-style | Object | - | CSS(Cascading Style Sheets)，used to set style on virtual component | N content | String | - | required | Y options | Object | { gfm: true, pedantic: false, breaks: true } | Typescript: `TdChatContentMDOptions ` `interface TdChatContentMDOptions {gfm?: boolean; pedantic?: boolean; smartLists?: boolean; breaks?: boolean}`。[see more ts definition](https://github.com/Tencent/tdesign-miniprogram/blob/develop/packages/pro-components/chat/chat-markdown/type.ts) | N -streaming | Object | - | Typescript: `TdChatMarkdownStreamingOptions` ` interface TdChatMarkdownStreamingOptions { hasNextChunk?: boolean; tail?: boolean \| { content?: string } }`。[see more ts definition](https://github.com/Tencent/tdesign-miniprogram/blob/develop/packages/pro-components/chat/chat-markdown/type.ts) | N +streaming | Object | - | Typescript: `TdChatMarkdownStreamingOptions` ` interface TdChatMarkdownStreamingOptions { hasNextChunk?: boolean; completeSyntax?: boolean; tail?: boolean \| { content?: string } }`。[see more ts definition](https://github.com/Tencent/tdesign-miniprogram/blob/develop/packages/pro-components/chat/chat-markdown/type.ts) | N ### ChatMarkdown Events name | params | description -- | -- | -- -click | `(detail: TdMarkdownClickContext)` | [see more ts definition](https://github.com/Tencent/tdesign-miniprogram/blob/develop/packages/components/chat-markdown/type.ts)。
`interface TdMarkdownClickContext {event: TouchEvent;node: TdMarkdownNode;}`

`interface TdMarkdownNode { type: string; raw?: string; text?: string; href?: string; title?: string; tokens?: TdMarkdownNode[]; [key: string]: unknown;}`
+click | `(detail: TdMarkdownClickContext)` | [see more ts definition](https://github.com/Tencent/tdesign-miniprogram/blob/develop/packages/pro-components/chat/chat-markdown/type.ts)。
`interface TdMarkdownClickContext {event: TouchEvent;node: TdMarkdownNode;}`

`interface TdMarkdownNode { type: string; raw?: string; text?: string; href?: string; title?: string; tokens?: TdMarkdownNode[]; [key: string]: unknown;}`
diff --git a/packages/pro-components/chat/chat-markdown/README.md b/packages/pro-components/chat/chat-markdown/README.md index cdb109ac48..0217766426 100644 --- a/packages/pro-components/chat/chat-markdown/README.md +++ b/packages/pro-components/chat/chat-markdown/README.md @@ -93,10 +93,10 @@ style | Object | - | 样式 | N custom-style | Object | - | 样式，一般用于开启虚拟化组件节点场景 | N content | String | - | 必需。markdown 内容文本 | Y options | Object | { gfm: true, pedantic: false, breaks: true } | Markdown 解析器基础配置。TS 类型：`TdChatContentMDOptions ` `interface TdChatContentMDOptions {gfm?: boolean; pedantic?: boolean; smartLists?: boolean; breaks?: boolean}`。[详细类型定义](https://github.com/Tencent/tdesign-miniprogram/blob/develop/packages/pro-components/chat/chat-markdown/type.ts) | N -streaming | Object | - | 流式输出配置，控制尾部光标的显示与隐藏。尾部光标配置，true 使用默认光标 ▋，传对象可自定义光标字符。TS 类型：`TdChatMarkdownStreamingOptions` ` interface TdChatMarkdownStreamingOptions { hasNextChunk?: boolean; tail?: boolean \| { content?: string } }`。[详细类型定义](https://github.com/Tencent/tdesign-miniprogram/blob/develop/packages/pro-components/chat/chat-markdown/type.ts) | N +streaming | Object | - | 流式输出配置：`hasNextChunk` 表示是否还有后续分片（输出中为 true，结束时为 false）；`completeSyntax` 默认为 false，仅控制是否处理流式末尾未闭合的 Markdown 语法。设为 true 时，会对末尾未闭合的链接、图片、加粗、行内代码等语法进行补全或隐藏，避免暴露原始符号与 URL；`tail` 控制尾部光标，true 使用默认光标 ▋，传对象可自定义光标字符。TS 类型：`TdChatMarkdownStreamingOptions` ` interface TdChatMarkdownStreamingOptions { hasNextChunk?: boolean; completeSyntax?: boolean; tail?: boolean \| { content?: string } }`。[详细类型定义](https://github.com/Tencent/tdesign-miniprogram/blob/develop/packages/pro-components/chat/chat-markdown/type.ts) | N ### ChatMarkdown Events 名称 | 参数 | 描述 -- | -- | -- -click | `(detail: TdMarkdownClickContext)` | 点击链接时触发。[详细类型定义](https://github.com/Tencent/tdesign-miniprogram/blob/develop/packages/components/chat-markdown/type.ts)。
`interface TdMarkdownClickContext {event: TouchEvent;node: TdMarkdownNode;}`

`interface TdMarkdownNode { type: string; raw?: string; text?: string; href?: string; title?: string; tokens?: TdMarkdownNode[]; [key: string]: unknown;}`
+click | `(detail: TdMarkdownClickContext)` | 点击链接时触发。[详细类型定义](https://github.com/Tencent/tdesign-miniprogram/blob/develop/packages/pro-components/chat/chat-markdown/type.ts)。
`interface TdMarkdownClickContext {event: TouchEvent;node: TdMarkdownNode;}`

`interface TdMarkdownNode { type: string; raw?: string; text?: string; href?: string; title?: string; tokens?: TdMarkdownNode[]; [key: string]: unknown;}`
diff --git a/packages/pro-components/chat/chat-markdown/__test__/index.test.js b/packages/pro-components/chat/chat-markdown/__test__/index.test.js new file mode 100644 index 0000000000..1e28546737 --- /dev/null +++ b/packages/pro-components/chat/chat-markdown/__test__/index.test.js @@ -0,0 +1,83 @@ +import path from 'path'; +import simulate from 'miniprogram-simulate'; + +jest.mock('marked', () => ({ + Lexer: class { + lex(source) { + const marker = '\uE000TD_CHAT_CODE_0\uE001'; + if (source.includes(marker)) { + return [ + { + type: 'paragraph', + raw: source, + text: source, + tokens: [ + { + type: 'strong', + raw: `**${marker}**`, + text: marker, + tokens: [{ type: 'text', raw: marker, text: marker }], + }, + ], + }, + ]; + } + return [ + { + type: 'paragraph', + raw: source, + text: source, + tokens: [ + { type: 'text', raw: 'a**', text: 'a**', escaped: false }, + { type: 'codespan', raw: '`code`', text: 'code' }, + { type: 'text', raw: '**', text: '**', escaped: false }, + ], + }, + ]; + } + }, +})); + +const chatMarkdown = load(path.resolve(__dirname, '../chat-markdown')); +const content = 'a**`code`**'; + +function render(streaming) { + const id = simulate.load({ + template: '', + data: { content, streaming }, + usingComponents: { 't-chat-markdown': chatMarkdown }, + }); + const container = simulate.render(id); + container.attach(document.createElement('parent-wrapper')); + return container.querySelector('#markdown'); +} + +function getStrongCodeToken(component) { + const strong = component.data.nodes[0].tokens.find((token) => token.type === 'strong'); + return strong?.tokens?.[0]; +} + +describe('ChatMarkdown compatibility', () => { + it.each([undefined, { hasNextChunk: true, tail: false }, { hasNextChunk: true, tail: false, completeSyntax: false }])( + '无论是否开启 completeSyntax 都修正强调语法中的行内代码：%p', + (streaming) => { + expect(getStrongCodeToken(render(streaming))).toBeTruthy(); + }, + ); + + it('无 streaming 的无动画模式正常渲染粗体', () => { + expect(getStrongCodeToken(render(undefined))).toEqual({ type: 'codespan', raw: '`code`', text: 'code' }); + }); + + it('光标注入粗体中的最后一个 codespan', () => { + expect(getStrongCodeToken(render({ hasNextChunk: true, tail: { content: '...' }, completeSyntax: false }))).toEqual( + { + type: 'codespan', + raw: '`code`', + text: 'code', + isTail: true, + tailContent: '...', + }, + ); + }); +}); diff --git a/packages/pro-components/chat/chat-markdown/__test__/stream-syntax.test.js b/packages/pro-components/chat/chat-markdown/__test__/stream-syntax.test.js new file mode 100644 index 0000000000..1b7ddc0688 --- /dev/null +++ b/packages/pro-components/chat/chat-markdown/__test__/stream-syntax.test.js @@ -0,0 +1,58 @@ +import completeUnclosedInlineSyntax from '../utils/stream-syntax'; + +describe('completeUnclosedInlineSyntax', () => { + it.each([ + ['[text', '[text]()'], + ['[text]', '[text]()'], + ['[text](url', '[text](url)'], + ['![alt', ''], + ['![alt]', ''], + ['![alt](url', ''], + ['![alt][ref', ''], + ])('补全或隐藏未闭合的链接和图片：%s', (source, expected) => { + expect(completeUnclosedInlineSyntax(source)).toBe(expected); + }); + + it.each([String.raw`\[literal`, String.raw`\![alt`, String.raw`\!`])('不处理转义语法：%s', (source) => { + expect(completeUnclosedInlineSyntax(source)).toBe(source); + }); + + it.each([ + ['!', ''], + ['text !', 'text '], + ['你好!', '你好!'], + ])('流式阶段区分图片起始符与普通末尾感叹号：%s', (source, expected) => { + expect(completeUnclosedInlineSyntax(source)).toBe(expected); + }); + + it.each(['`!`', '`[text`', '`![alt`', '``**code``'])('不处理行内代码中的语法：%s', (source) => { + expect(completeUnclosedInlineSyntax(source)).toBe(source); + }); + + it.each([ + ['[inline](https://example.com)', {}], + ['![inline](https://example.com/image.png)', {}], + ['[id]', { id: { href: 'https://example.com' } }], + ['![id]', { id: { href: 'https://example.com/image.png' } }], + ['[text][id]', { id: { href: 'https://example.com' } }], + ['![alt][id]', { id: { href: 'https://example.com/image.png' } }], + ['[text][pending]', {}], + ['![alt][pending]', {}], + ])('不修改完整内联语法和引用式语法：%s', (source, links) => { + expect(completeUnclosedInlineSyntax(source, links)).toBe(source); + }); + + it.each([ + ['`code', '`code`'], + ['``code', '``code``'], + ['``code`', '``code``'], + ['`code``', '`code`` `'], + ])('使用等长反引号补全行内代码：%s', (source, expected) => { + expect(completeUnclosedInlineSyntax(source)).toBe(expected); + }); + + it('忽略已闭合代码围栏中的反引号', () => { + const source = ['```js', 'const value = `code`;', '```', 'plain text'].join('\n'); + expect(completeUnclosedInlineSyntax(source)).toBe(source); + }); +}); diff --git a/packages/pro-components/chat/chat-markdown/__test__/tail-cursor.test.js b/packages/pro-components/chat/chat-markdown/__test__/tail-cursor.test.js new file mode 100644 index 0000000000..b818a71e14 --- /dev/null +++ b/packages/pro-components/chat/chat-markdown/__test__/tail-cursor.test.js @@ -0,0 +1,59 @@ +import { injectTailToTokens, resolveTailContent } from '../utils/tail-cursor'; + +describe('tail cursor', () => { + it('解析默认及自定义光标内容', () => { + expect(resolveTailContent()).toBeNull(); + expect(resolveTailContent(false)).toBeNull(); + expect(resolveTailContent(true)).toBe('▋'); + expect(resolveTailContent({ content: '' })).toBe('▋'); + expect(resolveTailContent({ content: '...' })).toBe('...'); + }); + + it('将光标注入最后一个可见的 codespan', () => { + const tokens = [ + { type: 'text', raw: 'prefix ', text: 'prefix ' }, + { type: 'codespan', raw: '`code`', text: 'code' }, + ]; + + expect(injectTailToTokens(tokens, '...')).toBe(true); + expect(tokens).toEqual([ + { type: 'text', raw: 'prefix ', text: 'prefix ' }, + { type: 'codespan', raw: '`code`', text: 'code', isTail: true, tailContent: '...' }, + ]); + }); + + it('将列表 text 包装节点的光标注入可见子节点', () => { + const tokens = [ + { + type: 'text', + raw: 'item', + text: 'item', + tokens: [{ type: 'text', raw: 'item', text: 'item' }], + }, + ]; + + expect(injectTailToTokens(tokens, '...')).toBe(true); + expect(tokens[0].isTail).toBeUndefined(); + expect(tokens[0].tokens[0]).toMatchObject({ isTail: true, tailContent: '...' }); + }); + + it('第三个参数继续作为递归深度使用', () => { + const tokens = [ + { + type: 'paragraph', + tokens: [{ type: 'text', raw: 'nested', text: 'nested' }], + }, + ]; + + expect(injectTailToTokens(tokens, '...', 2)).toBe(true); + expect(tokens[0].tokens[0]).toMatchObject({ isTail: true, tailContent: '...' }); + }); + + it('将光标注入代码块', () => { + const tokens = [{ type: 'code', raw: 'code', text: 'code' }]; + + injectTailToTokens(tokens, '...'); + + expect(tokens[0]).toEqual({ type: 'code', raw: 'code', text: 'code', isTail: true, tailContent: '...' }); + }); +}); diff --git a/packages/pro-components/chat/chat-markdown/_example/tail/index.js b/packages/pro-components/chat/chat-markdown/_example/tail/index.js index e961051323..db8f42df2d 100644 --- a/packages/pro-components/chat/chat-markdown/_example/tail/index.js +++ b/packages/pro-components/chat/chat-markdown/_example/tail/index.js @@ -13,7 +13,7 @@ Page({ ], selectedSegment: -1, // 初始不选中任何分段 content: '', - streaming: { hasNextChunk: false, tail: true }, + streaming: { hasNextChunk: false, tail: true, completeSyntax: true }, }, onLoad() { @@ -51,7 +51,7 @@ Page({ this.setData({ selectedSegment: segmentValue, - streaming: { hasNextChunk: true, tail: true }, + streaming: { hasNextChunk: true, tail: true, completeSyntax: true }, }); this._timer = setInterval(() => { @@ -59,7 +59,7 @@ Page({ const isDone = index >= markdownData.length; this.setData({ content: markdownData.slice(0, index), - streaming: { hasNextChunk: !isDone, tail: true }, + streaming: { hasNextChunk: !isDone, tail: true, completeSyntax: true }, }); if (isDone) { clearInterval(this._timer); diff --git a/packages/pro-components/chat/chat-markdown/chat-markdown-code/chat-markdown-code.wxml b/packages/pro-components/chat/chat-markdown/chat-markdown-code/chat-markdown-code.wxml index 20068babdd..ea3a1e6fd9 100644 --- a/packages/pro-components/chat/chat-markdown/chat-markdown-code/chat-markdown-code.wxml +++ b/packages/pro-components/chat/chat-markdown/chat-markdown-code/chat-markdown-code.wxml @@ -6,6 +6,7 @@ - {{node.text}} + {{node.text}} diff --git a/packages/pro-components/chat/chat-markdown/chat-markdown-node/chat-markdown-node.wxml b/packages/pro-components/chat/chat-markdown/chat-markdown-node/chat-markdown-node.wxml index c13664169b..5221dc8fda 100644 --- a/packages/pro-components/chat/chat-markdown/chat-markdown-node/chat-markdown-node.wxml +++ b/packages/pro-components/chat/chat-markdown/chat-markdown-node/chat-markdown-node.wxml @@ -55,7 +55,9 @@ - {{''+item.raw+''}} + + {{''+item.raw+''}} + @@ -112,6 +114,7 @@ > {{''+(item.text||item.raw)+''}} + { - if (item.tokens?.length) result.push(...item.tokens); - return result; - }, []); -} - -/** - * 从后往前遍历 token 树，找到最后一个非空 text 叶子节点，打上 isTail 标记。 - * - 有子节点（tokens / items）时优先递归 - * - 末尾是 code / table / image 等非 text 节点时静默跳过，不注入 - * @returns 是否成功注入 - */ -function injectTailToTokens(tokens: any[], tailChar: string, depth = 0): boolean { - for (let i = tokens.length - 1; i >= 0; i -= 1) { - const token = tokens[i]; - - // code 块作为叶子节点，直接注入，不递归 - if (token.type === 'code' && (token.text || token.raw)?.trim()) { - token.isTail = true; - token.tailContent = tailChar; - return true; - } - - // 叶子文本节点且内容非空 - if (token.type === 'text' && (token.text || token.raw)?.trim()) { - token.isTail = true; - token.tailContent = tailChar; - return true; - } - - // table 节点：从后往前遍历 rows，再遍历 header - if (token.type === 'table') { - const allRows: any[][] = [...(token.header ? [token.header] : []), ...(token.rows || [])]; - for (let r = allRows.length - 1; r >= 0; r -= 1) { - const row = allRows[r]; - for (let c = row.length - 1; c >= 0; c -= 1) { - const cell = row[c]; - if (cell.tokens?.length) { - if (injectTailToTokens(cell.tokens, tailChar, depth + 1)) return true; - } - } - } - } else { - // 有子节点时递归 - let children: any[] | null = null; - if (token.tokens?.length) { - children = token.tokens; - } else if (token.items?.length) { - children = flatListItems(token.items); - } - if (children?.length) { - if (injectTailToTokens(children, tailChar, depth + 1)) return true; - } - } - } - return false; -} - export interface ChatMarkdownProps extends TdChatMarkdownProps {} @wxComponent() @@ -110,11 +41,23 @@ export default class ChatMarkdown extends SuperComponent { // 解析markdown文本 parseMarkdown(markdown: string) { try { + const { streaming } = this.data; + // 流式输出时对末尾未闭合语法做补全/隐藏（需显式开启 streaming.completeSyntax） + const syntaxCompletionEnabled = streaming?.completeSyntax === true; + const shouldComplete = streaming?.hasNextChunk && syntaxCompletionEnabled; const lexer = new Lexer(this.data.options); - const tokens = lexer.lex(markdown); + const originalTokens = lexer.lex(markdown); + const source = shouldComplete ? completeUnclosedInlineSyntax(markdown, originalTokens.links) : markdown; + let tokens = source === markdown ? originalTokens : new Lexer(this.data.options).lex(source); + + // 修正 marked 对“强调语法内包含行内代码”场景的 token 泄漏；与流式语法补全开关无关。 + if (hasLeakedInlineFormatting(tokens)) { + const { source: maskedSource, masks } = maskInlineCodes(source); + tokens = new Lexer(this.data.options).lex(maskedSource); + restoreMaskedInlineCodes(tokens, masks); + } // 尾部光标注入 - const { streaming } = this.data; const tailChar = resolveTailContent(streaming?.tail); if (streaming?.hasNextChunk && tailChar) { injectTailToTokens(tokens, tailChar); diff --git a/packages/pro-components/chat/chat-markdown/props.ts b/packages/pro-components/chat/chat-markdown/props.ts index 115c234c86..7dbca3980d 100644 --- a/packages/pro-components/chat/chat-markdown/props.ts +++ b/packages/pro-components/chat/chat-markdown/props.ts @@ -17,7 +17,7 @@ const props: TdChatMarkdownProps = { type: Object, value: { gfm: true, pedantic: false, breaks: true }, }, - /** 流式输出配置，控制尾部光标的显示与隐藏。尾部光标配置，true 使用默认光标 ▋，传对象可自定义光标字符 */ + /** 流式输出配置：`hasNextChunk` 表示是否还有后续分片（输出中为 true，结束时为 false）；`completeSyntax` 默认为 false，仅控制是否处理流式末尾未闭合的 Markdown 语法。设为 true 时，会对末尾未闭合的链接、图片、加粗、行内代码等语法进行补全或隐藏，避免暴露原始符号与 URL；`tail` 控制尾部光标，true 使用默认光标 ▋，传对象可自定义光标字符 */ streaming: { type: Object, }, diff --git a/packages/pro-components/chat/chat-markdown/type.ts b/packages/pro-components/chat/chat-markdown/type.ts index 309028baea..1936b33269 100644 --- a/packages/pro-components/chat/chat-markdown/type.ts +++ b/packages/pro-components/chat/chat-markdown/type.ts @@ -23,7 +23,7 @@ export interface TdChatMarkdownProps { value?: TdChatContentMDOptions; }; /** - * 流式输出配置，控制尾部光标的显示与隐藏。尾部光标配置，true 使用默认光标 ▋，传对象可自定义光标字符 + * 流式输出配置：`hasNextChunk` 表示是否还有后续分片（输出中为 true，结束时为 false）；`completeSyntax` 默认为 false，仅控制是否处理流式末尾未闭合的 Markdown 语法。设为 true 时，会对末尾未闭合的链接、图片、加粗、行内代码等语法进行补全或隐藏，避免暴露原始符号与 URL；`tail` 控制尾部光标，true 使用默认光标 ▋，传对象可自定义光标字符 */ streaming?: { type: ObjectConstructor; @@ -40,5 +40,6 @@ export interface TdChatContentMDOptions { export interface TdChatMarkdownStreamingOptions { hasNextChunk?: boolean; + completeSyntax?: boolean; tail?: boolean | { content?: string }; } diff --git a/packages/pro-components/chat/chat-markdown/utils/marked-fallback.ts b/packages/pro-components/chat/chat-markdown/utils/marked-fallback.ts new file mode 100644 index 0000000000..6b3d3bc744 --- /dev/null +++ b/packages/pro-components/chat/chat-markdown/utils/marked-fallback.ts @@ -0,0 +1,203 @@ +interface InlineCodeMask { + marker: string; + raw: string; + text: string; +} + +type InlineCodeMaskMatch = { mask: InlineCodeMask; index: number }; + +/** 跳过 fenced code block。 */ +function getCodeFenceRanges(markdown: string): Array<[number, number]> { + const ranges: Array<[number, number]> = []; + let fence: { marker: string; start: number } | null = null; + let lineStart = 0; + + while (lineStart < markdown.length) { + const newlineIndex = markdown.indexOf('\n', lineStart); + const lineEnd = newlineIndex === -1 ? markdown.length : newlineIndex + 1; + const line = markdown.slice(lineStart, newlineIndex === -1 ? lineEnd : newlineIndex); + const matchedFence = line.match(/^ {0,3}(`{3,}|~{3,})(.*)$/); + + if (fence) { + if ( + matchedFence && + matchedFence[1][0] === fence.marker[0] && + matchedFence[1].length >= fence.marker.length && + /^[ \t]*$/.test(matchedFence[2]) + ) { + ranges.push([fence.start, lineEnd]); + fence = null; + } + } else if (matchedFence) { + const [, marker, info] = matchedFence; + if (marker[0] === '~' || !info.includes('`')) fence = { marker, start: lineStart }; + } + + lineStart = lineEnd; + } + + if (fence) ranges.push([fence.start, markdown.length]); + return ranges; +} + +function isEscaped(markdown: string, index: number): boolean { + let slashCount = 0; + for (let i = index - 1; i >= 0 && markdown[i] === '\\'; i -= 1) slashCount += 1; + return slashCount % 2 === 1; +} + +function normalizeCodeSpanText(text: string): string { + const normalized = text.replace(/[\r\n]+/g, ' '); + return /^\s/.test(normalized) && /\s$/.test(normalized) && /\S/.test(normalized) + ? normalized.slice(1, -1) + : normalized; +} + +/** marked 嵌套行内代码解析失败时的回退。 */ +export function maskInlineCodes(markdown: string): { source: string; masks: InlineCodeMask[] } { + const masks: InlineCodeMask[] = []; + const fenceRanges = getCodeFenceRanges(markdown); + let fenceIndex = 0; + let source = ''; + let index = 0; + + while (index < markdown.length) { + const range = fenceRanges[fenceIndex]; + if (range && index === range[0]) { + const [rangeStart, rangeEnd] = range; + source += markdown.slice(rangeStart, rangeEnd); + index = rangeEnd; + fenceIndex += 1; + } else if (markdown[index] !== '`' || isEscaped(markdown, index)) { + source += markdown[index]; + index += 1; + } else { + let delimiterEnd = index; + while (markdown[delimiterEnd] === '`') delimiterEnd += 1; + const delimiterLength = delimiterEnd - index; + const nextFenceStart = fenceRanges[fenceIndex]?.[0] ?? markdown.length; + let closingStart = delimiterEnd; + + while (closingStart < nextFenceStart) { + if (markdown[closingStart] === '`') { + let closingEnd = closingStart; + while (markdown[closingEnd] === '`') closingEnd += 1; + if (closingEnd - closingStart === delimiterLength) break; + closingStart = closingEnd; + } else { + closingStart += 1; + } + } + + if (closingStart === nextFenceStart) { + source += markdown.slice(index, delimiterEnd); + index = delimiterEnd; + } else { + const closingEnd = closingStart + delimiterLength; + const raw = markdown.slice(index, closingEnd); + const marker = `\uE000TD_CHAT_CODE_${masks.length}\uE001`; + masks.push({ marker, raw, text: normalizeCodeSpanText(markdown.slice(delimiterEnd, closingStart)) }); + source += marker; + index = closingEnd; + } + } + } + + return { source, masks }; +} + +export function restoreMaskedInlineCodes(tokens: any[], masks: InlineCodeMask[]) { + if (!masks.length) return; + + const findMask = (text: string, from: number): InlineCodeMaskMatch | null => { + let matchedMask: InlineCodeMask | null = null; + let matchedIndex = -1; + for (let i = 0; i < masks.length; i += 1) { + const mask = masks[i]; + const index = text.indexOf(mask.marker, from); + if (index !== -1 && (matchedIndex === -1 || index < matchedIndex)) { + matchedMask = mask; + matchedIndex = index; + } + } + return matchedMask ? { mask: matchedMask, index: matchedIndex } : null; + }; + const restoreString = (value: unknown): unknown => { + if (typeof value !== 'string') return value; + return masks.reduce((result, mask) => result.split(mask.marker).join(mask.raw), value); + }; + + const restore = (items: any[]): void => { + for (let i = 0; i < items.length; i += 1) { + const token = items[i]; + const raw = typeof token.raw === 'string' ? token.raw : token.text; + const firstMask = token.type === 'text' && typeof raw === 'string' ? findMask(raw, 0) : null; + if (firstMask) { + const restored: any[] = []; + let offset = 0; + let match: InlineCodeMaskMatch | null = firstMask; + while (match !== null) { + const current: InlineCodeMaskMatch = match; + if (current.index > offset) { + restored.push({ + type: 'text', + raw: raw.slice(offset, current.index), + text: raw.slice(offset, current.index), + }); + } + restored.push({ type: 'codespan', raw: current.mask.raw, text: current.mask.text }); + offset = current.index + current.mask.marker.length; + match = findMask(raw, offset); + } + if (offset < raw.length) restored.push({ type: 'text', raw: raw.slice(offset), text: raw.slice(offset) }); + if (token.isTail && restored.length) { + const last = restored[restored.length - 1]; + last.isTail = true; + last.tailContent = token.tailContent; + } + items.splice(i, 1, ...restored); + i += restored.length - 1; + } else { + token.raw = restoreString(token.raw); + token.text = restoreString(token.text); + if (token.tokens?.length) restore(token.tokens); + if (token.items?.length) token.items.forEach((item: any) => item.tokens?.length && restore(item.tokens)); + if (token.type === 'table') { + const rows = [...(token.header ? [token.header] : []), ...(token.rows || [])]; + rows.forEach((row: any[]) => row.forEach((cell: any) => cell.tokens?.length && restore(cell.tokens))); + } + } + } + }; + + restore(tokens); +} + +/** 检测 marked 未解析的嵌套格式符。 */ +export function hasLeakedInlineFormatting(tokens: any[]): boolean { + const containsLeakedFormatting = (items: any[]): boolean => { + const text = items + .filter((token) => token.type === 'text') + .map((token) => token.raw || token.text || '') + .join(''); + const hasCodeSpan = items.some((token) => token.type === 'codespan'); + const hasFormatting = /(\*{1,3})[\s\S]*\*{1,3}|_{1,3}[\s\S]*_{1,3}|~~[\s\S]*~~|\[[\s\S]*\]\([^)]*\)/.test(text); + + if (hasCodeSpan && hasFormatting) return true; + + return items.some((token): boolean => { + if (token.tokens?.length && containsLeakedFormatting(token.tokens)) return true; + if (token.items?.some((item: any): boolean => !!item.tokens?.length && containsLeakedFormatting(item.tokens))) + return true; + if (token.type === 'table') { + const rows = [...(token.header ? [token.header] : []), ...(token.rows || [])]; + return rows.some((row: any[]): boolean => + row.some((cell: any): boolean => !!cell.tokens?.length && containsLeakedFormatting(cell.tokens)), + ); + } + return false; + }); + }; + + return containsLeakedFormatting(tokens); +} diff --git a/packages/pro-components/chat/chat-markdown/utils/stream-syntax.ts b/packages/pro-components/chat/chat-markdown/utils/stream-syntax.ts new file mode 100644 index 0000000000..6ac18f06e7 --- /dev/null +++ b/packages/pro-components/chat/chat-markdown/utils/stream-syntax.ts @@ -0,0 +1,254 @@ +/** + * 流式输出时的未闭合 markdown 语法处理。 + * + * 大模型流式输出时，文本会逐字符到达，末尾常出现未输完的语法片段（如 + * `[text`、`![alt`、`**bold`、`` `code ``）。若直接交给 marked 解析，会暴露原始 + * 符号与 URL，造成闪烁与光标跳动。此模块在解析前对末尾未闭合语法做补全或隐藏， + * 待语法闭合后再正常渲染。 + */ + +/** 按 Markdown 围栏规则判断是否存在未闭合代码块。 */ +function hasUnclosedCodeFence(markdown: string): boolean { + let fence: string | null = null; + + markdown.split('\n').forEach((line) => { + const matchedFence = line.match(/^ {0,3}(`{3,}|~{3,})(.*)$/); + if (!matchedFence) return; + + const [, marker, info] = matchedFence; + if (fence) { + if (marker[0] === fence[0] && marker.length >= fence.length && /^[ \t]*$/.test(info)) fence = null; + return; + } + + if (marker[0] === '~' || !info.includes('`')) fence = marker; + }); + + return !!fence; +} + +/** 隐藏未输完的 1~2 位代码围栏标记。 */ +function hidePartialFence(markdown: string): string { + return markdown.replace(/(^|\n)[ \t]*[`~]{1,2}[ \t]*$/, '$1'); +} + +function isEscaped(markdown: string, index: number): boolean { + let slashCount = 0; + for (let i = index - 1; i >= 0 && markdown[i] === '\\'; i -= 1) slashCount += 1; + return slashCount % 2 === 1; +} + +interface BacktickRun { + index: number; + length: number; +} + +/** 收集 fenced code block 外的未转义反引号片段。 */ +function getInlineCodeRuns(markdown: string): BacktickRun[] { + const runs: BacktickRun[] = []; + let fence: string | null = null; + let lineStart = 0; + + while (lineStart < markdown.length) { + const newlineIndex = markdown.indexOf('\n', lineStart); + const lineEnd = newlineIndex === -1 ? markdown.length : newlineIndex; + const line = markdown.slice(lineStart, lineEnd); + const matchedFence = line.match(/^ {0,3}(`{3,}|~{3,})(.*)$/); + + if (fence) { + if ( + matchedFence && + matchedFence[1][0] === fence[0] && + matchedFence[1].length >= fence.length && + /^[ \t]*$/.test(matchedFence[2]) + ) { + fence = null; + } + } else if (matchedFence && (matchedFence[1][0] === '~' || !matchedFence[2].includes('`'))) { + [, fence] = matchedFence; + } else { + const re = /`+/g; + for (let match = re.exec(line); match; match = re.exec(line)) { + const index = lineStart + match.index; + if (!isEscaped(markdown, index)) runs.push({ index, length: match[0].length }); + } + } + + if (newlineIndex === -1) break; + lineStart = newlineIndex + 1; + } + + return runs; +} + +/** 将已闭合行内代码替换为等长空格，避免其中的 Markdown 字面量参与后续语法检测。 */ +function getInlineSyntaxScanSource(markdown: string): string { + const chars = markdown.split(''); + let opening: BacktickRun | null = null; + + const runs = getInlineCodeRuns(markdown); + for (let r = 0; r < runs.length; r += 1) { + const run = runs[r]; + if (!opening) { + opening = run; + } else if (run.length === opening.length) { + for (let i = opening.index; i < run.index + run.length; i += 1) chars[i] = ' '; + opening = null; + } + } + + return chars.join(''); +} + +type ReferenceLinks = Record; + +function normalizeReferenceLabel(label: string): string { + return label.trim().replace(/\s+/g, ' ').toLowerCase(); +} + +function hasReference(referenceLinks: ReferenceLinks, label: string): boolean { + return Object.prototype.hasOwnProperty.call(referenceLinks, normalizeReferenceLabel(label)); +} + +/** 隐藏末尾可能发展为图片的孤立 !，下一分片非 [ 或流结束时会由原文恢复。 */ +function hideImageMarker(markdown: string): string | null { + const scanSource = getInlineSyntaxScanSource(markdown); + const index = scanSource.length - 1; + if ( + index < 0 || + scanSource[index] !== '!' || + isEscaped(markdown, index) || + (index > 0 && !/\s/.test(scanSource[index - 1])) + ) + return null; + return markdown.slice(0, index); +} + +/** 隐藏末尾未闭合的图片语法；完整内联图片和已定义的引用式图片交给 marked。 */ +function closeImage(markdown: string, referenceLinks: ReferenceLinks): string | null { + const scanSource = getInlineSyntaxScanSource(markdown); + const reference = scanSource.match(/!\[([^\]\n]*)\]\[([^\]\n]*)\]$/); + if (reference?.index !== undefined && !isEscaped(markdown, reference.index)) return null; + + const openReference = scanSource.match(/!\[([^\]\n]*)\]\[[^\]\n]*$/); + if (openReference?.index !== undefined && !isEscaped(markdown, openReference.index)) { + return markdown.slice(0, openReference.index); + } + + const openAlt = scanSource.match(/!\[([^\]\n]*)$/); + const openDestination = scanSource.match(/!\[([^\]\n]*)\]\([^)\n]*$/); + const shortcutReference = scanSource.match(/!\[([^\]\n]*)\]$/); + const match = openAlt || openDestination || shortcutReference; + + if (!match || match.index === undefined || isEscaped(markdown, match.index)) return null; + if (shortcutReference && hasReference(referenceLinks, shortcutReference[1])) return null; + return markdown.slice(0, match.index); +} + +function isLinkCandidate(markdown: string, index: number): boolean { + if (isEscaped(markdown, index) || markdown[index - 1] === '!') return false; + const prefix = markdown.slice(0, index); + if (/!\[[^\]\n]*\]$/.test(prefix)) return false; + return !/(^|\n)[ \t]*[-+*][ \t]+$/.test(prefix); +} + +/** 补全末尾未闭合的链接语法；完整链接和已定义的引用式链接交给 marked。 */ +function closeLink(markdown: string, referenceLinks: ReferenceLinks): string | null { + const scanSource = getInlineSyntaxScanSource(markdown); + const reference = scanSource.match(/\[([^\]\n]*)\]\[([^\]\n]*)\]$/); + if (reference?.index !== undefined && isLinkCandidate(markdown, reference.index)) return null; + + const openDestination = scanSource.match(/\[([^\]\n]*)\]\([^)\n]*$/); + if (openDestination?.index !== undefined && isLinkCandidate(markdown, openDestination.index)) { + return `${markdown})`; + } + + const shortcutReference = scanSource.match(/\[([^\]\n]*)\]$/); + if (shortcutReference?.index !== undefined && isLinkCandidate(markdown, shortcutReference.index)) { + return hasReference(referenceLinks, shortcutReference[1]) ? null : `${markdown}()`; + } + + const openLabel = scanSource.match(/\[([^\]\n]*)$/); + if (openLabel?.index !== undefined && isLinkCandidate(markdown, openLabel.index)) return `${markdown}]()`; + return null; +} + +/** 补全末尾未闭合的行内代码；闭合符必须与起始反引号长度一致。 */ +function closeInlineCode(markdown: string): string | null { + const runs = getInlineCodeRuns(markdown); + let opening: BacktickRun | null = null; + for (let i = 0; i < runs.length; i += 1) { + const run = runs[i]; + if (!opening) opening = run; + else if (run.length === opening.length) opening = null; + } + + if (!opening) return null; + if (opening.index + opening.length === markdown.length) return markdown.slice(0, opening.index); + + const lastRun = runs[runs.length - 1]; + if (lastRun !== opening && lastRun.index + lastRun.length === markdown.length) { + if (lastRun.length < opening.length) return `${markdown}${'`'.repeat(opening.length - lastRun.length)}`; + return `${markdown} ${'`'.repeat(opening.length)}`; + } + return `${markdown}${'`'.repeat(opening.length)}`; +} + +/** + * 补全最后一行未配对的加粗/斜体（`*` `**` `***`）；无需处理时返回 null。 + * 仅处理 `*` 系语法，`_`、`~~` 未闭合时保持字面量交给 marked 常规解析（成对才渲染）。 + */ +function closeEmphasis(markdown: string): string | null { + const lastLine = markdown.slice(markdown.lastIndexOf('\n') + 1); + if (!lastLine || (lastLine.match(/`/g) || []).length % 2 === 1) return null; + + const line = lastLine.replace(/^\s*[*+-]\s+/, ''); + const trailingMatch = line.match(/(\*)\1*$/); + const trailing = trailingMatch ? trailingMatch[0] : ''; + const scanLine = trailingMatch ? line.slice(0, trailingMatch.index) : line; + + const hasBody = !!scanLine.replace(/[\s*`]/g, ''); + if (trailing && !hasBody) return markdown.slice(0, markdown.length - lastLine.length); + + const urlLess = getInlineSyntaxScanSource(scanLine).replace(/(?:https?|ftp):\/\/\S+/g, ' '); + const stack: string[] = []; + const re = /(\*{1,3})/g; + for (let m = re.exec(urlLess); m; m = re.exec(urlLess)) { + const escaped = m.index > 0 && urlLess[m.index - 1] === '\\'; + if (!escaped) { + if (stack[stack.length - 1] === m[0]) stack.pop(); + else stack.push(m[0]); + } + } + + const top = stack[stack.length - 1]; + const trimmed = markdown.replace(/[ \t]+$/, ''); + if (trailing && !top) return markdown.slice(0, markdown.length - trailing.length); + if (trailing && top && trailing[0] === top[0]) { + if (trailing.length === top.length) stack.pop(); + else if (trailing.length < top.length) return `${trimmed}${top.slice(trailing.length)}`; + } + return stack.length ? `${trimmed}${stack.reverse().join('')}` : null; +} + +/** 流式输出时补全/隐藏文本末尾未闭合的语法，避免原始符号与 URL 闪现 */ +export default function completeUnclosedInlineSyntax(markdown: string, referenceLinks: ReferenceLinks = {}): string { + // 未闭合代码块：仅隐藏未输完的围栏，块内内容交给 marked 渲染 + if (hasUnclosedCodeFence(markdown)) return hidePartialFence(markdown); + + // 行首未输完的起始围栏（1~2 个反引号）：先隐藏避免闪出 + const hiddenFence = hidePartialFence(markdown); + if (hiddenFence !== markdown) return hiddenFence; + + // 先补齐行内代码，避免其内部的 [text] / ![alt] 被误判为链接或图片语法。 + const codeClosed = closeInlineCode(markdown) ?? markdown; + + // 图片起始符、图片、链接命中即返回（整段隐藏或补全，不与强调叠加）。 + const imageMarkerHidden = hideImageMarker(codeClosed); + if (imageMarkerHidden !== null) return imageMarkerHidden; + + const closed = closeImage(codeClosed, referenceLinks) ?? closeLink(codeClosed, referenceLinks); + if (closed !== null) return closed; + + return closeEmphasis(codeClosed) ?? codeClosed; +} diff --git a/packages/pro-components/chat/chat-markdown/utils/tail-cursor.ts b/packages/pro-components/chat/chat-markdown/utils/tail-cursor.ts new file mode 100644 index 0000000000..37c3f0bb81 --- /dev/null +++ b/packages/pro-components/chat/chat-markdown/utils/tail-cursor.ts @@ -0,0 +1,82 @@ +/** + * 尾部光标（打字光标）注入。 + * + * 流式输出未结束时，需要在已渲染内容的末尾显示一个闪烁光标（如 ▋）。 + * 本模块负责解析光标字符，并在 marked 生成的 token 树中定位最后一个可见的 + * 文本/代码叶子节点，为其打上标记，供渲染层追加光标。 + */ + +const DEFAULT_TAIL_CONTENT = '▋'; + +/** 解析 tail 参数，返回光标字符；不需要显示时返回 null */ +export function resolveTailContent(tail?: boolean | { content?: string }): string | null { + if (!tail) return null; + if (typeof tail === 'boolean') return DEFAULT_TAIL_CONTENT; + return tail.content || DEFAULT_TAIL_CONTENT; +} + +/** + * 将列表项的子 tokens 展平，供 injectTailToTokens 递归使用。 + * marked 的 list token 结构：list.items[].tokens（而非 list.tokens） + */ +function flatListItems(items: any[]): any[] { + return items.reduce((result: any[], item: any) => { + if (item.tokens?.length) result.push(...item.tokens); + return result; + }, []); +} + +/** + * 从后往前遍历 token 树，找到最后一个非空文本或代码叶子节点，打上 isTail 标记。 + * @returns 是否成功注入 + */ +export function injectTailToTokens(tokens: any[], tailChar: string, depth = 0): boolean { + for (let i = tokens.length - 1; i >= 0; i -= 1) { + const token = tokens[i]; + + // code 块作为叶子节点，直接注入，不递归 + if (token.type === 'code' && (token.text || token.raw)?.trim()) { + token.isTail = true; + token.tailContent = tailChar; + return true; + } + + // text 包装节点需要优先进入子 tokens，确保光标标记在实际渲染的文本之后。 + if (token.type === 'text' && token.tokens?.length) { + if (injectTailToTokens(token.tokens, tailChar, depth + 1)) return true; + } + + // 普通文本和行内代码都是可见叶子节点，光标应位于其后。 + if (['text', 'codespan'].includes(token.type) && (token.text || token.raw)?.trim()) { + token.isTail = true; + token.tailContent = tailChar; + return true; + } + + // table 节点：从后往前遍历 rows，再遍历 header + if (token.type === 'table') { + const allRows: any[][] = [...(token.header ? [token.header] : []), ...(token.rows || [])]; + for (let r = allRows.length - 1; r >= 0; r -= 1) { + const row = allRows[r]; + for (let c = row.length - 1; c >= 0; c -= 1) { + const cell = row[c]; + if (cell.tokens?.length) { + if (injectTailToTokens(cell.tokens, tailChar, depth + 1)) return true; + } + } + } + } else { + // 有子节点时递归 + let children: any[] | null = null; + if (token.tokens?.length) { + children = token.tokens; + } else if (token.items?.length) { + children = flatListItems(token.items); + } + if (children?.length) { + if (injectTailToTokens(children, tailChar, depth + 1)) return true; + } + } + } + return false; +} diff --git a/packages/tdesign-miniprogram/.changelog/pr-4627.md b/packages/tdesign-miniprogram/.changelog/pr-4627.md new file mode 100644 index 0000000000..d7b05869c4 --- /dev/null +++ b/packages/tdesign-miniprogram/.changelog/pr-4627.md @@ -0,0 +1,6 @@ +--- +pr_number: 4627 +contributor: anlyyao +--- + +- feat(CharMarkdown): 新增可选配置 `streaming.completeSyntax`，默认为 false，仅控制是否处理流式末尾未闭合的 Markdown 语法。设为 true 时，会对末尾未闭合的链接、图片、加粗、行内代码等语法进行补全或隐藏，避免暴露原始符号与 URL。 @anlyyao ([#4627](https://github.com/Tencent/tdesign-miniprogram/pull/4627))