feat(chat-markdown): auto-complete unclosed markdown syntax during streaming

packages/pro-components/chat/chat-markdown/README.en-US.md

@@ -10,10 +10,10 @@ style | Object | - | CSS(Cascading Style Sheets) | N
 custom-style | Object | - | CSS(Cascading Style Sheets)，used to set style on virtual component | N
 content | String | - | required | Y
 options | Object | { gfm: true, pedantic: false, breaks: true } | Typescript: `TdChatContentMDOptions ` `interface TdChatContentMDOptions {gfm?: boolean; pedantic?: boolean; smartLists?: boolean; breaks?: boolean}`。[see more ts definition](https://github.com/Tencent/tdesign-miniprogram/blob/develop/packages/pro-components/chat/chat-markdown/type.ts) | N
-streaming | Object | - | Typescript: `TdChatMarkdownStreamingOptions` ` interface TdChatMarkdownStreamingOptions { hasNextChunk?: boolean; tail?: boolean \| { content?: string } }`。[see more ts definition](https://github.com/Tencent/tdesign-miniprogram/blob/develop/packages/pro-components/chat/chat-markdown/type.ts) | N
+streaming | Object | - | Typescript: `TdChatMarkdownStreamingOptions` ` interface TdChatMarkdownStreamingOptions { hasNextChunk?: boolean; completeSyntax?: boolean; tail?: boolean \| { content?: string } }`。[see more ts definition](https://github.com/Tencent/tdesign-miniprogram/blob/develop/packages/pro-components/chat/chat-markdown/type.ts) | N
 
 ### ChatMarkdown Events
 
 name | params | description
 -- | -- | --
-click | `(detail: TdMarkdownClickContext)` | [see more ts definition](https://github.com/Tencent/tdesign-miniprogram/blob/develop/packages/components/chat-markdown/type.ts)。<br/>`interface TdMarkdownClickContext {event: TouchEvent;node: TdMarkdownNode;}`<br/><br/>`interface TdMarkdownNode {   type: string;   raw?: string;   text?: string;   href?: string;   title?: string;   tokens?: TdMarkdownNode[];   [key: string]: unknown;}`<br/>
+click | `(detail: TdMarkdownClickContext)` | [see more ts definition](https://github.com/Tencent/tdesign-miniprogram/blob/develop/packages/pro-components/chat/chat-markdown/type.ts)。<br/>`interface TdMarkdownClickContext {event: TouchEvent;node: TdMarkdownNode;}`<br/><br/>`interface TdMarkdownNode {   type: string;   raw?: string;   text?: string;   href?: string;   title?: string;   tokens?: TdMarkdownNode[];   [key: string]: unknown;}`<br/>

packages/pro-components/chat/chat-markdown/README.md

@@ -93,10 +93,10 @@ style | Object | - | 样式 | N
 custom-style | Object | - | 样式，一般用于开启虚拟化组件节点场景 | N
 content | String | - | 必需。markdown 内容文本 | Y
 options | Object | { gfm: true, pedantic: false, breaks: true } | Markdown 解析器基础配置。TS 类型：`TdChatContentMDOptions ` `interface TdChatContentMDOptions {gfm?: boolean; pedantic?: boolean; smartLists?: boolean; breaks?: boolean}`。[详细类型定义](https://github.com/Tencent/tdesign-miniprogram/blob/develop/packages/pro-components/chat/chat-markdown/type.ts) | N
-streaming | Object | - | 流式输出配置，控制尾部光标的显示与隐藏。尾部光标配置，true 使用默认光标 ▋，传对象可自定义光标字符。TS 类型：`TdChatMarkdownStreamingOptions` ` interface TdChatMarkdownStreamingOptions { hasNextChunk?: boolean; tail?: boolean \| { content?: string } }`。[详细类型定义](https://github.com/Tencent/tdesign-miniprogram/blob/develop/packages/pro-components/chat/chat-markdown/type.ts) | N
+streaming | Object | - | 流式输出配置：`hasNextChunk` 表示是否还有后续分片（输出中为 true，结束时为 false）；`completeSyntax` 默认为 false，仅控制是否处理流式末尾未闭合的 Markdown 语法。设为 true 时，会对末尾未闭合的链接、图片、加粗、行内代码等语法进行补全或隐藏，避免暴露原始符号与 URL；`tail` 控制尾部光标，true 使用默认光标 ▋，传对象可自定义光标字符。TS 类型：`TdChatMarkdownStreamingOptions` ` interface TdChatMarkdownStreamingOptions { hasNextChunk?: boolean; completeSyntax?: boolean; tail?: boolean \| { content?: string } }`。[详细类型定义](https://github.com/Tencent/tdesign-miniprogram/blob/develop/packages/pro-components/chat/chat-markdown/type.ts) | N
 
 ### ChatMarkdown Events
 
 名称 | 参数 | 描述
 -- | -- | --
-click | `(detail: TdMarkdownClickContext)` | 点击链接时触发。[详细类型定义](https://github.com/Tencent/tdesign-miniprogram/blob/develop/packages/components/chat-markdown/type.ts)。<br/>`interface TdMarkdownClickContext {event: TouchEvent;node: TdMarkdownNode;}`<br/><br/>`interface TdMarkdownNode {   type: string;   raw?: string;   text?: string;   href?: string;   title?: string;   tokens?: TdMarkdownNode[];   [key: string]: unknown;}`<br/>
+click | `(detail: TdMarkdownClickContext)` | 点击链接时触发。[详细类型定义](https://github.com/Tencent/tdesign-miniprogram/blob/develop/packages/pro-components/chat/chat-markdown/type.ts)。<br/>`interface TdMarkdownClickContext {event: TouchEvent;node: TdMarkdownNode;}`<br/><br/>`interface TdMarkdownNode {   type: string;   raw?: string;   text?: string;   href?: string;   title?: string;   tokens?: TdMarkdownNode[];   [key: string]: unknown;}`<br/>

packages/pro-components/chat/chat-markdown/__test__/index.test.js

@@ -0,0 +1,83 @@
+import path from 'path';
+import simulate from 'miniprogram-simulate';
+
+jest.mock('marked', () => ({
+  Lexer: class {
+    lex(source) {
+      const marker = '\uE000TD_CHAT_CODE_0\uE001';
+      if (source.includes(marker)) {
+        return [
+          {
+            type: 'paragraph',
+            raw: source,
+            text: source,
+            tokens: [
+              {
+                type: 'strong',
+                raw: `**${marker}**`,
+                text: marker,
+                tokens: [{ type: 'text', raw: marker, text: marker }],
+              },
+            ],
+          },
+        ];
+      }
+      return [
+        {
+          type: 'paragraph',
+          raw: source,
+          text: source,
+          tokens: [
+            { type: 'text', raw: 'a**', text: 'a**', escaped: false },
+            { type: 'codespan', raw: '`code`', text: 'code' },
+            { type: 'text', raw: '**', text: '**', escaped: false },
+          ],
+        },
+      ];
+    }
+  },
+}));
+
+const chatMarkdown = load(path.resolve(__dirname, '../chat-markdown'));
+const content = 'a**`code`**';
+
+function render(streaming) {
+  const id = simulate.load({
+    template: '<t-chat-markdown id="markdown" content="{{content}}" streaming="{{streaming}}" />',
+    data: { content, streaming },
+    usingComponents: { 't-chat-markdown': chatMarkdown },
+  });
+  const container = simulate.render(id);
+  container.attach(document.createElement('parent-wrapper'));
+  return container.querySelector('#markdown');
+}
+
+function getStrongCodeToken(component) {
+  const strong = component.data.nodes[0].tokens.find((token) => token.type === 'strong');
+  return strong?.tokens?.[0];
+}
+
+describe('ChatMarkdown compatibility', () => {
+  it.each([undefined, { hasNextChunk: true, tail: false }, { hasNextChunk: true, tail: false, completeSyntax: false }])(
+    '无论是否开启 completeSyntax 都修正强调语法中的行内代码：%p',
+    (streaming) => {
+      expect(getStrongCodeToken(render(streaming))).toBeTruthy();
+    },
+  );
+
+  it('无 streaming 的无动画模式正常渲染粗体', () => {
+    expect(getStrongCodeToken(render(undefined))).toEqual({ type: 'codespan', raw: '`code`', text: 'code' });
+  });
+
+  it('光标注入粗体中的最后一个 codespan', () => {
+    expect(getStrongCodeToken(render({ hasNextChunk: true, tail: { content: '...' }, completeSyntax: false }))).toEqual(
+      {
+        type: 'codespan',
+        raw: '`code`',
+        text: 'code',
+        isTail: true,
+        tailContent: '...',
+      },
+    );
+  });
+});

packages/pro-components/chat/chat-markdown/__test__/stream-syntax.test.js

@@ -0,0 +1,58 @@
+import completeUnclosedInlineSyntax from '../utils/stream-syntax';
+
+describe('completeUnclosedInlineSyntax', () => {
+  it.each([
+    ['[text', '[text]()'],
+    ['[text]', '[text]()'],
+    ['[text](url', '[text](url)'],
+    ['![alt', ''],
+    ['![alt]', ''],
+    ['![alt](url', ''],
+    ['![alt][ref', ''],
+  ])('补全或隐藏未闭合的链接和图片：%s', (source, expected) => {
+    expect(completeUnclosedInlineSyntax(source)).toBe(expected);
+  });
+
+  it.each([String.raw`\[literal`, String.raw`\![alt`, String.raw`\!`])('不处理转义语法：%s', (source) => {
+    expect(completeUnclosedInlineSyntax(source)).toBe(source);
+  });
+
+  it.each([
+    ['!', ''],
+    ['text !', 'text '],
+    ['你好!', '你好!'],
+  ])('流式阶段区分图片起始符与普通末尾感叹号：%s', (source, expected) => {
+    expect(completeUnclosedInlineSyntax(source)).toBe(expected);
+  });
+
+  it.each(['`!`', '`[text`', '`![alt`', '``**code``'])('不处理行内代码中的语法：%s', (source) => {
+    expect(completeUnclosedInlineSyntax(source)).toBe(source);
+  });
+
+  it.each([
+    ['[inline](https://example.com)', {}],
+    ['![inline](https://example.com/image.png)', {}],
+    ['[id]', { id: { href: 'https://example.com' } }],
+    ['![id]', { id: { href: 'https://example.com/image.png' } }],
+    ['[text][id]', { id: { href: 'https://example.com' } }],
+    ['![alt][id]', { id: { href: 'https://example.com/image.png' } }],
+    ['[text][pending]', {}],
+    ['![alt][pending]', {}],
+  ])('不修改完整内联语法和引用式语法：%s', (source, links) => {
+    expect(completeUnclosedInlineSyntax(source, links)).toBe(source);
+  });
+
+  it.each([
+    ['`code', '`code`'],
+    ['``code', '``code``'],
+    ['``code`', '``code``'],
+    ['`code``', '`code`` `'],
+  ])('使用等长反引号补全行内代码：%s', (source, expected) => {
+    expect(completeUnclosedInlineSyntax(source)).toBe(expected);
+  });
+
+  it('忽略已闭合代码围栏中的反引号', () => {
+    const source = ['```js', 'const value = `code`;', '```', 'plain text'].join('\n');
+    expect(completeUnclosedInlineSyntax(source)).toBe(source);
+  });
+});

packages/pro-components/chat/chat-markdown/__test__/tail-cursor.test.js

@@ -0,0 +1,59 @@
+import { injectTailToTokens, resolveTailContent } from '../utils/tail-cursor';
+
+describe('tail cursor', () => {
+  it('解析默认及自定义光标内容', () => {
+    expect(resolveTailContent()).toBeNull();
+    expect(resolveTailContent(false)).toBeNull();
+    expect(resolveTailContent(true)).toBe('▋');
+    expect(resolveTailContent({ content: '' })).toBe('▋');
+    expect(resolveTailContent({ content: '...' })).toBe('...');
+  });
+
+  it('将光标注入最后一个可见的 codespan', () => {
+    const tokens = [
+      { type: 'text', raw: 'prefix ', text: 'prefix ' },
+      { type: 'codespan', raw: '`code`', text: 'code' },
+    ];
+
+    expect(injectTailToTokens(tokens, '...')).toBe(true);
+    expect(tokens).toEqual([
+      { type: 'text', raw: 'prefix ', text: 'prefix ' },
+      { type: 'codespan', raw: '`code`', text: 'code', isTail: true, tailContent: '...' },
+    ]);
+  });
+
+  it('将列表 text 包装节点的光标注入可见子节点', () => {
+    const tokens = [
+      {
+        type: 'text',
+        raw: 'item',
+        text: 'item',
+        tokens: [{ type: 'text', raw: 'item', text: 'item' }],
+      },
+    ];
+
+    expect(injectTailToTokens(tokens, '...')).toBe(true);
+    expect(tokens[0].isTail).toBeUndefined();
+    expect(tokens[0].tokens[0]).toMatchObject({ isTail: true, tailContent: '...' });
+  });
+
+  it('第三个参数继续作为递归深度使用', () => {
+    const tokens = [
+      {
+        type: 'paragraph',
+        tokens: [{ type: 'text', raw: 'nested', text: 'nested' }],
+      },
+    ];
+
+    expect(injectTailToTokens(tokens, '...', 2)).toBe(true);
+    expect(tokens[0].tokens[0]).toMatchObject({ isTail: true, tailContent: '...' });
+  });
+
+  it('将光标注入代码块', () => {
+    const tokens = [{ type: 'code', raw: 'code', text: 'code' }];
+
+    injectTailToTokens(tokens, '...');
+
+    expect(tokens[0]).toEqual({ type: 'code', raw: 'code', text: 'code', isTail: true, tailContent: '...' });
+  });
+});

packages/pro-components/chat/chat-markdown/_example/tail/index.js

@@ -13,7 +13,7 @@ Page({
     ],
     selectedSegment: -1, // 初始不选中任何分段
     content: '',
-    streaming: { hasNextChunk: false, tail: true },
+    streaming: { hasNextChunk: false, tail: true, completeSyntax: true },
   },
 
   onLoad() {
@@ -51,15 +51,15 @@ Page({
 
       this.setData({
         selectedSegment: segmentValue,
-        streaming: { hasNextChunk: true, tail: true },
+        streaming: { hasNextChunk: true, tail: true, completeSyntax: true },
       });
 
       this._timer = setInterval(() => {
         index += CHUNK_SIZE;
         const isDone = index >= markdownData.length;
         this.setData({
           content: markdownData.slice(0, index),
-          streaming: { hasNextChunk: !isDone, tail: true },
+          streaming: { hasNextChunk: !isDone, tail: true, completeSyntax: true },
         });
         if (isDone) {
           clearInterval(this._timer);

packages/pro-components/chat/chat-markdown/chat-markdown-code/chat-markdown-code.wxml

@@ -6,6 +6,7 @@
 
   <!-- 代码内容区域 -->
   <scroll-view class="{{classPrefix}}__content" scroll-x="{{true}}">
-    <text class="{{classPrefix}}__text" decode="{{true}}">{{node.text}}</text><tail-component wx:if="{{node.isTail}}" />
+    <text class="{{classPrefix}}__text" decode="{{true}}">{{node.text}}</text
+    ><tail-component wx:if="{{node.isTail}}" content="{{node.tailContent}}" />
   </scroll-view>
 </view>

packages/pro-components/chat/chat-markdown/chat-markdown-node/chat-markdown-node.wxml

@@ -55,7 +55,9 @@
       <block wx:if="{{item.tokens && item.tokens.length}}">
         <chat-markdown-node nodes="{{item.tokens}}" generic:tail-component="tail-component" />
       </block>
-      <block wx:else> {{''+item.raw+''}} <tail-component wx:if="{{item.isTail}}" /> </block>
+      <block wx:else>
+        {{''+item.raw+''}} <tail-component wx:if="{{item.isTail}}" content="{{item.tailContent}}" />
+      </block>
     </view>
   </block>
   <block wx:elif="{{item.type==='strong'}}">
@@ -112,6 +114,7 @@
     >
       {{''+(item.text||item.raw)+''}}
     </view>
+    <tail-component wx:if="{{item.isTail}}" content="{{item.tailContent}}" />
   </block>
   <block wx:else>
     <view

packages/pro-components/chat/chat-markdown/chat-markdown.ts

@@ -3,82 +3,13 @@ import { SuperComponent, wxComponent, ComponentsOptionsType } from '../../../com
 import config from '../../../components/common/config';
 import props from './props';
 import { TdChatMarkdownProps } from './type';
+import completeUnclosedInlineSyntax from './utils/stream-syntax';
+import { resolveTailContent, injectTailToTokens } from './utils/tail-cursor';
+import { hasLeakedInlineFormatting, maskInlineCodes, restoreMaskedInlineCodes } from './utils/marked-fallback';
 
 const { prefix } = config;
 const name = `${prefix}-chat-markdown`;
 
-const DEFAULT_TAIL_CONTENT = '▋';
-
-/** 解析 tail 参数，返回光标字符；不需要显示时返回 null */
-function resolveTailContent(tail?: boolean | { content?: string }): string | null {
-  if (!tail) return null;
-  if (typeof tail === 'boolean') return DEFAULT_TAIL_CONTENT;
-  return tail.content || DEFAULT_TAIL_CONTENT;
-}
-
-/**
- * 将列表项的子 tokens 展平，供 injectTailToTokens 递归使用。
- * marked 的 list token 结构：list.items[].tokens（而非 list.tokens）
- */
-function flatListItems(items: any[]): any[] {
-  return items.reduce((result: any[], item: any) => {
-    if (item.tokens?.length) result.push(...item.tokens);
-    return result;
-  }, []);
-}
-
-/**
- * 从后往前遍历 token 树，找到最后一个非空 text 叶子节点，打上 isTail 标记。
- * - 有子节点（tokens / items）时优先递归
- * - 末尾是 code / table / image 等非 text 节点时静默跳过，不注入
- * @returns 是否成功注入
- */
-function injectTailToTokens(tokens: any[], tailChar: string, depth = 0): boolean {
-  for (let i = tokens.length - 1; i >= 0; i -= 1) {
-    const token = tokens[i];
-
-    // code 块作为叶子节点，直接注入，不递归
-    if (token.type === 'code' && (token.text || token.raw)?.trim()) {
-      token.isTail = true;
-      token.tailContent = tailChar;
-      return true;
-    }
-
-    // 叶子文本节点且内容非空
-    if (token.type === 'text' && (token.text || token.raw)?.trim()) {
-      token.isTail = true;
-      token.tailContent = tailChar;
-      return true;
-    }
-
-    // table 节点：从后往前遍历 rows，再遍历 header
-    if (token.type === 'table') {
-      const allRows: any[][] = [...(token.header ? [token.header] : []), ...(token.rows || [])];
-      for (let r = allRows.length - 1; r >= 0; r -= 1) {
-        const row = allRows[r];
-        for (let c = row.length - 1; c >= 0; c -= 1) {
-          const cell = row[c];
-          if (cell.tokens?.length) {
-            if (injectTailToTokens(cell.tokens, tailChar, depth + 1)) return true;
-          }
-        }
-      }
-    } else {
-      // 有子节点时递归
-      let children: any[] | null = null;
-      if (token.tokens?.length) {
-        children = token.tokens;
-      } else if (token.items?.length) {
-        children = flatListItems(token.items);
-      }
-      if (children?.length) {
-        if (injectTailToTokens(children, tailChar, depth + 1)) return true;
-      }
-    }
-  }
-  return false;
-}
-
 export interface ChatMarkdownProps extends TdChatMarkdownProps {}
 
 @wxComponent()
@@ -110,11 +41,23 @@ export default class ChatMarkdown extends SuperComponent {
     // 解析markdown文本
     parseMarkdown(markdown: string) {
       try {
+        const { streaming } = this.data;
+        // 流式输出时对末尾未闭合语法做补全/隐藏（需显式开启 streaming.completeSyntax）
+        const syntaxCompletionEnabled = streaming?.completeSyntax === true;
+        const shouldComplete = streaming?.hasNextChunk && syntaxCompletionEnabled;
         const lexer = new Lexer(this.data.options);
-        const tokens = lexer.lex(markdown);
+        const originalTokens = lexer.lex(markdown);
+        const source = shouldComplete ? completeUnclosedInlineSyntax(markdown, originalTokens.links) : markdown;
+        let tokens = source === markdown ? originalTokens : new Lexer(this.data.options).lex(source);
+
+        // 修正 marked 对“强调语法内包含行内代码”场景的 token 泄漏；与流式语法补全开关无关。
+        if (hasLeakedInlineFormatting(tokens)) {
+          const { source: maskedSource, masks } = maskInlineCodes(source);
+          tokens = new Lexer(this.data.options).lex(maskedSource);
+          restoreMaskedInlineCodes(tokens, masks);
+        }
 
         // 尾部光标注入
-        const { streaming } = this.data;
         const tailChar = resolveTailContent(streaming?.tail);
         if (streaming?.hasNextChunk && tailChar) {
           injectTailToTokens(tokens, tailChar);