diff --git a/.changeset/healthy-geese-compare.md b/.changeset/healthy-geese-compare.md new file mode 100644 index 0000000..cbed694 --- /dev/null +++ b/.changeset/healthy-geese-compare.md @@ -0,0 +1,9 @@ +--- +'@bidilens/markdown': patch +--- + +Expand the optional Markdown-It peer range to cover widely deployed 13.x and 14.x +hosts without exposing either line's third-party types through the public API. +A packed strict TypeScript/peer consumer gate now runs all 932 canonical cases +plus host-structure fixtures through both parser lines and requires identical +batch, plugin, AST, security, isolation, and finalized streaming output. diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 50d4187..f97d35f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -97,6 +97,33 @@ jobs: - run: pnpm run packages:types - run: pnpm run release:check + markdown-it-compat: + name: Markdown-It 13 and 14 compatibility + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: pnpm/action-setup@0977fd99725f1db4007ccb2928dbb4e90d06cc86 # v6.0.10 + with: + version: 10.27.0 + - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 + with: + node-version: 22.12.0 + cache: pnpm + cache-dependency-path: pnpm-lock.yaml + - run: pnpm install --frozen-lockfile + - run: pnpm run markdown-it:compat + env: + BIDILENS_MARKDOWN_IT_REPORT_DIR: markdown-it-compat-reports + - if: ${{ failure() }} + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: markdown-it-compat-reports + path: markdown-it-compat-reports + if-no-files-found: error + retention-days: 7 + android: name: Android libraries and sample runs-on: ubuntu-latest diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 5cfa273..47b0cf6 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -30,6 +30,7 @@ jobs: - run: pnpm exec playwright install --with-deps chromium firefox webkit - run: pnpm run test:visual - run: pnpm run packages:types + - run: pnpm run markdown-it:compat - run: pnpm run deps:audit - run: pnpm run release:check - run: pnpm run sbom diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 83332ca..24770be 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -84,6 +84,7 @@ Changesets entry for a public API or behavior change. Before a release run: pnpm run check pnpm run test:visual pnpm run packages:types +pnpm run markdown-it:compat pnpm run deps:audit pnpm run release:check pnpm run sbom diff --git a/README.md b/README.md index f5c076e..e1239f5 100644 --- a/README.md +++ b/README.md @@ -146,7 +146,7 @@ reordering and shaping; BidiLens supplies the application structure they need. | `@bidilens/core` | Analysis, evidence, isolation, streaming, and security | | `@bidilens/dom` | DOM annotation, restoration, CSS policy, and observation | | `@bidilens/html` | XSS-safe semantic HTML serialization | -| `@bidilens/markdown` | unified/remark/rehype and Markdown-It plugins plus rich Markdown-It streaming | +| `@bidilens/markdown` | unified/remark/rehype and Markdown-It 13/14 plugins plus rich Markdown-It streaming | | `@bidilens/playwright` | Rendering, isolation, selection, clipboard, and geometry assertions | | `@bidilens/react` | SSR-safe components and streaming hooks | | `@bidilens/spec` | Versioned cross-language JSON Schemas and schema registry | @@ -289,7 +289,8 @@ cargo clippy --manifest-path rust/Cargo.toml --all-targets --all-features -- -D ``` `verify:production` runs the complete quality, three-browser visual, package -type, dependency-audit, SBOM, and clean-release sequence. `release:check` +type, packed Markdown-It 13/14 compatibility, dependency-audit, SBOM, and +clean-release sequence. `release:check` builds and packs every public package, inspects tarball contents, installs all tarballs into a temporary consumer, compiles with `skipLibCheck: false`, and executes adapter/runtime assertions—including the pure-LTR non-interference diff --git a/docs/LIMITATIONS.md b/docs/LIMITATIONS.md index 48bbd12..164ca43 100644 --- a/docs/LIMITATIONS.md +++ b/docs/LIMITATIONS.md @@ -42,7 +42,9 @@ deliberately checkpointed by source growth and structural boundaries; inspect `pendingSourceRange` before treating a live `document` as current. `finish()` is the exact batch-equivalence boundary. Unified/remark/rehype transforms remain supported as batch plugins, not as a -stateful unified streaming backend. +stateful unified streaming backend. Markdown-It 13.0.2+ and 14.x are verified +with packed strict TypeScript consumers; other parser major versions are not a +supported or tested claim. ## Validation boundaries diff --git a/docs/OUTREACH_LOG.md b/docs/OUTREACH_LOG.md index c54730d..fa842ee 100644 --- a/docs/OUTREACH_LOG.md +++ b/docs/OUTREACH_LOG.md @@ -42,7 +42,7 @@ endorsement. | Cline VS Code webview | [PR #12724](https://github.com/cline/cline/pull/12724) | Open, mergeable, and ready for review; all fresh hosted checks passed on the rebased head; maintainer review required | Applies `@bidilens/markdown` at the existing React Markdown render boundary and adds Persian-majority, English-majority, and pure-English identity tests. Quality, unit, platform-integration, VS Code on Ubuntu/Windows, E2E on Ubuntu/macOS/Windows, and both Socket security checks passed on 2026-08-13. | | Continue | [comment on issue #2767](https://github.com/continuedev/continue/issues/2767#issuecomment-5088124777) | Parent issue closed; reconsideration requested | Replaces the proposed global RTL rule with Auto/LTR/RTL policy and offers a focused GUI or fixtures-only patch. No new duplicate issue was created. | | assistant-ui | [Show and tell #5211](https://github.com/assistant-ui/assistant-ui/discussions/5211) | Open | Distinguishes the project's completed logical-layout work from per-message mixed-content direction and offers a documented `@bidilens/react` streaming recipe, adapter, hook, or fixtures-only path. | -| AnythingLLM | [comment on canonical RTL issue #3430](https://github.com/Mintplex-Labs/anything-llm/issues/3430#issuecomment-5126056699) | Parent issue open | Supplies a per-block acceptance fixture and integration guidance. A direct dependency patch was deferred because the host supports Node 18 and Markdown-It 13 while BidiLens 0.3 requires Node 22.12 and peers with Markdown-It 14; raising either compatibility floor would be a regression. | +| AnythingLLM | [comment on canonical RTL issue #3430](https://github.com/Mintplex-Labs/anything-llm/issues/3430#issuecomment-5126056699) | Parent issue open | Supplies a per-block acceptance fixture and integration guidance. [BidiLens PR #57](https://github.com/CodeinScrubs/BidiLens/pull/57) addresses the Markdown-It 13 peer/type gap with packed cross-version evidence, but AnythingLLM's Node 18 floor still conflicts with BidiLens 0.3's Node 22.12 minimum; a native fixtures-only patch remains the non-regressive route unless that runtime boundary changes. | | Vercel Streamdown | [PR #569](https://github.com/vercel/streamdown/pull/569) | Open, mergeable, and ready for review; review and security automation passed; maintainer review required | Adds a dependency-free native rehype pass because Streamdown supports Node 18. It assigns direction per semantic block, keeps code LTR, uses content majority with a first-strong tie-breaker, preserves pure-LTR behavior, and includes 985 passing package tests plus a production ESM/declaration build. The Vercel preview remains blocked on external-fork deployment authorization. | | Sentry | [feature request #120893](https://github.com/getsentry/sentry/issues/120893) | Open on the design-engineering backlog at low priority; a Sentry maintainer does not expect the BidiLens dependency to be adopted | Proposes content-aware behavior at Sentry's shared React Markdown renderer and Seer wrapper. After the maintainer noted Sentry's existing i18n tooling, the follow-up explicitly preferred a native/local implementation, offered fixtures only if requested, and committed not to bump the issue. This is backlog evidence, not dependency adoption. | | PostHog | [feature request #75474](https://github.com/PostHog/posthog/issues/75474) | Open | Proposes per-block handling in `LemonMarkdown` and PostHog AI's already memoized `MarkdownMessage` blocks. The public feature tracker was used instead of the automated email's sales-demo form, which requires company, role, and monthly-active-user data and is not an appropriate open-source engineering route. | @@ -59,7 +59,7 @@ production-adoption confirmation when this evidence was recorded. |---|---|---| | JetBrains AI Assistant YouTrack | JetBrains support directed the proposal to the public AI Assistant tracker. A duplicate search found the directly matching, open, assigned [LLM-25407](https://youtrack.jetbrains.com/issue/LLM-25407/Arabic-RTL-Bidirectional-text-rendering-issue-in-Codex-input-field), so no duplicate was created. | Pending account authentication: add one implementation-focused comment with the English-first/Persian-majority fixture, BidiLens corpus and limitation links, and an offer of a scoped renderer patch. | | Nous Research support | The automated receipt for the original proposal assigned reference `T-1645`, while Hermes PR #72508 remained open without human review. | Sent one short follow-up on 2026-08-13 pointing directly to the PR and asking for routing to the Hermes TUI maintainer. | -| n8n support | The support AI said the original proposal had been escalated to a human product/engineering queue, but no human response arrived. | Sent one short follow-up on 2026-08-13 asking whether the team received it or prefers a public feature-request route. | +| n8n support | Senior support engineer Mo Hamdy confirmed the supported contribution route is an n8n Community feature request before any enhancement PR. He also confirmed that `@n8n/chat` uses Markdown-It without raw HTML and that a third-party dependency needs prior maintainer agreement. | A concise reply and [community proposal](outreach/N8N_COMMUNITY_PROPOSAL.md) are drafted, committing to a dependency-free-first approach: fixtures and a native renderer pattern, with BidiLens optional. Sending and posting remain pending explicit user confirmation. | ## Direct organizational outreach @@ -93,7 +93,7 @@ unmonitored addresses, and delivery failures did not receive follow-up mail. | Zulip | The customer-experience lead identified [issue #39511](https://github.com/zulip/zulip/issues/39511) and the existing `#issues` topic for mixed LTR/RTL messages. | Replied that the issue directly overlaps the proposal, clarified the per-paragraph and inline-isolation scope, and committed to continue with small before/after fixtures in the public topic. | | Zed | A Zed representative confirmed that more RTL support is useful but stated that a non-Rust dependency is a non-starter. | The original reply accurately disclosed that no Rust port existed at that time. After [native Rust core PR #31](https://github.com/CodeinScrubs/BidiLens/pull/31) merged, a focused follow-up supplied the source-only status, 928-case/207-isolation evidence, Rust 1.85 target, and final Linux/macOS/Windows CI result. No response to that follow-up, Zed integration, or adoption is claimed. | | Ollama | The support AI requested product feedback in problem/current-behavior/expected-behavior form. After receiving that clarification, it confirmed the submission contains the information its team uses for feature review and made no timeline or adoption commitment. | Sent one structured clarification with the repository and regression-corpus context; stopped after the automated confirmation to avoid an AI-response loop. | -| n8n | The support AI said it escalated the proposal to a human product/engineering review queue. | After no human response arrived, sent one short follow-up on 2026-08-13 asking whether the team received the proposal or prefers a public feature-request route. No further email will be sent without a reply. | +| n8n | Senior support engineer Mo Hamdy replied with the public feature-request route and renderer constraints: Markdown-It, no raw HTML, and prior maintainer agreement before adding a dependency. | Prepared a human reply and a narrower community proposal strategy that leads with fixtures and a native renderer change. BidiLens PR #57 independently adds verified Markdown-It 13/14 compatibility, but it is not presented as a dependency mandate. | | Nous Research | An automated receipt assigned support reference `T-1645` and promised a later follow-up. | After no human response arrived, sent one short follow-up on 2026-08-13 that points directly to Hermes PR #72508 and asks for routing to the TUI maintainer. The public PR remains the primary technical channel. | | JetBrains and Sentry | These responses routed the proposal to public trackers. | The resulting tracker evidence is recorded above; no redundant support-email reply was sent. | | PostHog | An automated response identified an unmonitored inbox and linked support and sales-oriented routes that were unsuitable for an open-source engineering proposal. | No email reply was sent. The public feature request recorded above was opened independently. | @@ -115,7 +115,7 @@ human review. | Sentry | A design-engineering maintainer placed the behavior on the team's low-priority backlog, said Sentry already has i18n tooling, and did not expect to adopt the BidiLens dependency. | Replied once that the behavior is the request, not a dependency mandate; preferred a native/local implementation; offered a small fixture if requested; and committed not to bump the issue. | | Cline | PR #12724 had become conflicting with current upstream. After a narrow rebase, only the lockfile required regeneration; the component, package manifest, and regression tests applied cleanly. | Rebased with force-with-lease, preserved the host lockfile's formatting, and reran hosted CI. All quality, unit, platform, VS Code, three-OS E2E, and security checks passed. No maintainer approval is claimed. | | Nous Research | Support reference `T-1645` remains an automated receipt with no human follow-up. Separately, `hermes-sweeper` revalidated PR #72508 against current `main`, found no verified problem, and recommended keeping it open with high salvageability. | Kept the public PR as the primary technical channel and sent one short routing follow-up on 2026-08-13. Automation is not human review; no additional follow-up is planned without a reply. | -| n8n | The support AI's promised human product/engineering follow-up had not arrived by the audit date. | Sent one short routing/status follow-up on 2026-08-13. No escalation or adoption is claimed, and no additional follow-up is planned without a reply. | +| n8n | Senior support engineer Mo Hamdy supplied an actionable public route and technical constraints on 2026-08-13. | Drafted a reply that follows his guidance and converted the Markdown-It 13 peer/type incompatibility into BidiLens PR #57 with a strict packed 13/14 gate. The reply and community post are not recorded as sent. | | Jupyter, Zulip, Zed, Ollama, JetBrains, PostHog, Cohere, Supabase, Microsoft, and delivery systems | No new actionable human email response was found beyond the earlier audit. | No additional email was sent: these threads were already handled, routed to public channels, automated-only, unmonitored, rejected, or delivery failures. | The August 13 audit adds one maintainer backlog disposition and stronger hosted diff --git a/docs/PUBLISHING.md b/docs/PUBLISHING.md index 3e71765..b05d1ea 100644 --- a/docs/PUBLISHING.md +++ b/docs/PUBLISHING.md @@ -66,9 +66,10 @@ version. - verified `shayanay80` owner access to the `bidilens` npm organization and `@bidilens` scope; - identified bootstrap maintainer and CODEOWNERS; -- strict `main` protection requires all 23 verification contexts: the 18 CI - jobs (including the Android library/sample build, API 35 UI-test gate, Apple - and Windows compiler gates, and three-platform Rust gate) plus five CodeQL +- strict `main` protection requires all 24 verification contexts: the 19 CI + jobs (including the Markdown-It 13/14 packed compatibility gate, Android + library/sample build, API 35 UI-test gate, Apple and Windows compiler gates, + and three-platform Rust gate) plus five CodeQL language analyses for JavaScript/TypeScript, Kotlin, C#, Swift, and Rust. The branch must be up to date and use linear history, while force-pushes and branch deletion are blocked; repository administrators are also subject to @@ -137,8 +138,9 @@ git status --short ``` `verify:production` expands to the quality, three-browser visual, package type, -dependency-audit, SBOM, and clean-release gate set that CI and release -preparation enforce as separate steps. +packed Markdown-It 13/14 compatibility, dependency-audit, SBOM, and +clean-release gate set that CI and release preparation enforce as separate +steps. `release:check` rejects a dirty tree, packs each public package, checks files and dependency protocols, installs all local tarballs into a temporary strict diff --git a/docs/REQUIREMENT_MATRIX.md b/docs/REQUIREMENT_MATRIX.md index 90f6950..f6219ed 100644 --- a/docs/REQUIREMENT_MATRIX.md +++ b/docs/REQUIREMENT_MATRIX.md @@ -39,7 +39,7 @@ Status vocabulary: | Surface | Status | Evidence or exact gap | |---|---|---| | Framework-independent TypeScript core | Complete and tested | `@bidilens/core`; dependency-free runtime; generated Unicode data; analysis, evidence, isolation, security, streaming | -| unified/remark/rehype and markdown-it AST processing | Complete and tested | `@bidilens/markdown`; typed adapters, structural blocks, tables/lists/quotes/code/math, raw-HTML-safe defaults | +| unified/remark/rehype and markdown-it AST processing | Complete and tested | `@bidilens/markdown`; typed adapters, structural blocks, tables/lists/quotes/code/math, raw-HTML-safe defaults; packed strict consumers verify identical 940-fixture behavior on Markdown-It 13.0.2 and 14.3.0 without leaking either host type line | | Plain HTML and DOM | Complete and tested | `@bidilens/html` and `@bidilens/dom`; escaped serialization, restore/observe lifecycle, cross-realm tests | | Web Component | Complete and tested | `@bidilens/web-component`; side-effect-free/SSR-safe main import, explicit `/auto` registration, self-contained browser bundle, three-browser load test | | React | Complete and tested | `@bidilens/react`; React 18/19 probes, SSR, components and streaming hook | diff --git a/docs/outreach/N8N_COMMUNITY_PROPOSAL.md b/docs/outreach/N8N_COMMUNITY_PROPOSAL.md new file mode 100644 index 0000000..8ec4edd --- /dev/null +++ b/docs/outreach/N8N_COMMUNITY_PROPOSAL.md @@ -0,0 +1,76 @@ +# n8n Community proposal draft + +**Status:** prepared for human review; not posted. + +## Suggested title + +Per-block mixed RTL/LTR direction in `@n8n/chat` Markdown messages + +## Problem + +`@n8n/chat` can display Arabic, Persian, Hebrew, Urdu, and other RTL text, but +mixed-direction AI answers need a base direction per rendered block. CSS +`dir="auto"` follows the first strong character, so it misclassifies a +Persian-majority sentence that begins with a technical English token: + +```text +React یک کتابخانه جاوااسکریپت بسیار محبوب است. +``` + +The paragraph should have an RTL base while `React` remains an isolated LTR +run. The mirror case should remain LTR: + +```text +The Persian word کتاب means book. +``` + +A global RTL setting is not sufficient because one answer can contain +independent headings, paragraphs, list items, quotes, table cells, and code. + +## Proposed bounded change + +At `@n8n/chat`'s existing Markdown-It render boundary, add semantic `dir` +metadata to prose block tokens only when the content needs bidirectional +intervention. Keep code LTR and isolate opposite-direction inline technical +runs. Do not enable raw HTML, rewrite generated or stored Markdown, reverse +strings, or change chat layout/navigation direction. + +The first implementation can be dependency-free and local to n8n. BidiLens is +offered as a fixture/policy reference, not as a dependency requirement. If the +maintainers later prefer a package, BidiLens verifies strict packed consumers +against Markdown-It 13.0.2 and 14.3.0. + +## Minimum acceptance fixtures + +| Source | Expected block base | Additional requirement | +|---|---|---| +| `React یک کتابخانه جاوااسکریپت بسیار محبوب است.` | RTL | isolate `React` as LTR | +| `The Persian word کتاب means book.` | LTR | isolate `کتاب` as RTL | +| `Plain English Markdown stays exactly as it is.` | unchanged | no added BidiLens/native attributes or wrappers | +| Persian paragraph with `src/index.ts` | RTL | keep the path LTR | +| fenced TypeScript code inside an RTL answer | LTR code | surrounding prose resolves independently | +| two paragraphs with different majorities | independent | no message-wide direction override | + +For every fixture, logical source, `textContent`, selection, and clipboard text +must remain identical. Existing Markdown-It plugins and `html: false` behavior +must continue to work. + +## Rollout and rollback + +Start behind a chat-renderer feature flag or in shadow analysis, measure only +messages containing RTL characters, and compare source/copy identity before +enabling semantic markup. Pure LTR messages must take the identity path. The +change is confined to one renderer stage and should be removable in one +commit. + +## Evidence and limits + +- BidiLens repository: +- Markdown-It 13/14 compatibility PR: +- Canonical corpus: +- Exact limitations: + +BidiLens currently has zero native-speaker-certified corpus cases and no n8n +maintainer approval, downstream pilot, independent security review, or +accessibility lab result. This proposal asks for a scoped technical review, not +adoption or endorsement. diff --git a/package.json b/package.json index 5928180..bfb8e09 100644 --- a/package.json +++ b/package.json @@ -51,11 +51,12 @@ "npm:release:dry-run": "tsx scripts/publish-npm.ts", "npm:release:publish": "tsx scripts/publish-npm.ts --publish", "packages:types": "tsx scripts/check-package-types.ts", + "markdown-it:compat": "tsx scripts/check-markdown-it-compat.ts", "packages:depth": "tsx scripts/check-package-depth.ts", "sbom": "pnpm dlx @cyclonedx/cdxgen@12.7.1 -t js --no-install-deps --fail-on-error --spec-version 1.7 --json-pretty -o bidilens-sbom.cdx.json .", "sbom:check": "tsx scripts/check-sbom.ts", "deps:audit": "pnpm audit --audit-level low", - "verify:production": "pnpm run check && pnpm run test:visual && pnpm run packages:types && pnpm run deps:audit && pnpm run sbom && pnpm run sbom:check && pnpm run release:check", + "verify:production": "pnpm run check && pnpm run test:visual && pnpm run packages:types && pnpm run markdown-it:compat && pnpm run deps:audit && pnpm run sbom && pnpm run sbom:check && pnpm run release:check", "changeset": "changeset", "changeset:status": "changeset status" }, diff --git a/packages/markdown/README.md b/packages/markdown/README.md index a9b169c..672a8af 100644 --- a/packages/markdown/README.md +++ b/packages/markdown/README.md @@ -27,6 +27,12 @@ const html = await unified() For Markdown-It, install the optional peer and call the typed plugin once: +BidiLens verifies Markdown-It `13.0.2+` and `14.x` with strict packed +TypeScript consumers and the complete canonical corpus. Its public +`MarkdownItCompatible` boundary intentionally exposes only the stable parser +surface BidiLens uses, so a host's matching `@types/markdown-it` line remains +authoritative instead of being duplicated by this package. + ```bash npm install @bidilens/markdown markdown-it ``` diff --git a/packages/markdown/package.json b/packages/markdown/package.json index 52d46dd..cdd60fa 100644 --- a/packages/markdown/package.json +++ b/packages/markdown/package.json @@ -51,12 +51,11 @@ "dependencies": { "@bidilens/core": "workspace:*", "@types/hast": "^3.0.4", - "@types/markdown-it": "^14.1.2", "@types/mdast": "^4.0.4", "unist-util-visit": "^5.0.0" }, "peerDependencies": { - "markdown-it": "^14.0.0" + "markdown-it": "^13.0.2 || ^14.0.0" }, "peerDependenciesMeta": { "markdown-it": { @@ -64,6 +63,7 @@ } }, "devDependencies": { + "@types/markdown-it": "^14.1.2", "markdown-it": "^14.1.0", "rehype-stringify": "^10.0.1", "remark-parse": "^11.0.0", diff --git a/packages/markdown/src/index.ts b/packages/markdown/src/index.ts index 8282dd2..e88e9a7 100644 --- a/packages/markdown/src/index.ts +++ b/packages/markdown/src/index.ts @@ -1,7 +1,7 @@ import type { Element, ElementContent, Root as HastRoot, Text as HastText } from 'hast'; import type { Content, Root as MdastRoot } from 'mdast'; -import type MarkdownIt from 'markdown-it'; -import type Token from 'markdown-it/lib/token.mjs'; +import type InternalMarkdownIt from 'markdown-it'; +import type InternalToken from 'markdown-it/lib/token.mjs'; import { detectDirection, needsBidiIntervention, @@ -19,7 +19,8 @@ import type { BidiMarkdownDocument, BidiMarkdownStreamSession, BidiMarkdownStreamOptions, - MarkdownBidiOptions + MarkdownBidiOptions, + MarkdownItCompatible } from './types.js'; export type { @@ -33,10 +34,15 @@ export type { MarkdownBidiOptions, MarkdownBlockAnnotation, MarkdownDirtyRegion, + MarkdownItCompatible, MarkdownSecurityDelta, MarkdownSourceRange } from './types.js'; +function internalMarkdownIt(markdownIt: MarkdownItCompatible): InternalMarkdownIt { + return markdownIt as unknown as InternalMarkdownIt; +} + const MDAST_BLOCK_TYPES = new Set([ 'paragraph', 'heading', 'blockquote', 'listItem', 'tableCell', 'definition' ]); @@ -277,13 +283,13 @@ function markdownItConfigurationKey(options: MarkdownBidiOptions): string { }); } -function markdownItClass(token: Token | undefined, className: string): void { +function markdownItClass(token: InternalToken | undefined, className: string): void { if (!token) return; if (token.attrJoin) token.attrJoin('class', className); else token.attrSet('class', className); } -function markdownItBlockContent(tokens: Token[], index: number, closeType: string): string { +function markdownItBlockContent(tokens: InternalToken[], index: number, closeType: string): string { const openType = tokens[index]?.type; let nested = 0; const values: string[] = []; @@ -302,7 +308,8 @@ function markdownItBlockContent(tokens: Token[], index: number, closeType: strin } /** Markdown-It adapter with the same content-majority policy as the AST plugins. */ -export function markdownItBidi(md: MarkdownIt, inputOptions: MarkdownBidiOptions = {}): void { +export function markdownItBidi(markdownIt: MarkdownItCompatible, inputOptions: MarkdownBidiOptions = {}): void { + const md = internalMarkdownIt(markdownIt); const options: MarkdownBidiOptions = { ...inputOptions, ...(inputOptions.technicalIdentifiers @@ -321,8 +328,8 @@ export function markdownItBidi(md: MarkdownIt, inputOptions: MarkdownBidiOptions let activeDirection: 'ltr' | 'rtl' | null = null; const blockClassName = options.blockClassName ?? 'bidilens-block'; const codeClassName = options.codeClassName ?? 'bidilens-code'; - const interventionCache = new WeakMap(); - const tokensNeedIntervention = (tokens: Token[]): boolean => { + const interventionCache = new WeakMap(); + const tokensNeedIntervention = (tokens: InternalToken[]): boolean => { const cached = interventionCache.get(tokens); if (cached !== undefined) return cached; const required = tokens.some((token) => (token.type === 'inline' @@ -461,7 +468,7 @@ export function markdownItBidi(md: MarkdownIt, inputOptions: MarkdownBidiOptions const renderValue = (part: string): string => { if (!originalText) return escape(part); const copy = [...tokens]; - copy[index] = { ...tokens[index]!, content: part } as Token; + copy[index] = { ...tokens[index]!, content: part } as InternalToken; return originalText(copy, index, renderOptions, env, self); }; let rendered = ''; @@ -491,12 +498,12 @@ export function markdownItBidi(md: MarkdownIt, inputOptions: MarkdownBidiOptions /** Exact batch document used as the rich stream's final equivalence oracle. */ export function analyzeBidiMarkdown( - markdownIt: MarkdownIt, + markdownIt: MarkdownItCompatible, source: string, options: BidiMarkdownStreamOptions = {} ): BidiMarkdownDocument { markdownItBidi(markdownIt, options); - return analyzeConfiguredBidiMarkdown(markdownIt, source, options); + return analyzeConfiguredBidiMarkdown(internalMarkdownIt(markdownIt), source, options); } /** @@ -505,9 +512,9 @@ export function analyzeBidiMarkdown( * through the same batch pipeline as `analyzeBidiMarkdown()`. */ export function createBidiMarkdownStream( - markdownIt: MarkdownIt, + markdownIt: MarkdownItCompatible, options: BidiMarkdownStreamOptions = {} ): BidiMarkdownStreamSession { markdownItBidi(markdownIt, options); - return new BidiMarkdownStream(markdownIt, options); + return new BidiMarkdownStream(internalMarkdownIt(markdownIt), options); } diff --git a/packages/markdown/src/types.ts b/packages/markdown/src/types.ts index c36cc2c..5532d61 100644 --- a/packages/markdown/src/types.ts +++ b/packages/markdown/src/types.ts @@ -10,6 +10,21 @@ import type { ResolvedDirection } from '@bidilens/core'; +/** + * Version-neutral subset of the Markdown-It instance used by BidiLens. + * + * Keeping the public boundary structural prevents one `@types/markdown-it` + * major from leaking into consumers that use another supported parser line. + * The packed compatibility gate verifies the complete implementation against + * the real v13 and v14 parser/type packages. + */ +export interface MarkdownItCompatible { + parse(source: string, environment: unknown): unknown[]; + readonly options: unknown; + readonly renderer: unknown; + readonly utils: unknown; +} + export interface MarkdownBidiOptions extends DetectionOptions { fallback?: Direction; blockClassName?: string; diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 2f0590c..addfeaf 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -175,9 +175,6 @@ importers: '@types/hast': specifier: ^3.0.4 version: 3.0.5 - '@types/markdown-it': - specifier: ^14.1.2 - version: 14.1.2 '@types/mdast': specifier: ^4.0.4 version: 4.0.4 @@ -185,6 +182,9 @@ importers: specifier: ^5.0.0 version: 5.1.0 devDependencies: + '@types/markdown-it': + specifier: ^14.1.2 + version: 14.1.2 markdown-it: specifier: ^14.1.0 version: 14.3.0 diff --git a/scripts/check-markdown-it-compat.ts b/scripts/check-markdown-it-compat.ts new file mode 100644 index 0000000..ce08f3c --- /dev/null +++ b/scripts/check-markdown-it-compat.ts @@ -0,0 +1,319 @@ +import { spawn } from 'node:child_process'; +import { existsSync } from 'node:fs'; +import { mkdir, mkdtemp, readFile, readdir, rm, writeFile } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { basename, resolve } from 'node:path'; +import process from 'node:process'; + +interface CorpusFixture { + id: string; + text: string; + expectNoOp?: boolean; +} + +interface PackageManifest { + dependencies?: Record; + devDependencies?: Record; + peerDependencies?: Record; +} + +const root = process.cwd(); +const supportedMarkdownIt = [ + { version: '13.0.2', types: '13.0.9' }, + { version: '14.3.0', types: '14.1.2' } +] as const; +const expectedPeerRange = '^13.0.2 || ^14.0.0'; +const reportOutput = process.env.BIDILENS_MARKDOWN_IT_REPORT_DIR; +const structuralFixtures: CorpusFixture[] = [ + { + id: 'compat-pure-ltr-no-op', + text: 'Plain English Markdown stays exactly as it is.', + expectNoOp: true + }, + { + id: 'compat-heading-inline-code', + text: '# راهنمای React\n\nدستور `pnpm test` را اجرا کنید.' + }, + { + id: 'compat-list-and-blockquote', + text: '- React یک کتابخانه محبوب است.\n- Vue هم محبوب است.\n\n> API باید پایدار بماند.' + }, + { + id: 'compat-table', + text: '| ابزار | توضیح |\n| --- | --- |\n| React | یک کتابخانه محبوب |' + }, + { + id: 'compat-link-and-punctuation', + text: 'مستندات [React](https://react.dev/docs?q=rtl) را بخوانید؛ سپس اجرا کنید.' + }, + { + id: 'compat-fenced-code', + text: 'نمونه:\n\n```ts\nconst direction = "rtl";\n```\n\nاین کد امن است.' + }, + { + id: 'compat-raw-html-disabled', + text: 'متن React' + }, + { + id: 'compat-soft-break', + text: 'React یک کتابخانه محبوب است\nو TypeScript هم پشتیبانی می‌شود.' + } +]; + +function pnpmInvocation(args: string[]): { program: string; args: string[]; shell: boolean } { + if (process.platform !== 'win32') return { program: 'pnpm', args, shell: false }; + + const candidates = [ + process.env.npm_execpath, + process.env.PNPM_HOME ? resolve(process.env.PNPM_HOME, 'pnpm.cjs') : undefined, + process.env.APPDATA + ? resolve(process.env.APPDATA, 'npm', 'node_modules', 'pnpm', 'bin', 'pnpm.cjs') + : undefined + ].filter((candidate): candidate is string => Boolean(candidate)); + const cli = candidates.find((candidate) => existsSync(candidate)); + if (cli) return { program: process.execPath, args: [cli, ...args], shell: false }; + return { program: 'pnpm', args, shell: true }; +} + +function command(program: string, args: string[], cwd = root): Promise { + return new Promise((resolveCommand, reject) => { + const invocation = program === 'pnpm' + ? pnpmInvocation(args) + : { program, args, shell: false }; + const child = spawn(invocation.program, invocation.args, { + cwd, + shell: invocation.shell, + stdio: ['ignore', 'pipe', 'pipe'] + }); + let stdout = ''; + let stderr = ''; + child.stdout.setEncoding('utf8'); + child.stderr.setEncoding('utf8'); + child.stdout.on('data', (chunk: string) => { stdout += chunk; }); + child.stderr.on('data', (chunk: string) => { stderr += chunk; }); + child.on('error', reject); + child.on('close', (code) => { + if (code === 0) resolveCommand(stdout); + else reject(new Error(`${program} ${args.join(' ')} failed with ${code}.\n${stdout}${stderr}`)); + }); + }); +} + +function assert(condition: unknown, message: string): asserts condition { + if (!condition) throw new Error(message); +} + +function firstDifference(left: unknown, right: unknown, path = '$'): string | null { + if (Object.is(left, right)) return null; + if (typeof left !== typeof right || left === null || right === null) return path; + if (typeof left !== 'object') return path; + if (Array.isArray(left) !== Array.isArray(right)) return path; + const leftRecord = left as Record; + const rightRecord = right as Record; + const keys = new Set([...Object.keys(leftRecord), ...Object.keys(rightRecord)]); + for (const key of keys) { + const difference = firstDifference( + leftRecord[key], + rightRecord[key], + Array.isArray(left) ? `${path}[${key}]` : `${path}.${key}` + ); + if (difference) return difference; + } + return null; +} + +async function packPackage(packageName: string, destination: string): Promise { + const before = new Set(await readdir(destination)); + await command('pnpm', ['--filter', packageName, 'pack', '--pack-destination', destination]); + const created = (await readdir(destination)) + .find((file) => !before.has(file) && file.endsWith('.tgz')); + assert(created, `${packageName}: pnpm pack did not create a tarball.`); + return resolve(destination, created); +} + +async function runVersionProbe( + target: typeof supportedMarkdownIt[number], + coreTarball: string, + markdownTarball: string, + fixtures: CorpusFixture[], + temporary: string +): Promise { + const { version } = target; + const consumer = resolve(temporary, `markdown-it-${version}`); + await mkdir(consumer, { recursive: true }); + await writeFile(resolve(consumer, 'package.json'), JSON.stringify({ + name: `bidilens-markdown-it-${version.replaceAll('.', '-')}-consumer`, + private: true, + type: 'module', + dependencies: { + '@bidilens/core': `file:${coreTarball.replaceAll('\\', '/')}`, + '@bidilens/markdown': `file:${markdownTarball.replaceAll('\\', '/')}`, + 'markdown-it': version + }, + devDependencies: { + '@types/markdown-it': target.types, + typescript: '6.0.3' + }, + pnpm: { + overrides: { + '@bidilens/core': `file:${coreTarball.replaceAll('\\', '/')}`, + '@bidilens/markdown': `file:${markdownTarball.replaceAll('\\', '/')}` + } + } + }, null, 2)); + await writeFile(resolve(consumer, 'tsconfig.json'), JSON.stringify({ + compilerOptions: { + target: 'ES2022', + module: 'NodeNext', + moduleResolution: 'NodeNext', + strict: true, + noEmit: true, + skipLibCheck: false + }, + include: ['index.ts'] + }, null, 2)); + await writeFile(resolve(consumer, 'index.ts'), ` +import MarkdownIt from 'markdown-it'; +import { + analyzeBidiMarkdown, + createBidiMarkdownStream, + markdownItBidi, + type BidiMarkdownDocument, + type BidiMarkdownStreamSession +} from '@bidilens/markdown'; + +const source = 'React یک کتابخانه جاوااسکریپت بسیار محبوب است.'; +const parser = new MarkdownIt({ html: false, linkify: true, typographer: true }); +markdownItBidi(parser); +const document: BidiMarkdownDocument = analyzeBidiMarkdown(new MarkdownIt(), source); +const stream: BidiMarkdownStreamSession = createBidiMarkdownStream(new MarkdownIt()); +stream.push(source); +void [parser.render(source), document.html, stream.finish().document.html]; +`); + await writeFile(resolve(consumer, 'fixtures.json'), JSON.stringify(fixtures)); + await writeFile(resolve(consumer, 'probe.mjs'), ` +import { strict as assert } from 'node:assert'; +import { readFile } from 'node:fs/promises'; +import MarkdownIt from 'markdown-it'; +import { analyzeBidiMarkdown, createBidiMarkdownStream, markdownItBidi } from '@bidilens/markdown'; + +const fixtures = JSON.parse(await readFile(new URL('./fixtures.json', import.meta.url), 'utf8')); +const results = []; +for (const fixture of fixtures) { + const options = { html: false, linkify: true, typographer: true }; + const baseline = new MarkdownIt(options).render(fixture.text); + const pluginParser = new MarkdownIt(options); + markdownItBidi(pluginParser); + const pluginHtml = pluginParser.render(fixture.text); + const batch = analyzeBidiMarkdown(new MarkdownIt(options), fixture.text); + assert.equal(batch.source, fixture.text, fixture.id + ': batch source changed'); + assert.equal(batch.html, pluginHtml, fixture.id + ': plugin and batch HTML differ'); + if (fixture.expectNoOp) { + assert.equal(pluginHtml, baseline, fixture.id + ': pure-LTR output was modified'); + } + + const stream = createBidiMarkdownStream(new MarkdownIt(options)); + for (const character of fixture.text) stream.push(character); + const final = stream.finish(); + assert.equal(final.source, fixture.text, fixture.id + ': stream source changed'); + assert.deepEqual(final.document, batch, fixture.id + ': stream and batch documents differ'); + + results.push({ + id: fixture.id, + baseline, + pluginHtml, + blocks: batch.blocks.map((block) => ({ + kind: block.kind, + text: block.text, + direction: block.direction, + intervention: block.intervention, + isolation: block.analysis.isolations.map((item) => ({ + start: item.start, + end: item.end, + text: item.text, + direction: item.direction, + kind: item.kind + })) + })), + ast: batch.ast, + security: batch.security + }); +} +process.stdout.write(JSON.stringify(results)); +`); + await command('pnpm', ['install', '--strict-peer-dependencies'], consumer); + await command('pnpm', ['exec', 'tsc', '--noEmit'], consumer); + const declarations = await readFile(resolve( + consumer, + 'node_modules/@bidilens/markdown/dist/index.d.ts' + ), 'utf8'); + assert( + !/from ["']markdown-it(?:\/[^"']*)?["']/u.test(declarations), + `Markdown-It ${version}: packed declarations leaked host parser types.` + ); + return JSON.parse(await command(process.execPath, ['probe.mjs'], consumer)) as unknown; +} + +const manifest = JSON.parse( + await readFile(resolve(root, 'packages/markdown/package.json'), 'utf8') +) as PackageManifest; +assert( + manifest.peerDependencies?.['markdown-it'] === expectedPeerRange, + `@bidilens/markdown must declare markdown-it peer ${expectedPeerRange}.` +); +assert( + manifest.dependencies?.['@types/markdown-it'] === undefined, + '@bidilens/markdown must not expose one Markdown-It type line as a runtime dependency.' +); +assert( + manifest.devDependencies?.['@types/markdown-it'] !== undefined, + '@bidilens/markdown must retain Markdown-It types for its internal build.' +); +const fixtures = JSON.parse( + await readFile(resolve(root, 'corpus/cases.json'), 'utf8') +) as CorpusFixture[]; +assert(fixtures.length >= 900, 'The compatibility gate requires the complete canonical corpus.'); +const probeFixtures = [...fixtures, ...structuralFixtures]; + +await command('pnpm', ['--filter', '@bidilens/core', 'run', 'build']); +await command('pnpm', ['--filter', '@bidilens/markdown', 'run', 'build']); +const temporary = await mkdtemp(resolve(tmpdir(), 'bidilens-markdown-it-compat-')); +try { + const packs = resolve(temporary, 'packs'); + await mkdir(packs, { recursive: true }); + const coreTarball = await packPackage('@bidilens/core', packs); + const markdownTarball = await packPackage('@bidilens/markdown', packs); + const reports = new Map(); + for (const target of supportedMarkdownIt) { + const report = await runVersionProbe( + target, + coreTarball, + markdownTarball, + probeFixtures, + temporary + ); + reports.set(target.version, report); + if (reportOutput) { + await mkdir(resolve(reportOutput), { recursive: true }); + await writeFile( + resolve(reportOutput, `markdown-it-${target.version}.json`), + JSON.stringify(report, null, 2) + ); + } + console.log(`Markdown-It ${target.version}: strict TypeScript consumer, ${fixtures.length} canonical fixtures, and ${structuralFixtures.length} host-structure fixtures passed.`); + } + const referenceVersion = supportedMarkdownIt[0].version; + const reference = reports.get(referenceVersion); + for (const target of supportedMarkdownIt.slice(1)) { + const candidate = reports.get(target.version); + const difference = firstDifference(reference, candidate); + assert( + difference === null, + `Markdown-It ${referenceVersion} and ${target.version} produced different reports at ${difference ?? 'an unknown path'}.` + ); + } + console.log(`Markdown-It ${supportedMarkdownIt.map(({ version }) => version).join(' and ')} produced identical BidiLens reports for ${probeFixtures.length} fixtures.`); + console.log(`Packed @bidilens/markdown ${basename(markdownTarball)} installed under strict peer resolution for both supported parser lines.`); +} finally { + await rm(temporary, { recursive: true, force: true }); +}