diff --git a/README.md b/README.md index 599c62ab..a28ed953 100644 --- a/README.md +++ b/README.md @@ -29,6 +29,20 @@ conditional: [tenant, nav, ssb] ^^^^^^--- the keyword "tenant" is required to exclude all not mentioned ``` +## llms.txt curation + +Add selected entry-point pages to `llms.txt` with global ordering metadata: + +```yaml +llms: + section: Workloads + order: 30 +``` + +Pages are grouped by `section` and ordered by the numeric `order`. Every +rendered documentation page is also available as Markdown at its canonical +path with a `.md` suffix; the root `README.md` is `/index.md`. + ## Copilot Agents Four agents handle documentation tasks: diff --git a/docs/README.md b/docs/README.md index 31038957..395a2694 100644 --- a/docs/README.md +++ b/docs/README.md @@ -1,5 +1,8 @@ --- title: Nais Developer Documentation +llms: + section: Core and getting started + order: 10 hide: - feedback - footer diff --git a/docs/auth/README.md b/docs/auth/README.md index 69c990e8..c077774e 100644 --- a/docs/auth/README.md +++ b/docs/auth/README.md @@ -1,5 +1,8 @@ --- tags: [auth, explanation] +llms: + section: Authentication + order: 100 description: Services and addons to support authentication and authorization in your applications. --- diff --git a/docs/auth/entra-id/README.md b/docs/auth/entra-id/README.md index b8801a8b..460fe749 100644 --- a/docs/auth/entra-id/README.md +++ b/docs/auth/entra-id/README.md @@ -1,5 +1,8 @@ --- tags: [auth, entra-id, azure-ad, services, explanation] +llms: + section: Authentication + order: 110 conditional: [tenant, nav] --- diff --git a/docs/auth/idporten/README.md b/docs/auth/idporten/README.md index 8aaec287..945c592c 100644 --- a/docs/auth/idporten/README.md +++ b/docs/auth/idporten/README.md @@ -1,5 +1,8 @@ --- tags: [auth, idporten, services, explanation] +llms: + section: Authentication + order: 120 conditional: [tenant, nav] --- diff --git a/docs/build/README.md b/docs/build/README.md index 0648f897..3b156279 100644 --- a/docs/build/README.md +++ b/docs/build/README.md @@ -1,5 +1,8 @@ --- tags: [build, deploy, explanation, services] +llms: + section: Build and deploy + order: 40 --- # Build and deploy diff --git a/docs/build/how-to/build-and-deploy.md b/docs/build/how-to/build-and-deploy.md index 98cbc505..955c1bac 100644 --- a/docs/build/how-to/build-and-deploy.md +++ b/docs/build/how-to/build-and-deploy.md @@ -1,5 +1,8 @@ --- tags: [build, deploy, how-to] +llms: + section: Build and deploy + order: 50 --- # Build and deploy with GitHub Actions diff --git a/docs/explanations/nais.md b/docs/explanations/nais.md index e27d3202..73dbd891 100644 --- a/docs/explanations/nais.md +++ b/docs/explanations/nais.md @@ -1,5 +1,8 @@ --- tags: [explanation, nais] +llms: + section: Core and getting started + order: 20 --- # What is Nais? diff --git a/docs/observability/README.md b/docs/observability/README.md index fbca0ea2..3a70eeb8 100644 --- a/docs/observability/README.md +++ b/docs/observability/README.md @@ -5,6 +5,9 @@ description: >- search: boost: 1 tags: [explanation, observability] +llms: + section: Observability + order: 180 --- # Observability diff --git a/docs/observability/logging/README.md b/docs/observability/logging/README.md index 53d704cd..18aa2417 100644 --- a/docs/observability/logging/README.md +++ b/docs/observability/logging/README.md @@ -1,6 +1,9 @@ --- description: Understand how application logging works in Nais, its purpose, good practices, and the available log destinations. tags: [explanation, logging, observability, services] +llms: + section: Observability + order: 190 --- # Logging diff --git a/docs/observability/metrics/README.md b/docs/observability/metrics/README.md index ce0bada7..58ed9847 100644 --- a/docs/observability/metrics/README.md +++ b/docs/observability/metrics/README.md @@ -1,6 +1,9 @@ --- description: Metrics are a way to measure the state of your application and can be used to create alerts and dashboards in Grafana. tags: [explanation, metrics, observability, services] +llms: + section: Observability + order: 200 --- # Metrics diff --git a/docs/observability/tracing/README.md b/docs/observability/tracing/README.md index 9abb659e..83d3148d 100644 --- a/docs/observability/tracing/README.md +++ b/docs/observability/tracing/README.md @@ -2,6 +2,9 @@ description: >- Application Performance Monitoring or tracing using Grafana Tempo on Nais. tags: [explanation, observability, tracing, services] +llms: + section: Observability + order: 210 --- # Distributed Tracing diff --git a/docs/operate/README.md b/docs/operate/README.md index e48e1e1f..28784d43 100644 --- a/docs/operate/README.md +++ b/docs/operate/README.md @@ -1,5 +1,8 @@ --- tags: [explanation, operate] +llms: + section: Services and operations + order: 240 --- # Manage your workloads and services diff --git a/docs/persistence/README.md b/docs/persistence/README.md index f1a6a6bd..06a73d04 100644 --- a/docs/persistence/README.md +++ b/docs/persistence/README.md @@ -3,6 +3,9 @@ description: >- Nais offers several storage solutions for storing data. This page describes the different options and how to use them. tags: [persistence, explanation] +llms: + section: Persistence + order: 130 --- # Persistent Data Overview diff --git a/docs/persistence/cloudsql/README.md b/docs/persistence/cloudsql/README.md index 70e2e053..1b0bf173 100644 --- a/docs/persistence/cloudsql/README.md +++ b/docs/persistence/cloudsql/README.md @@ -1,5 +1,8 @@ --- tags: [explanation, persistence, services] +llms: + section: Persistence + order: 150 --- # Google Cloud SQL / PostgreSQL diff --git a/docs/persistence/kafka/README.md b/docs/persistence/kafka/README.md index 069c64b3..7d07a382 100644 --- a/docs/persistence/kafka/README.md +++ b/docs/persistence/kafka/README.md @@ -4,6 +4,9 @@ description: >- subscribe to streams of records. It is a good alternative to synchronous communication between services if you need to decouple services. tags: [kafka, explanation, persistence, services] +llms: + section: Persistence + order: 140 --- # Kafka diff --git a/docs/persistence/opensearch/README.md b/docs/persistence/opensearch/README.md index 102bd353..f3dceb62 100644 --- a/docs/persistence/opensearch/README.md +++ b/docs/persistence/opensearch/README.md @@ -1,5 +1,8 @@ --- tags: [opensearch, explanation, persistence, services] +llms: + section: Persistence + order: 160 description: >- Nais provides managed search index services through OpenSearch as a drop-in replacement for Elasticsearch. diff --git a/docs/persistence/valkey/README.md b/docs/persistence/valkey/README.md index e90b9bd1..56fa18a0 100644 --- a/docs/persistence/valkey/README.md +++ b/docs/persistence/valkey/README.md @@ -1,5 +1,8 @@ --- tags: [persistence, explanation, valkey, redis, services] +llms: + section: Persistence + order: 170 --- # Valkey diff --git a/docs/services/README.md b/docs/services/README.md index a804f9a4..6a2836a2 100644 --- a/docs/services/README.md +++ b/docs/services/README.md @@ -1,3 +1,9 @@ +--- +llms: + section: Services and operations + order: 220 +--- + # Other services This section covers the rest of the Nais functionality that didn't fit into any other categories. diff --git a/docs/services/secrets/README.md b/docs/services/secrets/README.md index 27ba9346..fdff53ce 100644 --- a/docs/services/secrets/README.md +++ b/docs/services/secrets/README.md @@ -1,5 +1,8 @@ --- tags: [secrets, explanation, services] +llms: + section: Services and operations + order: 230 --- # Secrets diff --git a/docs/tutorials/hello-nais.md b/docs/tutorials/hello-nais.md index dfd1cc12..4d24e124 100644 --- a/docs/tutorials/hello-nais.md +++ b/docs/tutorials/hello-nais.md @@ -1,5 +1,8 @@ --- tags: [tutorial] +llms: + section: Core and getting started + order: 30 --- # :wave: Hello Nais diff --git a/docs/workloads/README.md b/docs/workloads/README.md index 466cad1f..acc31586 100644 --- a/docs/workloads/README.md +++ b/docs/workloads/README.md @@ -1,5 +1,8 @@ --- tags: [workloads, explanation] +llms: + section: Workloads + order: 60 --- # Workloads diff --git a/docs/workloads/application/README.md b/docs/workloads/application/README.md index 8e725fd8..c9b0aca3 100644 --- a/docs/workloads/application/README.md +++ b/docs/workloads/application/README.md @@ -1,5 +1,8 @@ --- tags: [application, explanation, workloads, services] +llms: + section: Workloads + order: 70 --- # Application diff --git a/docs/workloads/application/how-to/create.md b/docs/workloads/application/how-to/create.md index 3724a2f5..8901b957 100644 --- a/docs/workloads/application/how-to/create.md +++ b/docs/workloads/application/how-to/create.md @@ -1,5 +1,8 @@ --- tags: [application, how-to] +llms: + section: Workloads + order: 80 --- # Create application diff --git a/docs/workloads/job/README.md b/docs/workloads/job/README.md index 7eda22de..7e23a525 100644 --- a/docs/workloads/job/README.md +++ b/docs/workloads/job/README.md @@ -1,5 +1,8 @@ --- tags: [job, explanation, workloads, services] +llms: + section: Workloads + order: 90 --- # Nais job diff --git a/svdoc/src/lib/content-store.ts b/svdoc/src/lib/content-store.ts index b371f756..93967fff 100644 --- a/svdoc/src/lib/content-store.ts +++ b/svdoc/src/lib/content-store.ts @@ -12,8 +12,8 @@ */ import type { Token, TokensList } from "marked"; -import { readdir, stat } from "node:fs/promises"; -import { relative, resolve } from "node:path"; +import { readFile, readdir, stat } from "node:fs/promises"; +import { dirname, relative, resolve } from "node:path"; import { IGNORED_DIRECTORIES } from "./constants"; import { extractHeadingsFromTokens, @@ -21,6 +21,7 @@ import { stripMarkdownTokens, type Heading, } from "./helpers/markdown-utils"; +import type { LlmsDocument } from "./llms"; import { processLinks } from "./helpers/process-links"; import { stripBase, stripMarkdownSuffix, withBase } from "./helpers/urls"; import { readMarkdownFile, type Attributes } from "./markdown"; @@ -132,6 +133,8 @@ interface ContentDocument { tags: string[]; /** Items to hide (e.g., "toc", "navigation") */ hide: string[]; + /** Whether an ancestor .pages file hides this document from navigation */ + hiddenFromNavigation: boolean; /** Extracted headings for search/TOC */ headings: Heading[]; /** Stripped content for search indexing */ @@ -178,6 +181,66 @@ export interface NavItem { hasContent: boolean; } +/** + * Flatten navigation into document paths. Directory README pages are included + * through `hasContent`; the set prevents an explicitly listed README from + * appearing again as a child entry. + */ +export function navigationDocumentPaths(items: NavItem[]): string[] { + const paths: string[] = []; + const seen = new Set(); + + const visit = (entries: NavItem[]) => { + for (const item of entries) { + if (item.hasContent) { + const path = stripBase(item.href).replace(/\/+$/, "") || "/"; + if (!seen.has(path)) { + seen.add(path); + paths.push(path); + } + } + if (item.children) visit(item.children); + } + }; + + visit(items); + return paths; +} + +/** + * Return navigation-visible documents in sidebar order. Documents that do not + * have a navigation item are appended by URL path so an accidental `.pages` + * omission cannot make otherwise visible documentation disappear. + */ +export function orderLlmsFullDocuments( + documents: LlmsDocument[], + navigation: NavItem[], +): LlmsDocument[] { + const byPath = new Map(documents.map((document) => [document.urlPath, document])); + const seen = new Set(); + const ordered: LlmsDocument[] = []; + const add = (document: LlmsDocument | undefined) => { + if ( + !document || + document.hiddenFromNavigation || + document.hide.includes("navigation") || + seen.has(document.urlPath) + ) + return; + seen.add(document.urlPath); + ordered.push(document); + }; + + for (const path of navigationDocumentPaths(navigation)) { + add(byPath.get(path)); + } + + for (const document of [...documents].sort((a, b) => a.urlPath.localeCompare(b.urlPath))) { + add(document); + } + return ordered; +} + /** * .pages file structure */ @@ -360,6 +423,7 @@ class ContentStore { description: attributes.description, tags: attributes.tags || [], hide: attributes.hide || [], + hiddenFromNavigation: await this.isHiddenFromNavigation(filePath), headings, searchContent, summary, @@ -556,6 +620,39 @@ class ContentStore { return this.allPaths; } + /** + * Get every routable document's Markdown route. Unlike HTML page entries, + * this deliberately includes pages hidden from navigation: `hide: + * [navigation]` hides a sidebar item, not the document itself. + */ + async getMarkdownPaths(): Promise { + await this.initialize(); + return Array.from(this.documentsByUrlPath.values()) + .filter((document) => !document.hiddenFromNavigation) + .map((document) => (document.urlPath === "/" ? "index" : document.urlPath.slice(1))) + .sort(); + } + + /** + * Get documents for llms-full.txt in navigation order. Pages that are + * visible but absent from navigation (for example, because a `.pages` + * entry was accidentally omitted) follow in URL-path order, making the + * fallback deterministic while retaining discoverability. + */ + async getLlmsFullDocuments(): Promise { + await this.initialize(); + return orderLlmsFullDocuments( + Array.from(this.documentsByUrlPath.values()), + this.navigation ?? [], + ); + } + + /** Get all tenant-visible documents for llms.txt curation. */ + async getLlmsDocuments(): Promise { + await this.initialize(); + return Array.from(this.documentsByUrlPath.values()); + } + /** * Get search index data */ @@ -621,7 +718,7 @@ class ContentStore { try { const { parse: parseYaml } = await import("yaml"); const pagesPath = `${dirPath}/.pages`; - const content = await Bun.file(pagesPath).text(); + const content = await readFile(pagesPath, "utf-8"); return parseYaml(content) as PagesFile; } catch { return null; @@ -647,12 +744,29 @@ class ContentStore { } } + /** + * `.pages` can hide a whole directory, including partials that remain + * routable by URL. Keep that distinction for llms-full.txt, whose fallback + * must include only documents omitted from navigation accidentally. + */ + private async isHiddenFromNavigation(filePath: string): Promise { + let directory = dirname(filePath); + while (directory.startsWith(DOCS_DIR)) { + const pagesFile = await this.readPagesFile(directory); + if (pagesFile?.hide === true) return true; + if (directory === DOCS_DIR) return false; + directory = dirname(directory); + } + return false; + } + /** * Check if a directory has a README.md that is included in the store */ private hasIncludedReadme(dirPath: string): boolean { const readmePath = `${dirPath}/README.md`; - return this.documents.has(readmePath); + const document = this.documents.get(readmePath); + return document !== undefined && !document.hide.includes("navigation"); } /** @@ -670,7 +784,8 @@ class ContentStore { } else { // A file is included if it's in the documents map const mdPath = fullPath.endsWith(".md") ? fullPath : `${fullPath}.md`; - return this.documents.has(mdPath); + const document = this.documents.get(mdPath); + return document !== undefined && !document.hide.includes("navigation"); } } diff --git a/svdoc/src/lib/helpers/templates.test.ts b/svdoc/src/lib/helpers/templates.test.ts index 66227c2d..380fde28 100644 --- a/svdoc/src/lib/helpers/templates.test.ts +++ b/svdoc/src/lib/helpers/templates.test.ts @@ -156,6 +156,36 @@ describe("elif and else on their own lines", () => { } }); +describe("nested conditionals", () => { + const input = [ + "before", + '{% if tenant() in ("nav", "test-nais") %}', + " outer branch", + ' {% if tenant() == "nav" %}', + " inner nav <>", + ' {% elif tenant() == "test-nais" %}', + " inner test <>", + " {% else %}", + " inner fallback", + " {% endif %}", + "{% else %}", + "outer fallback <>", + "{% endif %}", + "after", + ].join("\n"); + + test("does not process nested content when the outer branch is false", () => { + expect(render(input, "dev-nais")).toBe("before\nouter fallback dev-nais\nafter"); + }); + + test("resolves the selected inner branch when the outer branch is true", () => { + expect(render(input, "nav")).toBe("before\n outer branch\n inner nav nav\nafter"); + expect(render(input, "test-nais")).toBe( + "before\n outer branch\n inner test test-nais\nafter", + ); + }); +}); + describe("explicit whitespace markers", () => { test("are redundant for standalone tags, which already consume their line", () => { const withMarkers = 'a\n{%- if tenant() == "nav" -%}\nb\n{%- endif -%}\nc'; @@ -225,6 +255,29 @@ describe("set statements", () => { }); }); +describe("raw blocks", () => { + test("removes raw markers without processing their contents", () => { + expect( + render( + `before +{% raw %} +{{#each items}} +<> +{{/each}} +{% endraw %} +after`, + "nav", + ), + ).toBe(`before + +{{#each items}} +<> +{{/each}} + +after`); + }); +}); + describe("fenced code block inside an admonition", () => { // The regression that motivated line-consuming tags: a templated YAML // workflow nested inside a `???+ note` admonition must render as valid, diff --git a/svdoc/src/lib/helpers/templates.ts b/svdoc/src/lib/helpers/templates.ts index 8b63350d..982ff210 100644 --- a/svdoc/src/lib/helpers/templates.ts +++ b/svdoc/src/lib/helpers/templates.ts @@ -259,9 +259,20 @@ function processIncludes(content: string, basePath: string, maxDepth: number = 1 }); } -interface ConditionalBranch { - condition: string | null; - content: string; +type ConditionalTagType = "if" | "elif" | "else" | "endif"; + +interface ConditionalTag { + type: ConditionalTagType; + condition?: string; + start: number; + end: number; + text: string; +} + +interface ConditionalBlock { + openTag: ConditionalTag; + closeTag: ConditionalTag; + branchTags: ConditionalTag[]; } /** @@ -288,65 +299,87 @@ function expandTagLine(text: string, start: number, end: number): [number, numbe } /** - * Parse the branches (if/elif/else) from inner conditional content + * Find conditional tags and pair each opening tag with its matching endif. */ -function parseConditionalBranches(ifCondition: string, innerContent: string): ConditionalBranch[] { - const branches: ConditionalBranch[] = []; - const branchRegex = /\{%-?\s*(?:elif\s+(.+?)|else)\s*-?%\}/g; - - // Find all branch markers (elif/else) - const parts: { type: "elif" | "else"; condition?: string; index: number; length: number }[] = []; - let branchMatch; - - while ((branchMatch = branchRegex.exec(innerContent)) !== null) { - parts.push({ - type: branchMatch[1] ? "elif" : "else", - condition: branchMatch[1], - index: branchMatch.index, - length: branchMatch[0].length, +function findConditionalBlock(content: string): ConditionalBlock | undefined { + const tagRegex = /\{%-?\s*(?:(if|elif)\s+(.+?)|(else|endif))\s*-?%\}/g; + const tags: ConditionalTag[] = []; + let match: RegExpExecArray | null; + + while ((match = tagRegex.exec(content)) !== null) { + const keyword = match[1] ?? match[3]; + if (!keyword) { + continue; + } + + const type: ConditionalTagType = + keyword === "if" || keyword === "elif" ? keyword : keyword === "else" ? "else" : "endif"; + + tags.push({ + type, + condition: match[2], + start: match.index, + end: match.index + match[0].length, + text: match[0], }); } - if (parts.length === 0) { - // No elif or else, just the if content - branches.push({ condition: ifCondition, content: innerContent }); - } else { - // First branch: from start to first elif/else - const [firstEnd] = expandTagLine( - innerContent, - parts[0].index, - parts[0].index + parts[0].length, - ); - branches.push({ - condition: ifCondition, - content: innerContent.slice(0, firstEnd), - }); + const openIndex = tags.findIndex((tag) => tag.type === "if"); + if (openIndex === -1) { + return undefined; + } + + const openTag = tags[openIndex]; + const branchTags: ConditionalTag[] = []; + let depth = 1; + + for (let index = openIndex + 1; index < tags.length; index++) { + const tag = tags[index]; + if (tag.type === "if") { + depth++; + } else if (tag.type === "endif") { + depth--; + if (depth === 0) { + return { openTag, closeTag: tag, branchTags }; + } + } else if (depth === 1) { + branchTags.push(tag); + } + } + + return undefined; +} - // Middle and final branches - for (let i = 0; i < parts.length; i++) { - const [, startIndex] = expandTagLine( - innerContent, - parts[i].index, - parts[i].index + parts[i].length, - ); - const endIndex = - i + 1 < parts.length - ? expandTagLine( - innerContent, - parts[i + 1].index, - parts[i + 1].index + parts[i + 1].length, - )[0] - : innerContent.length; - const branchCondition = parts[i].type === "else" ? null : parts[i].condition!; - - branches.push({ - condition: branchCondition, - content: innerContent.slice(startIndex, endIndex), - }); +/** + * Select one branch from a conditional block. Nested tags remain in the + * selected output and are resolved by the next processing iteration. + */ +function selectConditionalBranch( + content: string, + block: ConditionalBlock, + innerStart: number, + innerEnd: number, + context: TemplateContext, +): string { + let condition: string | null = block.openTag.condition ?? ""; + let branchStart = innerStart; + + for (const branchTag of block.branchTags) { + const [branchEnd] = expandTagLine(content, branchTag.start, branchTag.end); + if (condition === null || evaluateCondition(condition, context)) { + return content.slice(branchStart, branchEnd); } + + const [, nextBranchStart] = expandTagLine(content, branchTag.start, branchTag.end); + branchStart = nextBranchStart; + condition = branchTag.type === "else" ? null : (branchTag.condition ?? ""); } - return branches; + if (condition === null || evaluateCondition(condition, context)) { + return content.slice(branchStart, innerEnd); + } + + return ""; } /** @@ -358,68 +391,39 @@ function processConditionals(content: string, context: TemplateContext): string const maxIterations = 100; // Prevent infinite loops // Keep processing until no more conditionals are found - // Process innermost conditionals first (ones without nested ifs) while (iterations < maxIterations) { - const ifRegex = - /(\{%-\s*if\s+(.+?)\s*-?%\}|\{%\s*if\s+(.+?)\s*-%\}|\{%\s*if\s+(.+?)\s*%\})([\s\S]*?)(\{%-?\s*endif\s*-?%\})/; - const match = ifRegex.exec(result); - - if (!match) { + const block = findConditionalBlock(result); + if (!block) { break; } - const fullMatch = match[0]; - const openTag = match[1]; - const condition = match[2] || match[3] || match[4]; - const closeTag = match[6]; + const { openTag, closeTag } = block; // Locate the open and close tags, then expand each to swallow its whole // line when it stands alone. Everything between them is the branch body. - const openStart = match.index; - const closeEnd = openStart + fullMatch.length; - const [blockStart, innerStart, openAlone] = expandTagLine( - result, - openStart, - openStart + openTag.length, - ); - const [innerEnd, blockEnd, closeAlone] = expandTagLine( - result, - closeEnd - closeTag.length, - closeEnd, - ); - - const innerContent = result.slice(innerStart, innerEnd); - // Parse branches - const branches = parseConditionalBranches(condition, innerContent); - - // Evaluate branches and find the first matching one - // A null condition is the else branch, which always matches once reached - let output = ""; - for (const branch of branches) { - if (branch.condition === null || evaluateCondition(branch.condition, context)) { - output = branch.content; - break; - } - } + const [blockStart, innerStart, openAlone] = expandTagLine(result, openTag.start, openTag.end); + const [innerEnd, blockEnd, closeAlone] = expandTagLine(result, closeTag.start, closeTag.end); + + let output = selectConditionalBranch(result, block, innerStart, innerEnd, context); // Explicit whitespace markers only matter for inline tags. A standalone // tag has already had its whole line removed, leaving nothing to strip. // These apply to the selected branch, since that is what ends up abutting // the tag in the output. - if (!openAlone && openTag.endsWith("-%}")) output = output.replace(/^\s*/, ""); - if (!closeAlone && closeTag.startsWith("{%-")) output = output.replace(/\s*$/, ""); + if (!openAlone && openTag.text.endsWith("-%}")) output = output.replace(/^\s*/, ""); + if (!closeAlone && closeTag.text.startsWith("{%-")) output = output.replace(/\s*$/, ""); let start = blockStart; let end = blockEnd; // {%- ... %} also eats whitespace preceding the opening tag - if (!openAlone && openTag.startsWith("{%-")) { + if (!openAlone && openTag.text.startsWith("{%-")) { const before = result.slice(0, start); start -= before.length - before.replace(/\s*$/, "").length; } // ... -%} also eats whitespace following the closing tag - if (!closeAlone && closeTag.endsWith("-%}")) { + if (!closeAlone && closeTag.text.endsWith("-%}")) { const after = result.slice(end); end += after.length - after.replace(/^\s*/, "").length; } @@ -467,8 +471,15 @@ function processVariables(content: string, context: TemplateContext): string { */ export function processTemplates(content: string, filePath?: string): string { const context = createTemplateContext(); + const rawBlocks: string[] = []; - let result = content; + let result = content.replace( + /\{%-?\s*raw\s*-?%\}([\s\S]*?)\{%-?\s*endraw\s*-?%\}/g, + (_, body: string) => { + const index = rawBlocks.push(body) - 1; + return `@@SVDOCRAW${index}@@`; + }, + ); // Process includes first (they may contain other template syntax) if (filePath) { @@ -484,5 +495,5 @@ export function processTemplates(content: string, filePath?: string): string { // Process << >> variables result = processVariables(result, context); - return result; + return result.replace(/@@SVDOCRAW(\d+)@@/g, (_, index: string) => rawBlocks[Number(index)] ?? ""); } diff --git a/svdoc/src/lib/helpers/urls.ts b/svdoc/src/lib/helpers/urls.ts index 909ead14..1714ff0c 100644 --- a/svdoc/src/lib/helpers/urls.ts +++ b/svdoc/src/lib/helpers/urls.ts @@ -160,3 +160,28 @@ export function transformMarkdownHref(href: string, basePath: string, isReadme: return anchor ? `${prefixed}#${anchor}` : prefixed; } + +/** + * Return the Markdown representation of a canonical documentation URL. + * + * The root README is exposed as `/index.md`; using `/README.md` would imply a + * source-file route and conflict with the directory-index convention used by + * every other page. + */ +export function markdownAlternativeUrl(href: string): string { + if (isExternalHref(href)) return href; + + const [pathWithQuery, anchor] = href.split("#"); + const [pathPart, query] = pathWithQuery.split("?"); + if (HAS_FILE_EXT_RE.test(pathPart.replace(/\/+$/, ""))) return href; + + const rawPath = stripBase(pathPart || "/"); + const normalizedPath = collapseSlashes(rawPath || "/"); + if (normalizedPath.startsWith("/tags/")) return href; + const markdownPath = + normalizedPath === "/" ? "/index.md" : `${normalizedPath.replace(/\/+$/, "")}.md`; + + const url = withBase(markdownPath); + const querySuffix = query ? `?${query}` : ""; + return anchor ? `${url}${querySuffix}#${anchor}` : `${url}${querySuffix}`; +} diff --git a/svdoc/src/lib/llms.test.ts b/svdoc/src/lib/llms.test.ts new file mode 100644 index 00000000..6fe3136e --- /dev/null +++ b/svdoc/src/lib/llms.test.ts @@ -0,0 +1,270 @@ +import { describe, expect, test } from "bun:test"; +import type { Tokens } from "marked"; +import { orderLlmsFullDocuments, type NavItem } from "./content-store"; +import { + curateLlmsSections, + formatLlmsFull, + formatLlmsIndex, + MARKDOWN_RESPONSE_HEADERS, + markdownPath, + serializeMarkdown, + type LlmsDocument, +} from "./llms"; +import type { AdmonitionToken, AnnotatedCodeToken, ContentTabsToken } from "./types/tokens"; + +function text(value: string): Tokens.Text { + return { type: "text", raw: value, text: value }; +} + +function page( + urlPath: string, + title: string, + options: { + description?: string; + summary?: string; + hide?: string[]; + hiddenFromNavigation?: boolean; + section?: string; + order?: number; + } = {}, +): LlmsDocument { + return { + urlPath, + title, + description: options.description, + summary: options.summary, + hide: options.hide ?? [], + hiddenFromNavigation: options.hiddenFromNavigation, + tokens: [], + attributes: + options.section === undefined || options.order === undefined + ? {} + : { llms: { section: options.section, order: options.order } }, + }; +} + +describe("serializeMarkdown", () => { + test("uses index.md for the root README", () => { + expect(markdownPath("/")).toBe("/index.md"); + }); + + test("keeps generated tag routes on their HTML URLs", () => { + expect(markdownPath("/tags/how-to")).toBe("/tags/how-to"); + }); + + test("rewrites documentation links while preserving anchors and image assets", () => { + const link: Tokens.Link = { + type: "link", + raw: "[Application](/workloads/application/#ingresses)", + href: "/workloads/application/#ingresses", + text: "Application", + tokens: [text("Application")], + }; + const image: Tokens.Image = { + type: "image", + raw: "![Logo](/assets/logo.svg)", + href: "/assets/logo.svg", + title: null, + text: "Logo", + tokens: [], + }; + const asset: Tokens.Link = { + type: "link", + raw: "[Download](/assets/guide.pdf)", + href: "/assets/guide.pdf", + text: "Download", + tokens: [text("Download")], + }; + + expect( + serializeMarkdown([ + { + type: "paragraph", + raw: "", + text: "", + tokens: [link, text(" "), image, text(" "), asset], + }, + ]), + ).toBe( + "[Application](/workloads/application.md#ingresses) ![Logo](/assets/logo.svg) [Download](/assets/guide.pdf)\n", + ); + }); + + test("rewrites links nested in block text tokens", () => { + const link: Tokens.Link = { + type: "link", + raw: "[Application](workloads/application.md)", + href: "/workloads/application/", + text: "Application", + tokens: [text("Application")], + }; + const blockText: Tokens.Text = { + type: "text", + raw: "[Application](workloads/application.md)", + text: "[Application](workloads/application.md)", + tokens: [link], + }; + + expect( + serializeMarkdown([ + { + type: "list", + raw: "", + ordered: false, + start: "", + loose: false, + items: [ + { + type: "list_item", + raw: "", + task: false, + loose: false, + text: "", + tokens: [blockText], + }, + ], + }, + ]), + ).toBe("- [Application](/workloads/application.md)\n"); + }); + + test("preserves code and converts admonitions and tabs to portable Markdown", () => { + const code: AnnotatedCodeToken = { + type: "code", + raw: "```yaml\nkey: value\n```", + lang: 'yaml title="app.yaml"', + text: "key: value", + annotations: [ + [{ type: "paragraph", raw: "", text: "", tokens: [text("This value matters.")] }], + ], + }; + const admonition: AdmonitionToken = { + type: "admonition", + raw: "", + admonitionType: "note", + title: "Important", + collapsible: true, + open: true, + tokens: [{ type: "paragraph", raw: "", text: "", tokens: [text("Read this.")] }], + }; + const tabs: ContentTabsToken = { + type: "content_tabs", + raw: "", + tabs: [ + { + type: "content_tab", + raw: "", + label: "CLI", + tokens: [{ type: "paragraph", raw: "", text: "", tokens: [text("Run it.")] }], + }, + { + type: "content_tab", + raw: "", + label: "Web", + tokens: [{ type: "paragraph", raw: "", text: "", tokens: [text("Open it.")] }], + }, + ], + }; + + expect(serializeMarkdown([code, admonition, tabs])).toBe( + "```yaml\nkey: value\n```\n\n1. This value matters.\n\n> **Important**\n> Read this.\n\n**CLI**\n\nRun it.\n\n**Web**\n\nOpen it.\n", + ); + }); + + test("keeps useful HTML blocks and inline HTML", () => { + const block: Tokens.HTML = { + type: "html", + raw: "
...", + pre: false, + text: "
More
", + block: true, + }; + const inline: Tokens.HTML = { + type: "html", + raw: "Note", + pre: false, + text: "Note", + block: false, + }; + + expect( + serializeMarkdown([ + block, + { + type: "paragraph", + raw: "", + text: "", + tokens: [text("A "), inline], + }, + ]), + ).toBe("
More
\n\nA Note\n"); + }); +}); + +describe("llms.txt curation", () => { + test("groups sections by first global order and falls back to summaries", () => { + const pages = [ + page("/logs", "Logs", { + section: "Observe", + order: 30, + summary: "Inspect application logs.", + }), + page("/start", "Start", { section: "Core", order: 10, description: "Start here." }), + page("/metrics", "Metrics", { + section: "Observe", + order: 20, + description: "Measure services.", + }), + ]; + + expect( + curateLlmsSections(pages).map((section) => [ + section.name, + section.pages.map((item) => item.title), + ]), + ).toEqual([ + ["Core", ["Start"]], + ["Observe", ["Metrics", "Logs"]], + ]); + expect(formatLlmsIndex(pages)).toContain("- [Logs](/logs.md): Inspect application logs."); + }); + + test("collapses description whitespace to one line", () => { + const pages = [ + page("/start", "Start", { + section: "Core", + order: 10, + summary: "Start here.\nThen deploy.", + }), + ]; + + expect(formatLlmsIndex(pages)).toContain("- [Start](/start.md): Start here. Then deploy."); + }); +}); + +describe("llms-full.txt ordering", () => { + test("follows navigation, excludes hidden pages, and appends unlisted pages deterministically", () => { + const pages = [ + page("/z-unlisted", "Z"), + page("/visible", "Visible"), + page("/hidden", "Hidden", { hide: ["navigation"] }), + page("/partial", "Partial", { hiddenFromNavigation: true }), + page("/a-unlisted", "A"), + ]; + const navigation: NavItem[] = [ + { title: "Visible", href: "/visible", hasContent: true }, + { title: "Hidden", href: "/hidden", hasContent: true }, + ]; + + const ordered = orderLlmsFullDocuments(pages, navigation); + expect(ordered.map((item) => item.title)).toEqual(["Visible", "A", "Z"]); + expect(formatLlmsFull(ordered)).toContain("Source: [/visible.md](/visible.md)"); + }); +}); + +test("Markdown endpoint responses are prerender-cacheable UTF-8 text", () => { + expect(MARKDOWN_RESPONSE_HEADERS).toEqual({ + "Cache-Control": "public, max-age=3600", + "Content-Type": "text/markdown; charset=utf-8", + }); +}); diff --git a/svdoc/src/lib/llms.ts b/svdoc/src/lib/llms.ts new file mode 100644 index 00000000..82866472 --- /dev/null +++ b/svdoc/src/lib/llms.ts @@ -0,0 +1,544 @@ +import type { Token, Tokens, TokensList } from "marked"; +import { markdownAlternativeUrl, withBase } from "./helpers/urls"; +import type { + AdmonitionToken, + AnnotatedCodeToken, + Attributes, + ContentTabToken, + ContentTabsToken, + DefinitionListItemToken, + DefinitionListToken, + DefinitionToken, + FootnoteToken, +} from "./types/tokens"; + +export interface LlmsDocument { + urlPath: string; + title: string; + description?: string; + summary?: string; + hide: string[]; + hiddenFromNavigation?: boolean; + tokens: Token[] | TokensList; + attributes: Attributes; +} + +export interface LlmsSection { + name: string; + pages: LlmsDocument[]; +} + +export const MARKDOWN_RESPONSE_HEADERS = { + "Cache-Control": "public, max-age=3600", + "Content-Type": "text/markdown; charset=utf-8", +}; + +type FootnoteReferenceToken = Tokens.Generic & { label: string; id: string }; +type FootnotesToken = Tokens.Generic & { items: FootnoteToken[] }; +type HtmlWithMarkdownToken = Tokens.Generic & { + openTag: string; + closeTag: string; + innerTokens: Token[]; +}; + +function isRecord(value: unknown): value is Record { + return typeof value === "object" && value !== null; +} + +function isToken(value: unknown): value is Token { + return isRecord(value) && typeof value.type === "string" && typeof value.raw === "string"; +} + +function isTokenArray(value: unknown): value is Token[] { + return Array.isArray(value) && value.every(isToken); +} + +function isTokenMatrix(value: unknown): value is Token[][] { + return Array.isArray(value) && value.every(isTokenArray); +} + +function isAdmonitionToken(token: Token): token is AdmonitionToken { + if (!isRecord(token)) return false; + return ( + token.type === "admonition" && + typeof token.admonitionType === "string" && + typeof token.title === "string" && + typeof token.collapsible === "boolean" && + typeof token.open === "boolean" && + isTokenArray(token.tokens) + ); +} + +function isContentTabsToken(token: Token): token is ContentTabsToken { + if (!isRecord(token)) return false; + return ( + token.type === "content_tabs" && Array.isArray(token.tabs) && token.tabs.every(isContentTab) + ); +} + +function isContentTab(value: unknown): value is ContentTabToken { + if (!isRecord(value)) return false; + return ( + value.type === "content_tab" && typeof value.label === "string" && isTokenArray(value.tokens) + ); +} + +function isContentTabToken(token: Token): token is ContentTabToken { + return isContentTab(token); +} + +function isDefinitionToken(value: unknown): value is DefinitionToken { + return isRecord(value) && value.type === "definition" && isTokenArray(value.tokens); +} + +function isDefinitionListItem(value: unknown): value is DefinitionListItemToken { + return ( + isRecord(value) && + value.type === "def_list_item" && + isTokenArray(value.termTokens) && + Array.isArray(value.definitions) && + value.definitions.every(isDefinitionToken) + ); +} + +function isDefinitionListToken(token: Token): token is DefinitionListToken { + if (!isRecord(token)) return false; + return ( + token.type === "def_list" && + Array.isArray(token.items) && + token.items.every(isDefinitionListItem) + ); +} + +function isDefinitionListItemToken(token: Token): token is DefinitionListItemToken { + return isDefinitionListItem(token); +} + +function isFootnote(value: unknown): value is FootnoteToken { + if (!isRecord(value)) return false; + return ( + value.type === "footnote" && + typeof value.id === "string" && + typeof value.label === "string" && + isTokenArray(value.tokens) + ); +} + +function isFootnoteToken(token: Token): token is FootnoteToken { + return isFootnote(token); +} + +function isFootnoteReferenceToken(token: Token): token is FootnoteReferenceToken { + if (!isRecord(token)) return false; + return ( + token.type === "footnoteRef" && typeof token.label === "string" && typeof token.id === "string" + ); +} + +function isFootnotesToken(token: Token): token is FootnotesToken { + if (!isRecord(token)) return false; + return token.type === "footnotes" && Array.isArray(token.items) && token.items.every(isFootnote); +} + +function isHtmlWithMarkdownToken(token: Token): token is HtmlWithMarkdownToken { + if (!isRecord(token)) return false; + return ( + token.type === "html_with_markdown" && + typeof token.openTag === "string" && + typeof token.closeTag === "string" && + isTokenArray(token.innerTokens) + ); +} + +function isBlockquoteToken(token: Token): token is Tokens.Blockquote { + return isRecord(token) && token.type === "blockquote" && isTokenArray(token.tokens); +} + +function isCheckboxToken(token: Token): token is Tokens.Checkbox { + return isRecord(token) && token.type === "checkbox" && typeof token.checked === "boolean"; +} + +function isCodeToken(token: Token): token is Tokens.Code { + return ( + isRecord(token) && + token.type === "code" && + typeof token.text === "string" && + (token.lang === undefined || typeof token.lang === "string") + ); +} + +function isCodespanToken(token: Token): token is Tokens.Codespan { + return isRecord(token) && token.type === "codespan" && typeof token.text === "string"; +} + +function isDelimitedToken( + token: Token, + type: "del" | "em" | "strong", +): token is Tokens.Del | Tokens.Em | Tokens.Strong { + return isRecord(token) && token.type === type && isTokenArray(token.tokens); +} + +function isTextToken(token: Token): token is Tokens.Escape | Tokens.Text { + return ( + isRecord(token) && + (token.type === "escape" || token.type === "text") && + typeof token.text === "string" + ); +} + +function isHeadingToken(token: Token): token is Tokens.Heading { + return ( + isRecord(token) && + token.type === "heading" && + typeof token.depth === "number" && + isTokenArray(token.tokens) + ); +} + +function isHtmlToken(token: Token): token is Tokens.HTML { + return ( + isRecord(token) && + token.type === "html" && + typeof token.text === "string" && + typeof token.block === "boolean" + ); +} + +function isImageToken(token: Token): token is Tokens.Image { + return ( + isRecord(token) && + token.type === "image" && + typeof token.href === "string" && + (token.title === null || typeof token.title === "string") && + typeof token.text === "string" + ); +} + +function isLinkToken(token: Token): token is Tokens.Link { + return ( + isRecord(token) && + token.type === "link" && + typeof token.href === "string" && + (token.title === undefined || token.title === null || typeof token.title === "string") && + isTokenArray(token.tokens) + ); +} + +function isListItemToken(token: Token): token is Tokens.ListItem { + return isRecord(token) && token.type === "list_item" && isTokenArray(token.tokens); +} + +function isListToken(token: Token): token is Tokens.List { + return ( + isRecord(token) && + token.type === "list" && + typeof token.ordered === "boolean" && + (typeof token.start === "number" || token.start === "") && + Array.isArray(token.items) && + token.items.every(isListItemToken) + ); +} + +function isParagraphToken(token: Token): token is Tokens.Paragraph { + return isRecord(token) && token.type === "paragraph" && isTokenArray(token.tokens); +} + +function isTableCell(value: unknown): value is Tokens.TableCell { + return isRecord(value) && isTokenArray(value.tokens); +} + +function isTableToken(token: Token): token is Tokens.Table { + return ( + isRecord(token) && + token.type === "table" && + Array.isArray(token.align) && + token.align.every( + (align: unknown) => + align === null || align === "left" || align === "right" || align === "center", + ) && + Array.isArray(token.header) && + token.header.every(isTableCell) && + Array.isArray(token.rows) && + token.rows.every((row: unknown) => Array.isArray(row) && row.every(isTableCell)) + ); +} + +function escapeLinkTitle(title: string): string { + return title.replaceAll('"', '\\"'); +} + +function isAnnotatedCodeToken(token: Tokens.Code): token is AnnotatedCodeToken { + return !("annotations" in token) || (isRecord(token) && isTokenMatrix(token.annotations)); +} + +function serializeInline(tokens: Token[]): string { + return tokens.map(serializeToken).join("").trim(); +} + +function indent(text: string, prefix: string): string { + return text + .split("\n") + .map((line) => (line ? `${prefix}${line}` : line)) + .join("\n"); +} + +function serializeCode(token: Tokens.Code): string { + const fence = token.text.includes("```") ? "~~~" : "```"; + const language = token.lang?.split(/\s+/, 1)[0] ?? ""; + return `${fence}${language}\n${token.text}\n${fence}\n\n`; +} + +function serializeCodeAnnotations(token: AnnotatedCodeToken): string { + if (!token.annotations || token.annotations.length === 0) return ""; + return ( + token.annotations + .map((annotation, index) => `${index + 1}. ${serializeTokens(annotation).trim()}`) + .join("\n") + "\n\n" + ); +} + +function serializeList(token: Tokens.List): string { + return ( + token.items + .map((item, index) => { + const marker = token.ordered ? `${Number(token.start || 1) + index}.` : "-"; + const content = serializeTokens(item.tokens).trim(); + return `${marker} ${indent(content, " ").trimStart()}`; + }) + .join("\n") + "\n\n" + ); +} + +function serializeTable(token: Tokens.Table): string { + const cell = (value: Tokens.TableCell) => + serializeInline(value.tokens).replaceAll("|", "\\|").replaceAll("\n", "
"); + const header = token.header.map(cell); + const alignment = token.align.map((align) => { + if (align === "left") return ":---"; + if (align === "right") return "---:"; + if (align === "center") return ":---:"; + return "---"; + }); + const rows = token.rows.map((row) => row.map(cell)); + return [ + `| ${header.join(" | ")} |`, + `| ${alignment.join(" | ")} |`, + ...rows.map((row) => `| ${row.join(" | ")} |`), + "", + "", + ].join("\n"); +} + +function serializeAdmonition(token: AdmonitionToken): string { + const title = token.title || token.admonitionType; + const content = serializeTokens(token.tokens).trim(); + return `> **${title}**\n${indent(content, "> ")}\n\n`; +} + +function serializeContentTabs(token: ContentTabsToken): string { + return ( + token.tabs + .map((tab) => `**${tab.label}**\n\n${serializeTokens(tab.tokens).trim()}`) + .join("\n\n") + "\n\n" + ); +} + +function serializeContentTab(token: ContentTabToken): string { + return `**${token.label}**\n\n${serializeTokens(token.tokens).trim()}\n\n`; +} + +function serializeDefinitionListItem(token: DefinitionListItemToken): string { + const term = serializeInline(token.termTokens); + const definitions = token.definitions + .map((definition) => serializeTokens(definition.tokens).trim()) + .join("\n\n"); + return `- **${term}**: ${indent(definitions, " ").trimStart()}\n\n`; +} + +function serializeDefinitionList(token: DefinitionListToken): string { + return token.items.map((item) => serializeDefinitionListItem(item).trim()).join("\n") + "\n\n"; +} + +function serializeTokens(tokens: Token[]): string { + return tokens.map(serializeToken).join(""); +} + +function isPresentationWrapper(openTag: string): boolean { + const tag = openTag.trimStart().toLowerCase(); + return tag.startsWith(" `[^${item.label}]: ${serializeTokens(item.tokens).trim()}`) + .join("\n") + "\n\n" + ); + } + if (isHtmlWithMarkdownToken(token)) { + if (isPresentationWrapper(token.openTag)) { + return `${serializeTokens(token.innerTokens).trim()}\n\n`; + } + return `${token.openTag}\n${serializeTokens(token.innerTokens).trim()}\n${token.closeTag}\n\n`; + } + + if (isBlockquoteToken(token)) { + return `${indent(serializeTokens(token.tokens).trim(), "> ")}\n\n`; + } + if (token.type === "br") return " \n"; + if (isCheckboxToken(token)) return token.checked ? "[x] " : "[ ] "; + if (isCodeToken(token)) { + return isAnnotatedCodeToken(token) + ? `${serializeCode(token)}${serializeCodeAnnotations(token)}` + : serializeCode(token); + } + if (isCodespanToken(token)) return `\`${token.text}\``; + if (token.type === "def" || token.type === "space") return ""; + if (isDelimitedToken(token, "del")) return `~~${serializeInline(token.tokens)}~~`; + if (isDelimitedToken(token, "em")) return `*${serializeInline(token.tokens)}*`; + if (isTextToken(token)) { + return "tokens" in token && isTokenArray(token.tokens) + ? serializeTokens(token.tokens) + : token.text; + } + if (isHeadingToken(token)) + return `${"#".repeat(token.depth)} ${serializeInline(token.tokens)}\n\n`; + if (token.type === "hr") return "---\n\n"; + if (isHtmlToken(token)) return token.block ? `${token.text}\n\n` : token.text; + if (isImageToken(token)) { + const title = token.title ? ` "${escapeLinkTitle(token.title)}"` : ""; + return `![${token.text}](${token.href}${title})`; + } + if (isLinkToken(token)) { + const title = token.title ? ` "${escapeLinkTitle(token.title)}"` : ""; + return `[${serializeInline(token.tokens)}](${markdownAlternativeUrl(token.href)}${title})`; + } + if (isListToken(token)) return serializeList(token); + if (isListItemToken(token)) return serializeTokens(token.tokens); + if (isParagraphToken(token)) return `${serializeInline(token.tokens)}\n\n`; + if (isDelimitedToken(token, "strong")) return `**${serializeInline(token.tokens)}**`; + if (isTableToken(token)) return serializeTable(token); + + // A new token must be represented deliberately rather than silently + // discarded. Marked's generic extension token type carries `raw`. + return token.raw; +} + +/** + * Serialize processed Marked tokens into portable Markdown. Templates, + * includes, link normalization, and custom block parsing have already run + * when the content store supplies these tokens. + */ +export function serializeMarkdown(tokens: Token[] | TokensList): string { + return `${serializeTokens(tokens) + .replace(/\n{3,}/g, "\n\n") + .trim()}\n`; +} + +export function markdownPath(urlPath: string): string { + return markdownAlternativeUrl(withBase(urlPath)); +} + +function compareCuratedPages(a: LlmsDocument, b: LlmsDocument): number { + const aOrder = a.attributes.llms?.order ?? Number.POSITIVE_INFINITY; + const bOrder = b.attributes.llms?.order ?? Number.POSITIVE_INFINITY; + return aOrder - bOrder || a.urlPath.localeCompare(b.urlPath); +} + +/** + * Group explicitly curated pages by section. A section's position is the + * lowest global order of any page it contains; ties are deterministic. + */ +export function curateLlmsSections(documents: LlmsDocument[]): LlmsSection[] { + const pages = documents.filter((document) => { + const metadata = document.attributes.llms; + return ( + metadata !== undefined && metadata.section.trim() !== "" && Number.isFinite(metadata.order) + ); + }); + const sections = new Map(); + for (const page of pages) { + const metadata = page.attributes.llms; + if (!metadata) continue; + const section = metadata.section; + sections.set(section, [...(sections.get(section) ?? []), page]); + } + + return [...sections.entries()] + .map(([name, sectionPages]) => ({ + name, + pages: sectionPages.sort(compareCuratedPages), + })) + .sort((a, b) => compareCuratedPages(a.pages[0], b.pages[0]) || a.name.localeCompare(b.name)); +} + +function pageDescription(page: LlmsDocument): string | undefined { + return (page.description ?? page.summary)?.replace(/\s+/g, " ").trim(); +} + +export function formatLlmsIndex(documents: LlmsDocument[]): string { + const sections = curateLlmsSections(documents); + const lines = [ + "# Nais Developer Documentation", + "", + "> Documentation for Nais, the application platform for developing and running services safely.", + "", + ]; + + for (const section of sections) { + lines.push(`## ${section.name}`, ""); + for (const page of section.pages) { + const description = pageDescription(page); + lines.push( + `- [${page.title}](${markdownPath(page.urlPath)})${description ? `: ${description}` : ""}`, + ); + } + lines.push(""); + } + + lines.push( + "## Complete documentation", + "", + `For every visible documentation page, see [the full documentation](${withBase("/llms-full.txt")}).`, + "", + ); + return lines.join("\n"); +} + +export function formatLlmsFull(documents: LlmsDocument[]): string { + const lines = [ + "# Nais Developer Documentation — Full documentation", + "", + "> All navigation-visible Nais documentation for this tenant, in site navigation order.", + "", + ]; + + for (const document of documents) { + const canonical = withBase(document.urlPath === "/" ? "/" : `${document.urlPath}/`); + lines.push( + `## ${document.title}`, + "", + `Source: [${markdownPath(document.urlPath)}](${markdownPath(document.urlPath)})`, + "", + `Canonical: [${canonical}](${canonical})`, + "", + serializeMarkdown(document.tokens).trim(), + "", + "---", + "", + ); + } + + return lines.join("\n"); +} diff --git a/svdoc/src/lib/markdown.ts b/svdoc/src/lib/markdown.ts index 5e742b24..424446a0 100644 --- a/svdoc/src/lib/markdown.ts +++ b/svdoc/src/lib/markdown.ts @@ -1,12 +1,14 @@ import fm from "front-matter"; import { gemoji } from "gemoji"; import { Marked, type Token, type Tokens, type TokensList } from "marked"; +import { readFile } from "node:fs/promises"; import { getGitInfo } from "./helpers/git"; import { getIconSvg, isIconShortcode } from "./helpers/icons"; import { preHighlightCodeTokens } from "./helpers/shiki"; import { processTemplates } from "./helpers/templates"; import type { AdmonitionToken, + AnnotatedCodeToken, Attributes, ContentTabsToken, ContentTabToken, @@ -582,6 +584,19 @@ function processHtmlMarkdownBlocks(tokens: Token[] | TokensList): Token[] | Toke * Process code blocks to attach annotation content from following ordered lists. * Annotations in code (like `# (1)`) reference items in the list that follows. */ +function isCodeToken(token: Token): token is Tokens.Code { + return token.type === "code" && typeof token.text === "string"; +} + +function isFirstOrderedList(token: Token): token is Tokens.List { + return ( + token.type === "list" && + token.ordered === true && + token.start === 1 && + Array.isArray(token.items) + ); +} + function processCodeAnnotations(tokens: Token[] | TokensList): Token[] { const result: Token[] = []; let i = 0; @@ -589,8 +604,8 @@ function processCodeAnnotations(tokens: Token[] | TokensList): Token[] { while (i < tokens.length) { const token = tokens[i]; - if (token.type === "code") { - const codeToken = token as Tokens.Code; + if (isCodeToken(token)) { + const codeToken: AnnotatedCodeToken = token; // Look for an ordered list following this code block (skip space tokens) let nextIdx = i + 1; @@ -599,11 +614,10 @@ function processCodeAnnotations(tokens: Token[] | TokensList): Token[] { } if (nextIdx < tokens.length) { - const nextToken = tokens[nextIdx] as Tokens.List; - if (nextToken.type === "list" && nextToken.ordered && nextToken.start === 1) { + const nextToken = tokens[nextIdx]; + if (isFirstOrderedList(nextToken)) { // Store annotations on the code token - (codeToken as Tokens.Code & { annotations?: Token[][] }).annotations = - nextToken.items.map((item) => item.tokens); + codeToken.annotations = nextToken.items.map((item) => item.tokens); // Skip the annotation list i = nextIdx; } @@ -692,7 +706,7 @@ function processFootnotes(tokens: Token[] | TokensList): Token[] | TokensList { export async function readMarkdownFile( path: string, ): Promise<{ tokens: Token[] | TokensList; attributes: Attributes }> { - const source = await Bun.file(path).text(); + const source = await readFile(path, "utf-8"); const { attributes, body } = fm(source); // Get git information for the file diff --git a/svdoc/src/lib/types/tokens.ts b/svdoc/src/lib/types/tokens.ts index 00ca2eff..74bcaa3a 100644 --- a/svdoc/src/lib/types/tokens.ts +++ b/svdoc/src/lib/types/tokens.ts @@ -1,10 +1,10 @@ -import type { Token } from "marked"; +import type { Token, Tokens } from "marked"; /** * Admonition token for Material for MkDocs style admonitions * Syntax: !!! type "title" or ??? type "title" (collapsible) or ???+ type "title" (collapsible, open) */ -export interface AdmonitionToken { +export interface AdmonitionToken extends Tokens.Generic { type: "admonition"; raw: string; admonitionType: string; @@ -18,7 +18,7 @@ export interface AdmonitionToken { * Content tab token for Material for MkDocs style tabs * Syntax: === "Tab Title" */ -export interface ContentTabToken { +export interface ContentTabToken extends Tokens.Generic { type: "content_tab"; raw: string; label: string; @@ -28,7 +28,7 @@ export interface ContentTabToken { /** * Container for multiple content tabs */ -export interface ContentTabsToken { +export interface ContentTabsToken extends Tokens.Generic { type: "content_tabs"; raw: string; tabs: ContentTabToken[]; @@ -37,7 +37,7 @@ export interface ContentTabsToken { /** * Footnote definition token ([^1]: content) */ -export interface FootnoteToken { +export interface FootnoteToken extends Tokens.Generic { type: "footnote"; raw: string; id: string; @@ -48,7 +48,7 @@ export interface FootnoteToken { /** * Definition list item (term + definitions) */ -export interface DefinitionListItemToken { +export interface DefinitionListItemToken extends Tokens.Generic { type: "def_list_item"; raw: string; term: string; @@ -59,7 +59,7 @@ export interface DefinitionListItemToken { /** * Single definition in a definition list */ -export interface DefinitionToken { +export interface DefinitionToken extends Tokens.Generic { type: "definition"; raw: string; tokens: Token[]; @@ -68,18 +68,28 @@ export interface DefinitionToken { /** * Definition list container */ -export interface DefinitionListToken { +export interface DefinitionListToken extends Tokens.Generic { type: "def_list"; raw: string; items: DefinitionListItemToken[]; } +/** A code block with prose annotations extracted from the following list. */ +export interface AnnotatedCodeToken extends Tokens.Code { + annotations?: Token[][]; +} + /** * Frontmatter attributes extracted from markdown files */ export interface Attributes { title?: string; description?: string; + /** Curated placement in llms.txt. */ + llms?: { + section: string; + order: number; + }; tags?: string[]; hide?: string[]; /** Conditional visibility based on tenant */ diff --git a/svdoc/src/routes/[...path].md/+server.ts b/svdoc/src/routes/[...path].md/+server.ts new file mode 100644 index 00000000..76200dd2 --- /dev/null +++ b/svdoc/src/routes/[...path].md/+server.ts @@ -0,0 +1,27 @@ +import { contentStore } from "$lib/content-store"; +import { MARKDOWN_RESPONSE_HEADERS, serializeMarkdown } from "$lib/llms"; +import { error } from "@sveltejs/kit"; +import type { EntryGenerator, RequestHandler } from "./$types"; + +/** + * A `.md` suffix route outranks the HTML `[...path]` route, so Markdown + * representations remain unambiguous while retaining the site's clean URLs. + */ +export const entries: EntryGenerator = async () => { + const paths = await contentStore.getMarkdownPaths(); + return paths.map((path) => ({ path })); +}; + +export const prerender = true; + +export const GET: RequestHandler = async ({ params }) => { + const urlPath = params.path === "index" ? "/" : `/${params.path}`; + const document = await contentStore.getDocumentByPath(urlPath); + if (!document) { + error(404, { message: "Page not found" }); + } + + return new Response(serializeMarkdown(document.tokens), { + headers: MARKDOWN_RESPONSE_HEADERS, + }); +}; diff --git a/svdoc/src/routes/[...path]/+page.svelte b/svdoc/src/routes/[...path]/+page.svelte index e1a2bf21..9d46b3d2 100644 --- a/svdoc/src/routes/[...path]/+page.svelte +++ b/svdoc/src/routes/[...path]/+page.svelte @@ -1,6 +1,7 @@