diff --git a/astro.config.mjs b/astro.config.mjs index 3b75936528..d6b50b2bc7 100644 --- a/astro.config.mjs +++ b/astro.config.mjs @@ -11,21 +11,17 @@ import Icons from 'unplugin-icons/vite'; import rehypeSlug from 'rehype-slug'; import rehypeAutolinkHeadings from 'rehype-autolink-headings'; import redirects from './src/config/redirects/index.js'; -import movedPages from './src/config/redirects/moved-pages.json' with { type: 'json' }; import rehypeAccessibleTables from './src/plugins/rehype-accessible-tables.mjs'; import remarkRewriteLocalizedLinks from './src/plugins/remark-rewrite-localized-links.mjs'; import remarkCodeTabs from './src/plugins/remark-code-tabs.mjs'; import rehypeRewriteLocalizedLinks from './src/plugins/rehype-rewrite-localized-links.mjs'; +import { isCanonicalSitemapPage } from './src/utils/sitemap.mjs'; /* https://docs.netlify.com/configure-builds/environment-variables/#read-only-variables */ const NETLIFY_PREVIEW_SITE = process.env.CONTEXT !== 'production' && process.env.DEPLOY_PRIME_URL; const site = NETLIFY_PREVIEW_SITE || 'https://expressjs.com'; -// Unlocalized paths of moved/removed pages. `[...path].astro` emits `noindex` -// redirect stubs for these in every locale; keep those stubs out of the sitemap. -const movedPagePaths = new Set(Object.keys(movedPages)); - // https://astro.build/config export default defineConfig({ redirects, @@ -73,19 +69,9 @@ export default defineConfig({ icon(), react(), sitemap({ - // Only the canonical URLs belong in the sitemap. The catch-all - // `[...path].astro` also emits `noindex` redirect stubs: language-less ones - // (e.g. `/guide/x`) and per-locale ones for moved pages - // (e.g. `/en/resources/middleware/csurf`). Exclude both. - // Keep this locale list in sync with `i18n.locales` below. - filter: (page) => { - const { pathname } = new URL(page); - // The homepage is served at `/` and is the canonical for `/en/`. - if (pathname === '/') return true; - if (!/^\/(de|en|es|fr|it|ja|ko|pt-br|zh-cn|zh-tw)(\/|$)/.test(pathname)) return false; - const unlocalized = pathname.replace(/^\/[a-z-]+/, '').replace(/\/$/, ''); - return !movedPagePaths.has(unlocalized); - }, + // Only the canonical URLs belong in the sitemap. + // Keep `LOCALES` in the filter's module in sync with `i18n.locales` below. + filter: isCanonicalSitemapPage, i18n: { defaultLocale: 'en', locales: { diff --git a/src/plugins/rewrite-localized-links-core.mjs b/src/plugins/rewrite-localized-links-core.mjs index 9534963eeb..33241e6e00 100644 --- a/src/plugins/rewrite-localized-links-core.mjs +++ b/src/plugins/rewrite-localized-links-core.mjs @@ -35,7 +35,7 @@ import { DEFAULT_VERSION } from '../config/versions.ts'; const DEFAULT_PREFIXES = ['guide', 'starter', 'api', 'resources', 'advanced', 'support', 'blog']; /** Sections that carry a version (every other section is language-only). */ -const DEFAULT_VERSIONED_SECTIONS = ['api', 'starter', 'guide', 'advanced']; +export const DEFAULT_VERSIONED_SECTIONS = ['api', 'starter', 'guide', 'advanced']; /** Language used when it cannot be inferred from the source path. */ const DEFAULT_LANG = 'en'; @@ -101,7 +101,7 @@ function collectGlobalPaths(menu) { * Default unversioned targets: the "global" pages declared in the menu. Computed * once so consumers (and `astro.config`) don't have to wire this up themselves. */ -const DEFAULT_UNVERSIONED_PATHS = collectGlobalPaths(docsMenu); +export const DEFAULT_UNVERSIONED_PATHS = collectGlobalPaths(docsMenu); /** * Validates whether a path segment matches version format (e.g. `5x`, `4x`). diff --git a/src/utils/sitemap.mjs b/src/utils/sitemap.mjs new file mode 100644 index 0000000000..9f2ae4565f --- /dev/null +++ b/src/utils/sitemap.mjs @@ -0,0 +1,44 @@ +import movedPages from '../config/redirects/moved-pages.json' with { type: 'json' }; +import { + DEFAULT_UNVERSIONED_PATHS, + DEFAULT_VERSIONED_SECTIONS, +} from '../plugins/rewrite-localized-links-core.mjs'; + +// Keep this locale list in sync with the sitemap's `i18n.locales` in `astro.config.mjs`. +const LOCALES = new Set(['de', 'en', 'es', 'fr', 'it', 'ja', 'ko', 'pt-br', 'zh-cn', 'zh-tw']); +const VERSIONED_SECTIONS = new Set(DEFAULT_VERSIONED_SECTIONS); +const UNVERSIONED_PATHS = new Set(DEFAULT_UNVERSIONED_PATHS); + +/** + * Whether a generated page is the canonical URL of its content, mirroring the + * `rel="canonical"` set by `Layout.astro` and `[...slug].astro`. Excludes: + * - `/en/`, because the English homepage is served at `/`. + * - The `noindex` redirect stubs from `[...path].astro`: language-less ones + * (e.g. `/guide/x`) and per-locale ones for moved pages + * (e.g. `/en/resources/middleware/csurf`). + * - Unversioned copies of the default version (e.g. `/es/guide/routing`), whose + * canonical is the versioned URL (e.g. `/es/5x/guide/routing`). "Global" pages + * (e.g. `/es/guide/migrating-5`) only exist unversioned, so they stay. + * - Other locales' copies of the untranslated blog and API pages + * (e.g. `/es/5x/api/express`), whose canonical is the English URL. + * + * @param {string} page Absolute page URL. + * @returns {boolean} + */ +export function isCanonicalSitemapPage(page) { + const { pathname } = new URL(page); + if (pathname === '/') return true; + + const [locale, ...segments] = pathname.split('/').filter(Boolean); + if (!LOCALES.has(locale)) return false; + + const slug = segments.join('/'); + if (!slug) return locale !== 'en'; + if (Object.hasOwn(movedPages, `/${slug}`)) return false; + + const [section, subsection] = segments; + if (VERSIONED_SECTIONS.has(section) && !UNVERSIONED_PATHS.has(slug)) return false; + + const isUntranslated = section === 'blog' || (/^\d+x$/.test(section) && subsection === 'api'); + return locale === 'en' || !isUntranslated; +} diff --git a/tests/unit/sitemap.test.mjs b/tests/unit/sitemap.test.mjs new file mode 100644 index 0000000000..e4e43f80d6 --- /dev/null +++ b/tests/unit/sitemap.test.mjs @@ -0,0 +1,75 @@ +import assert from 'node:assert/strict'; +import { readdir } from 'node:fs/promises'; +import { test } from 'node:test'; + +import movedPages from '../../src/config/redirects/moved-pages.json' with { type: 'json' }; +import { isCanonicalSitemapPage } from '../../src/utils/sitemap.mjs'; + +const isCanonical = (pathname) => isCanonicalSitemapPage(`https://expressjs.com${pathname}`); + +test('keeps the homepage at the root and in translated locales', () => { + assert.equal(isCanonical('/'), true); + assert.equal(isCanonical('/es/'), true); + assert.equal(isCanonical('/pt-br/'), true); +}); + +test('drops /en/, which the homepage at the root replaces', () => { + assert.equal(isCanonical('/en/'), false); +}); + +test('drops language-less redirect stubs and removed locales', () => { + assert.equal(isCanonical('/guide/routing/'), false); + assert.equal(isCanonical('/5x/api/express/'), false); + assert.equal(isCanonical('/blog/'), false); + assert.equal(isCanonical('/tr/guide/routing/'), false); +}); + +test('drops per-locale redirect stubs for every moved page', () => { + for (const path of Object.keys(movedPages)) { + assert.equal(isCanonical(`/en${path}/`), false, `/en${path}/`); + assert.equal(isCanonical(`/es${path}/`), false, `/es${path}/`); + } +}); + +test('keeps versioned docs in every locale', () => { + assert.equal(isCanonical('/en/5x/guide/routing/'), true); + assert.equal(isCanonical('/es/5x/guide/routing/'), true); + assert.equal(isCanonical('/fr/4x/starter/installing/'), true); + assert.equal(isCanonical('/de/4x/advanced/developing-template-engines/'), true); +}); + +test('drops unversioned copies of default-version docs and API pages', () => { + assert.equal(isCanonical('/en/guide/routing/'), false); + assert.equal(isCanonical('/es/starter/installing/'), false); + assert.equal(isCanonical('/en/advanced/developing-template-engines/'), false); + assert.equal(isCanonical('/en/api/'), false); + assert.equal(isCanonical('/en/api/express/'), false); + assert.equal(isCanonical('/es/api/express/'), false); +}); + +test('keeps every page of the unversioned pages collection', async () => { + const contentDir = new URL('../../src/content/pages/en/', import.meta.url); + const files = await readdir(contentDir, { recursive: true }); + const pages = files.filter((file) => /\.mdx?$/.test(file)); + assert.ok(pages.length > 0); + + for (const file of pages) { + const slug = file.replace(/\.mdx?$/, '').replace(/(?:^|\/)index$/, ''); + assert.equal(isCanonical(`/en/${slug}/`), true, `/en/${slug}/`); + assert.equal(isCanonical(`/es/${slug}/`), true, `/es/${slug}/`); + } +}); + +test('keeps blog and API pages only in English', () => { + assert.equal(isCanonical('/en/blog/'), true); + assert.equal(isCanonical('/en/blog/2/'), true); + assert.equal(isCanonical('/en/blog/2024-10-15-v5-release/'), true); + assert.equal(isCanonical('/en/5x/api/'), true); + assert.equal(isCanonical('/en/4x/api/express/'), true); + + assert.equal(isCanonical('/es/blog/'), false); + assert.equal(isCanonical('/es/blog/2/'), false); + assert.equal(isCanonical('/es/blog/2024-10-15-v5-release/'), false); + assert.equal(isCanonical('/es/5x/api/'), false); + assert.equal(isCanonical('/zh-tw/4x/api/express/'), false); +});