From 0a4f39c413d0b68c9d979b2ea32904c9b52baffa Mon Sep 17 00:00:00 2001 From: NopeCharon <115794254+NopeCharon@users.noreply.github.com> Date: Fri, 6 Nov 2026 19:49:19 +0530 Subject: [PATCH] fix(sitemap): list only self-canonical URLs The sitemap filter removed redirect stubs but kept pages whose rel="canonical" points elsewhere: /en/ (served at /), unversioned copies of default-version docs and API pages, and other locales' copies of the untranslated blog and API pages. Those made up 635 of the 1,311 URLs in the published sitemap, contradicting the canonical tags on the pages. Move the filter to src/utils/sitemap.mjs so it can be unit tested, and make it follow the same rules as Layout.astro and [...slug].astro. The versioned sections and "global" pages come from the link-localization core, so a page marked global in the menu stays listed without further configuration. Fixes #2568 Signed-off-by: NopeCharon <115794254+NopeCharon@users.noreply.github.com> --- astro.config.mjs | 22 ++---- src/plugins/rewrite-localized-links-core.mjs | 4 +- src/utils/sitemap.mjs | 44 ++++++++++++ tests/unit/sitemap.test.mjs | 75 ++++++++++++++++++++ 4 files changed, 125 insertions(+), 20 deletions(-) create mode 100644 src/utils/sitemap.mjs create mode 100644 tests/unit/sitemap.test.mjs diff --git a/astro.config.mjs b/astro.config.mjs index 3b75936528..d6b50b2bc7 100644 --- a/astro.config.mjs +++ b/astro.config.mjs @@ -11,21 +11,17 @@ import Icons from 'unplugin-icons/vite'; import rehypeSlug from 'rehype-slug'; import rehypeAutolinkHeadings from 'rehype-autolink-headings'; import redirects from './src/config/redirects/index.js'; -import movedPages from './src/config/redirects/moved-pages.json' with { type: 'json' }; import rehypeAccessibleTables from './src/plugins/rehype-accessible-tables.mjs'; import remarkRewriteLocalizedLinks from './src/plugins/remark-rewrite-localized-links.mjs'; import remarkCodeTabs from './src/plugins/remark-code-tabs.mjs'; import rehypeRewriteLocalizedLinks from './src/plugins/rehype-rewrite-localized-links.mjs'; +import { isCanonicalSitemapPage } from './src/utils/sitemap.mjs'; /* https://docs.netlify.com/configure-builds/environment-variables/#read-only-variables */ const NETLIFY_PREVIEW_SITE = process.env.CONTEXT !== 'production' && process.env.DEPLOY_PRIME_URL; const site = NETLIFY_PREVIEW_SITE || 'https://expressjs.com'; -// Unlocalized paths of moved/removed pages. `[...path].astro` emits `noindex` -// redirect stubs for these in every locale; keep those stubs out of the sitemap. -const movedPagePaths = new Set(Object.keys(movedPages)); - // https://astro.build/config export default defineConfig({ redirects, @@ -73,19 +69,9 @@ export default defineConfig({ icon(), react(), sitemap({ - // Only the canonical URLs belong in the sitemap. The catch-all - // `[...path].astro` also emits `noindex` redirect stubs: language-less ones - // (e.g. `/guide/x`) and per-locale ones for moved pages - // (e.g. `/en/resources/middleware/csurf`). Exclude both. - // Keep this locale list in sync with `i18n.locales` below. - filter: (page) => { - const { pathname } = new URL(page); - // The homepage is served at `/` and is the canonical for `/en/`. - if (pathname === '/') return true; - if (!/^\/(de|en|es|fr|it|ja|ko|pt-br|zh-cn|zh-tw)(\/|$)/.test(pathname)) return false; - const unlocalized = pathname.replace(/^\/[a-z-]+/, '').replace(/\/$/, ''); - return !movedPagePaths.has(unlocalized); - }, + // Only the canonical URLs belong in the sitemap. + // Keep `LOCALES` in the filter's module in sync with `i18n.locales` below. + filter: isCanonicalSitemapPage, i18n: { defaultLocale: 'en', locales: { diff --git a/src/plugins/rewrite-localized-links-core.mjs b/src/plugins/rewrite-localized-links-core.mjs index 9534963eeb..33241e6e00 100644 --- a/src/plugins/rewrite-localized-links-core.mjs +++ b/src/plugins/rewrite-localized-links-core.mjs @@ -35,7 +35,7 @@ import { DEFAULT_VERSION } from '../config/versions.ts'; const DEFAULT_PREFIXES = ['guide', 'starter', 'api', 'resources', 'advanced', 'support', 'blog']; /** Sections that carry a version (every other section is language-only). */ -const DEFAULT_VERSIONED_SECTIONS = ['api', 'starter', 'guide', 'advanced']; +export const DEFAULT_VERSIONED_SECTIONS = ['api', 'starter', 'guide', 'advanced']; /** Language used when it cannot be inferred from the source path. */ const DEFAULT_LANG = 'en'; @@ -101,7 +101,7 @@ function collectGlobalPaths(menu) { * Default unversioned targets: the "global" pages declared in the menu. Computed * once so consumers (and `astro.config`) don't have to wire this up themselves. */ -const DEFAULT_UNVERSIONED_PATHS = collectGlobalPaths(docsMenu); +export const DEFAULT_UNVERSIONED_PATHS = collectGlobalPaths(docsMenu); /** * Validates whether a path segment matches version format (e.g. `5x`, `4x`). diff --git a/src/utils/sitemap.mjs b/src/utils/sitemap.mjs new file mode 100644 index 0000000000..9f2ae4565f --- /dev/null +++ b/src/utils/sitemap.mjs @@ -0,0 +1,44 @@ +import movedPages from '../config/redirects/moved-pages.json' with { type: 'json' }; +import { + DEFAULT_UNVERSIONED_PATHS, + DEFAULT_VERSIONED_SECTIONS, +} from '../plugins/rewrite-localized-links-core.mjs'; + +// Keep this locale list in sync with the sitemap's `i18n.locales` in `astro.config.mjs`. +const LOCALES = new Set(['de', 'en', 'es', 'fr', 'it', 'ja', 'ko', 'pt-br', 'zh-cn', 'zh-tw']); +const VERSIONED_SECTIONS = new Set(DEFAULT_VERSIONED_SECTIONS); +const UNVERSIONED_PATHS = new Set(DEFAULT_UNVERSIONED_PATHS); + +/** + * Whether a generated page is the canonical URL of its content, mirroring the + * `rel="canonical"` set by `Layout.astro` and `[...slug].astro`. Excludes: + * - `/en/`, because the English homepage is served at `/`. + * - The `noindex` redirect stubs from `[...path].astro`: language-less ones + * (e.g. `/guide/x`) and per-locale ones for moved pages + * (e.g. `/en/resources/middleware/csurf`). + * - Unversioned copies of the default version (e.g. `/es/guide/routing`), whose + * canonical is the versioned URL (e.g. `/es/5x/guide/routing`). "Global" pages + * (e.g. `/es/guide/migrating-5`) only exist unversioned, so they stay. + * - Other locales' copies of the untranslated blog and API pages + * (e.g. `/es/5x/api/express`), whose canonical is the English URL. + * + * @param {string} page Absolute page URL. + * @returns {boolean} + */ +export function isCanonicalSitemapPage(page) { + const { pathname } = new URL(page); + if (pathname === '/') return true; + + const [locale, ...segments] = pathname.split('/').filter(Boolean); + if (!LOCALES.has(locale)) return false; + + const slug = segments.join('/'); + if (!slug) return locale !== 'en'; + if (Object.hasOwn(movedPages, `/${slug}`)) return false; + + const [section, subsection] = segments; + if (VERSIONED_SECTIONS.has(section) && !UNVERSIONED_PATHS.has(slug)) return false; + + const isUntranslated = section === 'blog' || (/^\d+x$/.test(section) && subsection === 'api'); + return locale === 'en' || !isUntranslated; +} diff --git a/tests/unit/sitemap.test.mjs b/tests/unit/sitemap.test.mjs new file mode 100644 index 0000000000..e4e43f80d6 --- /dev/null +++ b/tests/unit/sitemap.test.mjs @@ -0,0 +1,75 @@ +import assert from 'node:assert/strict'; +import { readdir } from 'node:fs/promises'; +import { test } from 'node:test'; + +import movedPages from '../../src/config/redirects/moved-pages.json' with { type: 'json' }; +import { isCanonicalSitemapPage } from '../../src/utils/sitemap.mjs'; + +const isCanonical = (pathname) => isCanonicalSitemapPage(`https://expressjs.com${pathname}`); + +test('keeps the homepage at the root and in translated locales', () => { + assert.equal(isCanonical('/'), true); + assert.equal(isCanonical('/es/'), true); + assert.equal(isCanonical('/pt-br/'), true); +}); + +test('drops /en/, which the homepage at the root replaces', () => { + assert.equal(isCanonical('/en/'), false); +}); + +test('drops language-less redirect stubs and removed locales', () => { + assert.equal(isCanonical('/guide/routing/'), false); + assert.equal(isCanonical('/5x/api/express/'), false); + assert.equal(isCanonical('/blog/'), false); + assert.equal(isCanonical('/tr/guide/routing/'), false); +}); + +test('drops per-locale redirect stubs for every moved page', () => { + for (const path of Object.keys(movedPages)) { + assert.equal(isCanonical(`/en${path}/`), false, `/en${path}/`); + assert.equal(isCanonical(`/es${path}/`), false, `/es${path}/`); + } +}); + +test('keeps versioned docs in every locale', () => { + assert.equal(isCanonical('/en/5x/guide/routing/'), true); + assert.equal(isCanonical('/es/5x/guide/routing/'), true); + assert.equal(isCanonical('/fr/4x/starter/installing/'), true); + assert.equal(isCanonical('/de/4x/advanced/developing-template-engines/'), true); +}); + +test('drops unversioned copies of default-version docs and API pages', () => { + assert.equal(isCanonical('/en/guide/routing/'), false); + assert.equal(isCanonical('/es/starter/installing/'), false); + assert.equal(isCanonical('/en/advanced/developing-template-engines/'), false); + assert.equal(isCanonical('/en/api/'), false); + assert.equal(isCanonical('/en/api/express/'), false); + assert.equal(isCanonical('/es/api/express/'), false); +}); + +test('keeps every page of the unversioned pages collection', async () => { + const contentDir = new URL('../../src/content/pages/en/', import.meta.url); + const files = await readdir(contentDir, { recursive: true }); + const pages = files.filter((file) => /\.mdx?$/.test(file)); + assert.ok(pages.length > 0); + + for (const file of pages) { + const slug = file.replace(/\.mdx?$/, '').replace(/(?:^|\/)index$/, ''); + assert.equal(isCanonical(`/en/${slug}/`), true, `/en/${slug}/`); + assert.equal(isCanonical(`/es/${slug}/`), true, `/es/${slug}/`); + } +}); + +test('keeps blog and API pages only in English', () => { + assert.equal(isCanonical('/en/blog/'), true); + assert.equal(isCanonical('/en/blog/2/'), true); + assert.equal(isCanonical('/en/blog/2024-10-15-v5-release/'), true); + assert.equal(isCanonical('/en/5x/api/'), true); + assert.equal(isCanonical('/en/4x/api/express/'), true); + + assert.equal(isCanonical('/es/blog/'), false); + assert.equal(isCanonical('/es/blog/2/'), false); + assert.equal(isCanonical('/es/blog/2024-10-15-v5-release/'), false); + assert.equal(isCanonical('/es/5x/api/'), false); + assert.equal(isCanonical('/zh-tw/4x/api/express/'), false); +});