fix(sitemap): list only self-canonical URLs

astro.config.mjs

@@ -11,21 +11,17 @@ import Icons from 'unplugin-icons/vite';
 import rehypeSlug from 'rehype-slug';
 import rehypeAutolinkHeadings from 'rehype-autolink-headings';
 import redirects from './src/config/redirects/index.js';
-import movedPages from './src/config/redirects/moved-pages.json' with { type: 'json' };
 import rehypeAccessibleTables from './src/plugins/rehype-accessible-tables.mjs';
 import remarkRewriteLocalizedLinks from './src/plugins/remark-rewrite-localized-links.mjs';
 import remarkCodeTabs from './src/plugins/remark-code-tabs.mjs';
 import rehypeRewriteLocalizedLinks from './src/plugins/rehype-rewrite-localized-links.mjs';
+import { isCanonicalSitemapPage } from './src/utils/sitemap.mjs';
 
 /* https://docs.netlify.com/configure-builds/environment-variables/#read-only-variables */
 const NETLIFY_PREVIEW_SITE = process.env.CONTEXT !== 'production' && process.env.DEPLOY_PRIME_URL;
 
 const site = NETLIFY_PREVIEW_SITE || 'https://expressjs.com';
 
-// Unlocalized paths of moved/removed pages. `[...path].astro` emits `noindex`
-// redirect stubs for these in every locale; keep those stubs out of the sitemap.
-const movedPagePaths = new Set(Object.keys(movedPages));
-
 // https://astro.build/config
 export default defineConfig({
   redirects,
@@ -73,19 +69,9 @@ export default defineConfig({
     icon(),
     react(),
     sitemap({
-      // Only the canonical URLs belong in the sitemap. The catch-all
-      // `[...path].astro` also emits `noindex` redirect stubs: language-less ones
-      // (e.g. `/guide/x`) and per-locale ones for moved pages
-      // (e.g. `/en/resources/middleware/csurf`). Exclude both.
-      // Keep this locale list in sync with `i18n.locales` below.
-      filter: (page) => {
-        const { pathname } = new URL(page);
-        // The homepage is served at `/` and is the canonical for `/en/`.
-        if (pathname === '/') return true;
-        if (!/^\/(de|en|es|fr|it|ja|ko|pt-br|zh-cn|zh-tw)(\/|$)/.test(pathname)) return false;
-        const unlocalized = pathname.replace(/^\/[a-z-]+/, '').replace(/\/$/, '');
-        return !movedPagePaths.has(unlocalized);
-      },
+      // Only the canonical URLs belong in the sitemap.
+      // Keep `LOCALES` in the filter's module in sync with `i18n.locales` below.
+      filter: isCanonicalSitemapPage,
       i18n: {
         defaultLocale: 'en',
         locales: {

src/plugins/rewrite-localized-links-core.mjs

@@ -35,7 +35,7 @@ import { DEFAULT_VERSION } from '../config/versions.ts';
 const DEFAULT_PREFIXES = ['guide', 'starter', 'api', 'resources', 'advanced', 'support', 'blog'];
 
 /** Sections that carry a version (every other section is language-only). */
-const DEFAULT_VERSIONED_SECTIONS = ['api', 'starter', 'guide', 'advanced'];
+export const DEFAULT_VERSIONED_SECTIONS = ['api', 'starter', 'guide', 'advanced'];
 
 /** Language used when it cannot be inferred from the source path. */
 const DEFAULT_LANG = 'en';
@@ -101,7 +101,7 @@ function collectGlobalPaths(menu) {
  * Default unversioned targets: the "global" pages declared in the menu. Computed
  * once so consumers (and `astro.config`) don't have to wire this up themselves.
  */
-const DEFAULT_UNVERSIONED_PATHS = collectGlobalPaths(docsMenu);
+export const DEFAULT_UNVERSIONED_PATHS = collectGlobalPaths(docsMenu);
 
 /**
  * Validates whether a path segment matches version format (e.g. `5x`, `4x`).

src/utils/sitemap.mjs

@@ -0,0 +1,44 @@
+import movedPages from '../config/redirects/moved-pages.json' with { type: 'json' };
+import {
+  DEFAULT_UNVERSIONED_PATHS,
+  DEFAULT_VERSIONED_SECTIONS,
+} from '../plugins/rewrite-localized-links-core.mjs';
+
+// Keep this locale list in sync with the sitemap's `i18n.locales` in `astro.config.mjs`.
+const LOCALES = new Set(['de', 'en', 'es', 'fr', 'it', 'ja', 'ko', 'pt-br', 'zh-cn', 'zh-tw']);
+const VERSIONED_SECTIONS = new Set(DEFAULT_VERSIONED_SECTIONS);
+const UNVERSIONED_PATHS = new Set(DEFAULT_UNVERSIONED_PATHS);
+
+/**
+ * Whether a generated page is the canonical URL of its content, mirroring the
+ * `rel="canonical"` set by `Layout.astro` and `[...slug].astro`. Excludes:
+ * - `/en/`, because the English homepage is served at `/`.
+ * - The `noindex` redirect stubs from `[...path].astro`: language-less ones
+ *   (e.g. `/guide/x`) and per-locale ones for moved pages
+ *   (e.g. `/en/resources/middleware/csurf`).
+ * - Unversioned copies of the default version (e.g. `/es/guide/routing`), whose
+ *   canonical is the versioned URL (e.g. `/es/5x/guide/routing`). "Global" pages
+ *   (e.g. `/es/guide/migrating-5`) only exist unversioned, so they stay.
+ * - Other locales' copies of the untranslated blog and API pages
+ *   (e.g. `/es/5x/api/express`), whose canonical is the English URL.
+ *
+ * @param {string} page Absolute page URL.
+ * @returns {boolean}
+ */
+export function isCanonicalSitemapPage(page) {
+  const { pathname } = new URL(page);
+  if (pathname === '/') return true;
+
+  const [locale, ...segments] = pathname.split('/').filter(Boolean);
+  if (!LOCALES.has(locale)) return false;
+
+  const slug = segments.join('/');
+  if (!slug) return locale !== 'en';
+  if (Object.hasOwn(movedPages, `/${slug}`)) return false;
+
+  const [section, subsection] = segments;
+  if (VERSIONED_SECTIONS.has(section) && !UNVERSIONED_PATHS.has(slug)) return false;
+
+  const isUntranslated = section === 'blog' || (/^\d+x$/.test(section) && subsection === 'api');
+  return locale === 'en' || !isUntranslated;
+}

tests/unit/sitemap.test.mjs

@@ -0,0 +1,75 @@
+import assert from 'node:assert/strict';
+import { readdir } from 'node:fs/promises';
+import { test } from 'node:test';
+
+import movedPages from '../../src/config/redirects/moved-pages.json' with { type: 'json' };
+import { isCanonicalSitemapPage } from '../../src/utils/sitemap.mjs';
+
+const isCanonical = (pathname) => isCanonicalSitemapPage(`https://expressjs.com${pathname}`);
+
+test('keeps the homepage at the root and in translated locales', () => {
+  assert.equal(isCanonical('/'), true);
+  assert.equal(isCanonical('/es/'), true);
+  assert.equal(isCanonical('/pt-br/'), true);
+});
+
+test('drops /en/, which the homepage at the root replaces', () => {
+  assert.equal(isCanonical('/en/'), false);
+});
+
+test('drops language-less redirect stubs and removed locales', () => {
+  assert.equal(isCanonical('/guide/routing/'), false);
+  assert.equal(isCanonical('/5x/api/express/'), false);
+  assert.equal(isCanonical('/blog/'), false);
+  assert.equal(isCanonical('/tr/guide/routing/'), false);
+});
+
+test('drops per-locale redirect stubs for every moved page', () => {
+  for (const path of Object.keys(movedPages)) {
+    assert.equal(isCanonical(`/en${path}/`), false, `/en${path}/`);
+    assert.equal(isCanonical(`/es${path}/`), false, `/es${path}/`);
+  }
+});
+
+test('keeps versioned docs in every locale', () => {
+  assert.equal(isCanonical('/en/5x/guide/routing/'), true);
+  assert.equal(isCanonical('/es/5x/guide/routing/'), true);
+  assert.equal(isCanonical('/fr/4x/starter/installing/'), true);
+  assert.equal(isCanonical('/de/4x/advanced/developing-template-engines/'), true);
+});
+
+test('drops unversioned copies of default-version docs and API pages', () => {
+  assert.equal(isCanonical('/en/guide/routing/'), false);
+  assert.equal(isCanonical('/es/starter/installing/'), false);
+  assert.equal(isCanonical('/en/advanced/developing-template-engines/'), false);
+  assert.equal(isCanonical('/en/api/'), false);
+  assert.equal(isCanonical('/en/api/express/'), false);
+  assert.equal(isCanonical('/es/api/express/'), false);
+});
+
+test('keeps every page of the unversioned pages collection', async () => {
+  const contentDir = new URL('../../src/content/pages/en/', import.meta.url);
+  const files = await readdir(contentDir, { recursive: true });
+  const pages = files.filter((file) => /\.mdx?$/.test(file));
+  assert.ok(pages.length > 0);
+
+  for (const file of pages) {
+    const slug = file.replace(/\.mdx?$/, '').replace(/(?:^|\/)index$/, '');
+    assert.equal(isCanonical(`/en/${slug}/`), true, `/en/${slug}/`);
+    assert.equal(isCanonical(`/es/${slug}/`), true, `/es/${slug}/`);
+  }
+});
+
+test('keeps blog and API pages only in English', () => {
+  assert.equal(isCanonical('/en/blog/'), true);
+  assert.equal(isCanonical('/en/blog/2/'), true);
+  assert.equal(isCanonical('/en/blog/2024-10-15-v5-release/'), true);
+  assert.equal(isCanonical('/en/5x/api/'), true);
+  assert.equal(isCanonical('/en/4x/api/express/'), true);
+
+  assert.equal(isCanonical('/es/blog/'), false);
+  assert.equal(isCanonical('/es/blog/2/'), false);
+  assert.equal(isCanonical('/es/blog/2024-10-15-v5-release/'), false);
+  assert.equal(isCanonical('/es/5x/api/'), false);
+  assert.equal(isCanonical('/zh-tw/4x/api/express/'), false);
+});