chore: seo fixes

Author: asmyshlyaev177

packages/example-nextjs16/src/app/i18n/index.ts

@@ -8,13 +8,14 @@
 
 import {
   ALL_LOCALES,
+  INDEXED_LOCALES,
   LOCALES,
   SOURCE,
   SOURCE_LOCALE,
   type Locale,
 } from '../../../../../scripts/i18n/locales.mjs';
 
-export { ALL_LOCALES, LOCALES, SOURCE, SOURCE_LOCALE };
+export { ALL_LOCALES, INDEXED_LOCALES, LOCALES, SOURCE, SOURCE_LOCALE };
 export type { Locale };
 
 /**
@@ -67,19 +68,28 @@ export function stripLocale(pathname: string): string {
 }
 
 /** The public pages, locale-independent. */
-export const PAGES = ['', '/react-router', '/remix', '/astro', '/nextjs', '/vs/nuqs'] as const;
+export const PAGES = [
+  '',
+  '/react-router',
+  '/remix',
+  '/astro',
+  '/nextjs',
+  '/vs/nuqs',
+] as const;
 
 /**
  * The reciprocal hreflang cluster for one page.
  *
  * Next's `alternates.languages` renders these as `<link rel="alternate"
- * hreflang>`. Every locale of a page lists every other one *and* itself, plus
- * `x-default` for English — a cluster whose members disagree about who is in
- * it is discarded wholesale rather than partially honoured.
+ * hreflang>`. Every *indexed* locale of a page lists every other one *and*
+ * itself, plus `x-default` for English — a cluster whose members disagree
+ * about who is in it is discarded wholesale rather than partially honoured,
+ * and one naming a `noindex` page is an error, so an unindexed locale is left
+ * out on both sides (its own pages declare none — see `localeMetadata`).
  */
 export function languageAlternates(origin: string, path: string) {
   const out: Record<string, string> = {};
-  for (const locale of ALL_LOCALES) {
+  for (const locale of INDEXED_LOCALES) {
     out[locale.code] = `${origin}${localePrefix(locale.code)}${path}`;
   }
   out['x-default'] = `${origin}${path}`;

packages/example-nextjs16/src/app/seoStuff.ts

@@ -263,9 +263,14 @@ export function localeMetadata({
   return {
     title: page.title,
     description: page.description,
+    // An unindexed locale (`indexed: false` in the locale table) declines the
+    // index and names no cluster; the rest of its head stays for sharing.
+    ...(locale.indexed ? {} : { robots: { index: false, follow: true } }),
     alternates: {
       canonical: url,
-      languages: languageAlternates(vercelUrl, path),
+      ...(locale.indexed
+        ? { languages: languageAlternates(vercelUrl, path) }
+        : {}),
       types: { 'text/markdown': markdownAlternates(mirror) },
     },
     openGraph: {

packages/example-nextjs16/src/app/sitemap.ts

@@ -2,10 +2,16 @@ import { MetadataRoute } from 'next';
 
 import { CONTENT_LAST_MODIFIED } from './contentDate';
 import { siteUrl } from './domain';
-import { ALL_LOCALES, languageAlternates, localePrefix, PAGES } from './i18n';
+import {
+  INDEXED_LOCALES,
+  languageAlternates,
+  localePrefix,
+  PAGES,
+} from './i18n';
 
 /**
- * Every public page, in every language.
+ * Every public page, in every indexed language — an unindexed locale is
+ * `noindex` and belongs in neither the list nor the clusters.
  *
  * Generated from the locale table rather than listed, because this file and
  * robots.txt and next.config.mjs's `headers()` all need the same set and a
@@ -24,14 +30,15 @@ export default function sitemap(): MetadataRoute.Sitemap {
   // relative within a sitemap, so giving them all a 1 said nothing.
   const priorityOf = (path: string) => (path === '' ? 1 : 0.8);
 
-  return ALL_LOCALES.flatMap((locale) =>
+  return INDEXED_LOCALES.flatMap((locale) =>
     PAGES.map((path) => ({
       url: `${siteUrl}${localePrefix(locale.code)}${path}` || siteUrl,
       lastModified,
       changeFrequency: 'monthly' as const,
       // A translation is not more important than its source, and the English
       // pages are the ones that are actually written rather than generated.
-      priority: locale.code === 'en' ? priorityOf(path) : priorityOf(path) * 0.8,
+      priority:
+        locale.code === 'en' ? priorityOf(path) : priorityOf(path) * 0.8,
       alternates: { languages: languageAlternates(siteUrl, path) },
     })),
   );

scripts/i18n/AGENTS.md

@@ -4,7 +4,11 @@ Repo-wide guide: [`AGENTS.md`](../../AGENTS.md). Translator prompt:
 [`TRANSLATING.md`](TRANSLATING.md). Protected terms: [`GLOSSARY.md`](GLOSSARY.md).
 
 Eight languages besides English: `zh-CN`, `ja`, `ko`, `ru`, `es`, `pt-BR`, `fr`,
-`vi`. Declared once in [`locales.mjs`](locales.mjs); the README suffixes and the
+`vi`. Declared once in [`locales.mjs`](locales.mjs) — with an `indexed` flag
+per locale: only `ja`, `ru` and `vi` are told to crawlers, the rest are
+`noindex` (`localeMetadata` in `seoStuff.ts`), out of every hreflang cluster
+and out of `sitemap.ts`, while staying built and linked. The Search Console
+figures behind the split are in the table's comment. The README suffixes and the
 language-switcher line both derive from that list, so adding a language is an
 edit there plus `pnpm i18n:init`.
 

scripts/i18n/locales.d.mts

@@ -13,12 +13,16 @@ export interface Locale {
   label: string;
   /** English name, for reports and prompts. */
   english: string;
+  /** Told to crawlers, or `noindex` and out of every hreflang cluster and the sitemap. */
+  indexed: boolean;
 }
 
 export const SOURCE_LOCALE: 'en';
 export const SOURCE: Locale;
 export const LOCALES: readonly Locale[];
 export const ALL_LOCALES: readonly Locale[];
+/** The locales a crawler is told about — `ALL_LOCALES` with `indexed`. */
+export const INDEXED_LOCALES: readonly Locale[];
 export const CODES: readonly string[];
 export function findLocale(input: string): Locale | undefined;
 export function localizedName(baseName: string, code: string): string;

scripts/i18n/locales.mjs

@@ -18,28 +18,85 @@ export const SOURCE = {
   label: 'English',
   /** Exonym, for prompts and reports written in English. */
   english: 'English',
+  indexed: true,
 };
 
-/** @type {ReadonlyArray<{code: string, dir: string, label: string, english: string}>} */
+/**
+ * `indexed: false` keeps a translation out of the index — `noindex` on its
+ * pages, no hreflang or sitemap entry — while it stays built, linked from the
+ * switcher and readable. Set from Search Console demand, Jun–Sep 2026
+ * impressions: vi 17, ja 12, ru 11, es 4, fr 4, pt-BR 3, ko 2, zh-CN 1,
+ * against 101 English; a locale earns `true` at 10. Eight LLM translations of
+ * every page is the shape Google's scaled-content policy describes, and it
+ * cost a sister site (x-pat.pages.dev) its whole index in Aug 2026.
+ */
+/** @type {ReadonlyArray<{code: string, dir: string, label: string, english: string, indexed: boolean}>} */
 export const LOCALES = [
-  { code: 'zh-CN', dir: 'zh-cn', label: '简体中文', english: 'Chinese (Simplified)' },
-  { code: 'ja', dir: 'ja', label: '日本語', english: 'Japanese' },
-  { code: 'ko', dir: 'ko', label: '한국어', english: 'Korean' },
-  { code: 'ru', dir: 'ru', label: 'Русский', english: 'Russian' },
-  { code: 'es', dir: 'es', label: 'Español', english: 'Spanish' },
-  { code: 'pt-BR', dir: 'pt-br', label: 'Português (BR)', english: 'Portuguese (Brazil)' },
-  { code: 'fr', dir: 'fr', label: 'Français', english: 'French' },
-  { code: 'vi', dir: 'vi', label: 'Tiếng Việt', english: 'Vietnamese' },
+  {
+    code: 'zh-CN',
+    dir: 'zh-cn',
+    label: '简体中文',
+    english: 'Chinese (Simplified)',
+    indexed: false,
+  },
+  {
+    code: 'ja',
+    dir: 'ja',
+    label: '日本語',
+    english: 'Japanese',
+    indexed: true,
+  },
+  { code: 'ko', dir: 'ko', label: '한국어', english: 'Korean', indexed: false },
+  {
+    code: 'ru',
+    dir: 'ru',
+    label: 'Русский',
+    english: 'Russian',
+    indexed: true,
+  },
+  {
+    code: 'es',
+    dir: 'es',
+    label: 'Español',
+    english: 'Spanish',
+    indexed: false,
+  },
+  {
+    code: 'pt-BR',
+    dir: 'pt-br',
+    label: 'Português (BR)',
+    english: 'Portuguese (Brazil)',
+    indexed: false,
+  },
+  {
+    code: 'fr',
+    dir: 'fr',
+    label: 'Français',
+    english: 'French',
+    indexed: false,
+  },
+  {
+    code: 'vi',
+    dir: 'vi',
+    label: 'Tiếng Việt',
+    english: 'Vietnamese',
+    indexed: true,
+  },
 ];
 
 export const ALL_LOCALES = [SOURCE, ...LOCALES];
 
+/** The locales a crawler is told about — see `indexed` above. */
+export const INDEXED_LOCALES = ALL_LOCALES.filter((l) => l.indexed);
+
 export const CODES = LOCALES.map((l) => l.code);
 
 /** Lookup accepting either spelling, so CLI args can be `pt-BR` or `pt-br`. */
 export function findLocale(input) {
   const needle = String(input).toLowerCase();
-  return ALL_LOCALES.find((l) => l.code.toLowerCase() === needle || l.dir === needle);
+  return ALL_LOCALES.find(
+    (l) => l.code.toLowerCase() === needle || l.dir === needle,
+  );
 }
 
 /**
@@ -63,6 +120,8 @@ export function localizedName(baseName, code) {
  */
 export function switcherLine(baseName, current) {
   return ALL_LOCALES.map((l) =>
-    l.code === current ? l.label : `[${l.label}](./${localizedName(baseName, l.code)})`,
+    l.code === current
+      ? l.label
+      : `[${l.label}](./${localizedName(baseName, l.code)})`,
   ).join(' · ');
 }

tests/landing/robots.spec.ts

@@ -18,16 +18,34 @@ test.describe('robots.txt (landing only)', () => {
     .readdirSync(
       // `(en)` is a route group: it shapes the layout tree, not the URLs,
       // so these are still served at `/test-ssr`, `/useUrlState` and so on.
-      path.join(process.cwd(), 'packages/example-nextjs16/src/app/(en)/(tests)'),
+      path.join(
+        process.cwd(),
+        'packages/example-nextjs16/src/app/(en)/(tests)',
+      ),
       { withFileTypes: true },
     )
     .filter((entry) => entry.isDirectory())
     .map((entry) => `/${entry.name}`);
 
   // The public pages, English and translated. A locale prefix must not
   // be caught by a fixture Disallow, and the fixtures exist only under the
-  // unprefixed English tree.
-  const documented = ['/', '/react-router', '/remix', '/astro', '/nextjs', '/vs/nuqs', '/ja', '/ja/react-router', '/zh-cn/remix', '/ko/astro', '/ru/nextjs', '/ja/vs/nuqs'];
+  // unprefixed English tree. ja, ru and vi are the indexed translations
+  // (scripts/i18n/locales.mjs); zh-cn and ko are kept out of the index.
+  const documented = [
+    '/',
+    '/react-router',
+    '/remix',
+    '/astro',
+    '/nextjs',
+    '/vs/nuqs',
+    '/ja',
+    '/ja/react-router',
+    '/vi/remix',
+    '/ru/astro',
+    '/ru/nextjs',
+    '/ja/vs/nuqs',
+  ];
+  const unindexed = ['/zh-cn', '/zh-cn/remix', '/ko/astro'];
 
   const disallowedPrefixes = async (request: {
     get: (url: string) => Promise<{ text: () => Promise<string> }>;
@@ -78,5 +96,10 @@ test.describe('robots.txt (landing only)', () => {
     for (const route of documented.filter((r) => r !== '/')) {
       expect(sitemap).toContain(route);
     }
+    for (const route of unindexed) {
+      expect(sitemap, `${route} is noindex and must stay out`).not.toContain(
+        `${route}<`,
+      );
+    }
   });
 });