From dbe8bdd0c237320729e86b4dfe516feccccb1175 Mon Sep 17 00:00:00 2001
From: Lucas Jia <lucasjia@amazon.com>
Date: Wed, 8 Jul 2026 14:25:10 -0700
Subject: [PATCH] docs: serve robots.txt opting V2 docs out of AI training
 crawls

Add a robots.txt at the docs site root that disallows AI training and
data-collection crawlers (GPTBot, ClaudeBot, CCBot, Google-Extended,
etc.) from the deprecated V2 docs path (/en/v2/). This is the Layer A
(training-data) crawl opt-out lever from the V2->V3 version-management
strategy, reducing the public V2 corpus fed into future model training.

ReadTheDocs serves robots.txt only from the default version at the
domain root, so the file must live on this V3/default build; a
robots.txt inside the non-default V2 build would land at
/en/v2/robots.txt and never be read. Wire it up via html_extra_path so
Sphinx copies it into the build root.

Generic search bots are intentionally left allowed: robots.txt Disallow
and the noindex meta tag are mutually exclusive on the same path, and
V2 search de-indexing is handled by noindex + canonical on the V2 build.
---
 docs/conf.py    |   7 ++++
 docs/robots.txt | 104 ++++++++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 111 insertions(+)
 create mode 100644 docs/robots.txt

diff --git a/docs/conf.py b/docs/conf.py
index 0a1c277695..67caa8c41e 100644
--- a/docs/conf.py
+++ b/docs/conf.py
@@ -26,6 +26,13 @@
 ]
 
 templates_path = ['_templates']
+
+# Serve robots.txt at the docs site root. ReadTheDocs serves robots.txt only from
+# the default version, so it must live on this V3/default build (not the V2 build).
+# It opts the deprecated V2 docs (/en/v2/) out of AI-training crawls. Sphinx copies
+# files listed here verbatim into the build root.
+html_extra_path = ['robots.txt']
+
 exclude_patterns = [
     '_build', 
     'Thumbs.db', 
diff --git a/docs/robots.txt b/docs/robots.txt
new file mode 100644
index 0000000000..b266c7b4f6
--- /dev/null
+++ b/docs/robots.txt
@@ -0,0 +1,104 @@
+# robots.txt — served at the docs site root (https://sagemaker.readthedocs.io/robots.txt).
+#
+# This file MUST live on the DEFAULT ReadTheDocs version (the V3 / master build):
+# ReadTheDocs only serves robots.txt from the default version at the domain root.
+# A robots.txt shipped inside the non-default V2 build would land at
+# /en/v2/robots.txt, which crawlers never read, so it would have no effect.
+#
+# Purpose: opt the deprecated SageMaker Python SDK V2 docs (/en/v2/) OUT of AI
+# training / data-collection crawls. This is an ADVISORY signal: only compliant
+# crawlers honor it (well-behaved bots from major AI vendors do; abusive crawlers
+# ignore it). It is the crawl opt-out lever.
+#
+# Search de-indexing of the V2 docs is handled SEPARATELY, on the V2 build, by a
+# <meta name="robots" content="noindex"> tag injected on every page plus a
+# <link rel="canonical"> pointing at the V3 page. Those are per-page <head> tags
+# and live on the V2 branch, not here.
+#
+# IMPORTANT — do NOT add a "User-agent: *  Disallow: /en/v2/" rule:
+#   robots.txt Disallow and the noindex meta tag are mutually exclusive on the same
+#   path. A path blocked here is never fetched, so a search crawler would never see
+#   the page's noindex meta and could keep a bare, snippet-less URL in its index.
+#   Generic search bots (Googlebot, Bingbot, ...) must stay ALLOWED so they can read
+#   noindex + canonical and consolidate on V3. Only AI-training/collection crawlers
+#   are blocked below.
+
+# --- OpenAI ---
+User-agent: GPTBot
+Disallow: /en/v2/
+
+User-agent: OAI-SearchBot
+Disallow: /en/v2/
+
+User-agent: ChatGPT-User
+Disallow: /en/v2/
+
+# --- Anthropic ---
+User-agent: ClaudeBot
+Disallow: /en/v2/
+
+User-agent: anthropic-ai
+Disallow: /en/v2/
+
+User-agent: Claude-Web
+Disallow: /en/v2/
+
+# --- Google (Gemini / Vertex AI training; distinct from Googlebot search) ---
+User-agent: Google-Extended
+Disallow: /en/v2/
+
+# --- Common Crawl (open dataset that feeds many LLM training corpora) ---
+User-agent: CCBot
+Disallow: /en/v2/
+
+# --- Meta (Llama training) ---
+User-agent: Meta-ExternalAgent
+Disallow: /en/v2/
+
+User-agent: meta-externalagent
+Disallow: /en/v2/
+
+User-agent: FacebookBot
+Disallow: /en/v2/
+
+# --- Amazon ---
+User-agent: Amazonbot
+Disallow: /en/v2/
+
+# --- Apple (Apple Intelligence training) ---
+User-agent: Applebot-Extended
+Disallow: /en/v2/
+
+# --- ByteDance ---
+User-agent: Bytespider
+Disallow: /en/v2/
+
+# --- Perplexity ---
+User-agent: PerplexityBot
+Disallow: /en/v2/
+
+# --- Cohere ---
+User-agent: cohere-ai
+Disallow: /en/v2/
+
+# --- Others (search-adjacent AI answer engines / data collectors) ---
+User-agent: Diffbot
+Disallow: /en/v2/
+
+User-agent: Omgilibot
+Disallow: /en/v2/
+
+User-agent: Omgili
+Disallow: /en/v2/
+
+User-agent: YouBot
+Disallow: /en/v2/
+
+User-agent: ImagesiftBot
+Disallow: /en/v2/
+
+User-agent: PetalBot
+Disallow: /en/v2/
+
+User-agent: Timpibot
+Disallow: /en/v2/