diff --git a/docs/conf.py b/docs/conf.py index 0a1c277695..67caa8c41e 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -26,6 +26,13 @@ ] templates_path = ['_templates'] + +# Serve robots.txt at the docs site root. ReadTheDocs serves robots.txt only from +# the default version, so it must live on this V3/default build (not the V2 build). +# It opts the deprecated V2 docs (/en/v2/) out of AI-training crawls. Sphinx copies +# files listed here verbatim into the build root. +html_extra_path = ['robots.txt'] + exclude_patterns = [ '_build', 'Thumbs.db', diff --git a/docs/robots.txt b/docs/robots.txt new file mode 100644 index 0000000000..b266c7b4f6 --- /dev/null +++ b/docs/robots.txt @@ -0,0 +1,104 @@ +# robots.txt — served at the docs site root (https://sagemaker.readthedocs.io/robots.txt). +# +# This file MUST live on the DEFAULT ReadTheDocs version (the V3 / master build): +# ReadTheDocs only serves robots.txt from the default version at the domain root. +# A robots.txt shipped inside the non-default V2 build would land at +# /en/v2/robots.txt, which crawlers never read, so it would have no effect. +# +# Purpose: opt the deprecated SageMaker Python SDK V2 docs (/en/v2/) OUT of AI +# training / data-collection crawls. This is an ADVISORY signal: only compliant +# crawlers honor it (well-behaved bots from major AI vendors do; abusive crawlers +# ignore it). It is the crawl opt-out lever. +# +# Search de-indexing of the V2 docs is handled SEPARATELY, on the V2 build, by a +# tag injected on every page plus a +# pointing at the V3 page. Those are per-page tags +# and live on the V2 branch, not here. +# +# IMPORTANT — do NOT add a "User-agent: * Disallow: /en/v2/" rule: +# robots.txt Disallow and the noindex meta tag are mutually exclusive on the same +# path. A path blocked here is never fetched, so a search crawler would never see +# the page's noindex meta and could keep a bare, snippet-less URL in its index. +# Generic search bots (Googlebot, Bingbot, ...) must stay ALLOWED so they can read +# noindex + canonical and consolidate on V3. Only AI-training/collection crawlers +# are blocked below. + +# --- OpenAI --- +User-agent: GPTBot +Disallow: /en/v2/ + +User-agent: OAI-SearchBot +Disallow: /en/v2/ + +User-agent: ChatGPT-User +Disallow: /en/v2/ + +# --- Anthropic --- +User-agent: ClaudeBot +Disallow: /en/v2/ + +User-agent: anthropic-ai +Disallow: /en/v2/ + +User-agent: Claude-Web +Disallow: /en/v2/ + +# --- Google (Gemini / Vertex AI training; distinct from Googlebot search) --- +User-agent: Google-Extended +Disallow: /en/v2/ + +# --- Common Crawl (open dataset that feeds many LLM training corpora) --- +User-agent: CCBot +Disallow: /en/v2/ + +# --- Meta (Llama training) --- +User-agent: Meta-ExternalAgent +Disallow: /en/v2/ + +User-agent: meta-externalagent +Disallow: /en/v2/ + +User-agent: FacebookBot +Disallow: /en/v2/ + +# --- Amazon --- +User-agent: Amazonbot +Disallow: /en/v2/ + +# --- Apple (Apple Intelligence training) --- +User-agent: Applebot-Extended +Disallow: /en/v2/ + +# --- ByteDance --- +User-agent: Bytespider +Disallow: /en/v2/ + +# --- Perplexity --- +User-agent: PerplexityBot +Disallow: /en/v2/ + +# --- Cohere --- +User-agent: cohere-ai +Disallow: /en/v2/ + +# --- Others (search-adjacent AI answer engines / data collectors) --- +User-agent: Diffbot +Disallow: /en/v2/ + +User-agent: Omgilibot +Disallow: /en/v2/ + +User-agent: Omgili +Disallow: /en/v2/ + +User-agent: YouBot +Disallow: /en/v2/ + +User-agent: ImagesiftBot +Disallow: /en/v2/ + +User-agent: PetalBot +Disallow: /en/v2/ + +User-agent: Timpibot +Disallow: /en/v2/