diff --git a/app/llms-full.txt/route.ts b/app/llms-full.txt/route.ts index a0bdda0..0765cf8 100644 --- a/app/llms-full.txt/route.ts +++ b/app/llms-full.txt/route.ts @@ -3,7 +3,7 @@ import path from 'path' import matter from 'gray-matter' import { NextResponse } from 'next/server' -import { SITE_URL } from '@/lib/seo' +import { CONTENT_SIGNAL, SITE_URL } from '@/lib/seo' /** Posts or guides as Markdown links, newest first, with their own descriptions */ function contentLinks(folder: 'blog' | 'guides'): string { @@ -26,7 +26,7 @@ export function GET() { > Dev Weekends is a free, nonprofit community of engineers founded in 2018, with programs, mentorship, open source and partnerships that open doors to the world. -Everything Dev Weekends runs is free for members. Any page below can be read as Markdown by asking for it with the header \`Accept: text/markdown\`. Content signals for this site: ai-train=no, search=yes, ai-input=no. +Everything Dev Weekends runs is free for members. Any page below can be read as Markdown by asking for it with the header \`Accept: text/markdown\`. Content signals for this site: ${CONTENT_SIGNAL}. You may use these pages to answer questions and cite them; please do not train models on them. ## About diff --git a/app/llms.txt/route.ts b/app/llms.txt/route.ts index 48471de..cb7d714 100644 --- a/app/llms.txt/route.ts +++ b/app/llms.txt/route.ts @@ -1,5 +1,7 @@ import { NextResponse } from 'next/server' +import { CONTENT_SIGNAL } from '@/lib/seo' + // Follows the llms.txt format: a title, a one-line summary, then sections of // Markdown links, each with a short note on what the page is for. export function GET() { @@ -7,7 +9,7 @@ export function GET() { > Dev Weekends is a free community of engineers with programs, mentorship, open source and partnerships that open doors to the world. -Every public page is also available as Markdown: request it with the header \`Accept: text/markdown\`. Content signals for this site: ai-train=no, search=yes, ai-input=no. +Every public page is also available as Markdown: request it with the header \`Accept: text/markdown\`. Content signals for this site: ${CONTENT_SIGNAL}. You may use these pages to answer questions and cite them; please do not train models on them. ## Primary pages diff --git a/app/robots.txt/route.ts b/app/robots.txt/route.ts index bbee371..c037a8b 100644 --- a/app/robots.txt/route.ts +++ b/app/robots.txt/route.ts @@ -1,6 +1,6 @@ import { NextResponse } from 'next/server' -import { SITE_URL } from '@/lib/seo' +import { CONTENT_SIGNAL, SITE_URL } from '@/lib/seo' // Logins, dashboards and API routes have nothing to rank for const blocked = [ @@ -13,14 +13,19 @@ const blocked = [ '/dsoc/applications/', ] +// The content signal belongs inside a group: a line outside any group is ignored function group(agent: string) { - return [`User-agent: ${agent}`, 'Allow: /', ...blocked.map((path) => `Disallow: ${path}`)].join('\n') + return [ + `User-agent: ${agent}`, + `Content-Signal: ${CONTENT_SIGNAL}`, + 'Allow: /', + ...blocked.map((path) => `Disallow: ${path}`), + ].join('\n') } export function GET() { const body = `${['*', 'GPTBot', 'ClaudeBot', 'PerplexityBot'].map(group).join('\n\n')} -Content-Signal: ai-train=no, search=yes, ai-input=no Host: ${SITE_URL} Sitemap: ${SITE_URL}/sitemap.xml ` diff --git a/lib/agent-skills.ts b/lib/agent-skills.ts index 5d31fa6..14a975f 100644 --- a/lib/agent-skills.ts +++ b/lib/agent-skills.ts @@ -59,7 +59,8 @@ Use this skill when you need to verify or reason about Dev Weekends AI usage pre ## Policy source - robots policy endpoint: \`/robots.txt\` -- Content-Signal directive: \`ai-train=no, search=yes, ai-input=no\` +- Content-Signal directive: \`search=yes, ai-input=yes, ai-train=no\` +- Meaning: the content may appear in search and be used to answer questions with a citation, but not to train models ## Validation checklist diff --git a/lib/seo.ts b/lib/seo.ts index 9dca7f6..7e690fe 100644 --- a/lib/seo.ts +++ b/lib/seo.ts @@ -17,6 +17,14 @@ export const SOCIAL_PROFILES = [ 'https://github.com/devweekends', ] +/** + * What the site allows machines to do with its content, in the Content Signals + * vocabulary: show it in search, use it to answer questions (AI answers that + * cite the site), but not train models on it. Stated in robots.txt and + * repeated in the llms files. + */ +export const CONTENT_SIGNAL = 'search=yes, ai-input=yes, ai-train=no' + const ORGANIZATION_ID = `${SITE_URL}/#organization` /** Organization and WebSite, rendered once in the root layout */