Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions app/llms-full.txt/route.ts
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ import path from 'path'
import matter from 'gray-matter'
import { NextResponse } from 'next/server'

import { SITE_URL } from '@/lib/seo'
import { CONTENT_SIGNAL, SITE_URL } from '@/lib/seo'

/** Posts or guides as Markdown links, newest first, with their own descriptions */
function contentLinks(folder: 'blog' | 'guides'): string {
Expand All @@ -26,7 +26,7 @@ export function GET() {

> Dev Weekends is a free, nonprofit community of engineers founded in 2018, with programs, mentorship, open source and partnerships that open doors to the world.

Everything Dev Weekends runs is free for members. Any page below can be read as Markdown by asking for it with the header \`Accept: text/markdown\`. Content signals for this site: ai-train=no, search=yes, ai-input=no.
Everything Dev Weekends runs is free for members. Any page below can be read as Markdown by asking for it with the header \`Accept: text/markdown\`. Content signals for this site: ${CONTENT_SIGNAL}. You may use these pages to answer questions and cite them; please do not train models on them.

## About

Expand Down
4 changes: 3 additions & 1 deletion app/llms.txt/route.ts
Original file line number Diff line number Diff line change
@@ -1,13 +1,15 @@
import { NextResponse } from 'next/server'

import { CONTENT_SIGNAL } from '@/lib/seo'

// Follows the llms.txt format: a title, a one-line summary, then sections of
// Markdown links, each with a short note on what the page is for.
export function GET() {
const body = `# Dev Weekends

> Dev Weekends is a free community of engineers with programs, mentorship, open source and partnerships that open doors to the world.

Every public page is also available as Markdown: request it with the header \`Accept: text/markdown\`. Content signals for this site: ai-train=no, search=yes, ai-input=no.
Every public page is also available as Markdown: request it with the header \`Accept: text/markdown\`. Content signals for this site: ${CONTENT_SIGNAL}. You may use these pages to answer questions and cite them; please do not train models on them.

## Primary pages

Expand Down
11 changes: 8 additions & 3 deletions app/robots.txt/route.ts
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
import { NextResponse } from 'next/server'

import { SITE_URL } from '@/lib/seo'
import { CONTENT_SIGNAL, SITE_URL } from '@/lib/seo'

// Logins, dashboards and API routes have nothing to rank for
const blocked = [
Expand All @@ -13,14 +13,19 @@ const blocked = [
'/dsoc/applications/',
]

// The content signal belongs inside a group: a line outside any group is ignored
function group(agent: string) {
return [`User-agent: ${agent}`, 'Allow: /', ...blocked.map((path) => `Disallow: ${path}`)].join('\n')
return [
`User-agent: ${agent}`,
`Content-Signal: ${CONTENT_SIGNAL}`,
'Allow: /',
...blocked.map((path) => `Disallow: ${path}`),
].join('\n')
}

export function GET() {
const body = `${['*', 'GPTBot', 'ClaudeBot', 'PerplexityBot'].map(group).join('\n\n')}

Content-Signal: ai-train=no, search=yes, ai-input=no
Host: ${SITE_URL}
Sitemap: ${SITE_URL}/sitemap.xml
`
Expand Down
3 changes: 2 additions & 1 deletion lib/agent-skills.ts
Original file line number Diff line number Diff line change
Expand Up @@ -59,7 +59,8 @@ Use this skill when you need to verify or reason about Dev Weekends AI usage pre
## Policy source

- robots policy endpoint: \`/robots.txt\`
- Content-Signal directive: \`ai-train=no, search=yes, ai-input=no\`
- Content-Signal directive: \`search=yes, ai-input=yes, ai-train=no\`
- Meaning: the content may appear in search and be used to answer questions with a citation, but not to train models

## Validation checklist

Expand Down
8 changes: 8 additions & 0 deletions lib/seo.ts
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,14 @@ export const SOCIAL_PROFILES = [
'https://github.com/devweekends',
]

/**
* What the site allows machines to do with its content, in the Content Signals
* vocabulary: show it in search, use it to answer questions (AI answers that
* cite the site), but not train models on it. Stated in robots.txt and
* repeated in the llms files.
*/
export const CONTENT_SIGNAL = 'search=yes, ai-input=yes, ai-train=no'

const ORGANIZATION_ID = `${SITE_URL}/#organization`

/** Organization and WebSite, rendered once in the root layout */
Expand Down
Loading