Add docs/llms-full.txt so agents can read the guides in one file

Author: ellmetha

.github/workflows/qa.yml

@@ -43,7 +43,7 @@ jobs:
       - uses: actions/setup-node@v7
         with:
           node-version: 22.x
-      - name: Check llms.txt
+      - name: Check llms.txt and llms-full.txt
         run: node scripts/generate_llms_txt.js --check
   generated-app:
     runs-on: ubuntu-latest

AGENTS.md

@@ -126,7 +126,7 @@ When adding or changing user-visible behavior:
 The only docs tree to edit by hand is `docs/docs/`.
 
 - **User guides**: `docs/docs/` (Docusaurus, live preview via `cd docs && npm run start`)
-- **Agent index**: `docs/static/llms.txt` is generated from `docs/sidebars.js` and each page's front matter (`node scripts/generate_llms_txt.js`). It is published at <https://martenframework.com/docs/llms.txt> and lists the guides for the version under development (`/docs/next/`). Regenerate it when a guide is added, renamed, removed, or its description changes. `make qa` checks that the committed file matches.
+- **Agent guides**: `docs/static/llms.txt` and `docs/static/llms-full.txt` are generated from `docs/sidebars.js` and the guide pages (`node scripts/generate_llms_txt.js`). `llms.txt` is the index. `llms-full.txt` is the guide text. They are published at <https://martenframework.com/docs/llms.txt> and <https://martenframework.com/docs/llms-full.txt> for the version under development (`/docs/next/`). Regenerate them when a guide is added, renamed, removed, or its content changes. `make qa` checks that the committed files match.
 - **API reference**: generated with `make docs_api` → `docs/static/api/dev/`
 - **Snapshots**: `docs/versioned_docs/` and `docs/static/api/0.5`, `docs/static/api/0.6`, and `docs/static/api/0.7` are published copies. Leave them unchanged.
 

Makefile

@@ -70,12 +70,12 @@ typos:
 	typos
 
 .PHONY: llms_txt
-## Regenerate docs/static/llms.txt from the docs sidebar.
+## Regenerate docs/static/llms.txt and docs/static/llms-full.txt from the docs sidebar.
 llms_txt:
 	node scripts/generate_llms_txt.js
 
 .PHONY: llms_txt_check
-## Fail when docs/static/llms.txt does not match the docs sidebar.
+## Fail when docs/static/llms.txt or docs/static/llms-full.txt does not match the docs.
 llms_txt_check:
 	node scripts/generate_llms_txt.js --check
 

docs/docs/the-marten-project/release-notes/1.0.md

@@ -28,6 +28,7 @@ _Under development._
 
 * The [`serve`](../../development/reference/management-commands.md#serve) management command now displays how long project compilation took after each successful build.
 * The [`new`](../../development/reference/management-commands.md#new) command now writes an `AGENTS.md` file when creating a project. The file records the Marten version, links to the matching guides and API, and describes the project layout, generators, requires, migrations, CSRF, and specs, so coding agents can work in the generated project. Projects created with `--edge` link to the in-development guides.
+* The in-development guides are published as a single file at [`/docs/llms-full.txt`](https://martenframework.com/docs/llms-full.txt), next to the guide index at [`/docs/llms.txt`](https://martenframework.com/docs/llms.txt).
 * The [`new`](../../development/reference/management-commands.md#new) command prints its usage and exits with status 1 when `type` or `name` is omitted and stdin is not a terminal, instead of waiting for prompts. Passing both arguments still runs without prompts.
 
 ## Backward incompatible changes

docs/static/llms-full.txt

@@ -2,7 +2,7 @@
 
 > Marten is a batteries-included web framework for Crystal. This file indexes the guides for the version under development.
 
-These URLs are the in-development guides at https://martenframework.com/docs/next/. The latest stable guides are at https://martenframework.com/docs/. The generated API reference for this version is at https://martenframework.com/docs/api/dev/.
+The full text of these guides is at https://martenframework.com/docs/llms-full.txt. These URLs are the in-development guides at https://martenframework.com/docs/next/. The latest stable guides are at https://martenframework.com/docs/. The generated API reference for this version is at https://martenframework.com/docs/api/dev/.
 
 ## Prologue
 

docs/static/llms.txt

@@ -1,9 +1,10 @@
 #!/usr/bin/env node
 
-// Generates docs/static/llms.txt from docs/sidebars.js and each page's front matter.
+// Generates docs/static/llms.txt and docs/static/llms-full.txt from docs/sidebars.js
+// and the guide pages.
 //
-//   node scripts/generate_llms_txt.js           # write the file
-//   node scripts/generate_llms_txt.js --check   # exit 1 when the file is stale
+//   node scripts/generate_llms_txt.js           # write the files
+//   node scripts/generate_llms_txt.js --check   # exit 1 when either file is stale
 
 const {execFileSync} = require("child_process");
 const fs = require("fs");
@@ -13,7 +14,9 @@ const path = require("path");
 const ROOT = path.join(__dirname, "..");
 const DOCS = path.join(ROOT, "docs", "docs");
 const OUTPUT = path.join(ROOT, "docs", "static", "llms.txt");
+const FULL_OUTPUT = path.join(ROOT, "docs", "static", "llms-full.txt");
 const NEXT_DOCS = "https://martenframework.com/docs/next";
+const DOCS_ORIGIN = "https://martenframework.com/docs";
 const RELEASE_NOTES_ID = "the-marten-project/release-notes";
 
 const sidebar = require(path.join(ROOT, "docs", "sidebars.js")).sidebar;
@@ -51,9 +54,19 @@ function isUnderDevelopment(id) {
   return readPage(id).body.startsWith("_Under development._");
 }
 
+function pageUrl(id) {
+  let slug;
+  try {
+    slug = readPage(id).frontMatter.slug;
+  } catch (_error) {
+    return `${NEXT_DOCS}/${id}`;
+  }
+  return slug === "/" ? `${NEXT_DOCS}/` : `${NEXT_DOCS}/${id}`;
+}
+
 function pageLink(id) {
   const page = readPage(id);
-  const {title, description, slug} = page.frontMatter;
+  const {title, description} = page.frontMatter;
 
   if (!title) {
     throw new Error(`${path.relative(ROOT, page.file)} is missing a title`);
@@ -65,8 +78,123 @@ function pageLink(id) {
     );
   }
 
-  const url = slug === "/" ? `${NEXT_DOCS}/` : `${NEXT_DOCS}/${id}`;
-  return `- [${title}](${url}): ${description}`;
+  return `- [${title}](${pageUrl(id)}): ${description}`;
+}
+
+function docCardsToMarkdown(body) {
+  return body.replace(/<DocCard\s+item=\{\{([\s\S]*?)\}\}\s*\/>/g, (_match, attrs) => {
+    const label = attrs.match(/label:\s*"([^"]+)"/);
+    const docId = attrs.match(/docId:\s*"([^"]+)"/);
+    if (!label || !docId) return "";
+    return `- [${label[1]}](${pageUrl(docId[1])})`;
+  });
+}
+
+function stripJsxTags(text) {
+  let result = "";
+  let index = 0;
+
+  while (index < text.length) {
+    const next = text[index + 1];
+    if (text[index] === "<" && next && /[A-Za-z/!]/.test(next)) {
+      let cursor = index + 1;
+      let braces = 0;
+      while (cursor < text.length) {
+        const char = text[cursor];
+        if (char === "{") braces += 1;
+        else if (char === "}") braces = Math.max(0, braces - 1);
+        else if (char === ">" && braces === 0) {
+          cursor += 1;
+          break;
+        }
+        cursor += 1;
+      }
+      index = cursor;
+      continue;
+    }
+
+    result += text[index];
+    index += 1;
+  }
+
+  return result;
+}
+
+function rewriteTarget(target, id) {
+  if (target.startsWith("pathname:///")) {
+    return `${DOCS_ORIGIN}${target.slice("pathname://".length)}`;
+  }
+  if (/^[a-z][a-z0-9+.-]*:/i.test(target) || target.startsWith("#") || target.startsWith("//")) {
+    return target;
+  }
+  if (/\s/.test(target)) return target;
+
+  const hashAt = target.indexOf("#");
+  const hash = hashAt === -1 ? "" : target.slice(hashAt);
+  const pathPart = hashAt === -1 ? target : target.slice(0, hashAt);
+  if (!pathPart) return target;
+
+  const base = path.posix.dirname(id);
+  const resolved = path.posix.normalize(path.posix.join(base === "." ? "" : base, pathPart));
+  if (resolved.startsWith("..")) return target;
+
+  if (/\.mdx?$/.test(pathPart)) {
+    return `${pageUrl(resolved.replace(/\.mdx?$/, ""))}${hash}`;
+  }
+
+  if (!path.posix.extname(pathPart)) {
+    try {
+      readPage(resolved);
+    } catch (_error) {
+      return target;
+    }
+    return `${pageUrl(resolved)}${hash}`;
+  }
+
+  const assetAt = resolved.indexOf("static/");
+  if (assetAt !== -1 && /\.(png|jpe?g|gif|svg|webp)$/i.test(pathPart)) {
+    return `${DOCS_ORIGIN}/${resolved.slice(assetAt + "static/".length)}${hash}`;
+  }
+
+  return target;
+}
+
+function rewriteLinks(segment, id) {
+  const linkStart = /(!?)\[((?:[^\[\]]|\[[^\]]*\])*)\]\(/g;
+  let result = "";
+  let last = 0;
+
+  for (const match of segment.matchAll(linkStart)) {
+    const open = match.index + match[0].length;
+    let depth = 1;
+    let cursor = open;
+    while (cursor < segment.length && depth > 0) {
+      if (segment[cursor] === "(") depth += 1;
+      else if (segment[cursor] === ")") depth -= 1;
+      if (depth > 0) cursor += 1;
+    }
+    if (depth !== 0) continue;
+
+    result += segment.slice(last, match.index);
+    result += `${match[1]}[${match[2]}](${rewriteTarget(segment.slice(open, cursor).trim(), id)})`;
+    last = cursor + 1;
+  }
+
+  return result + segment.slice(last);
+}
+
+function prepareBody(id) {
+  const page = readPage(id);
+  let body = page.body;
+
+  if (page.file.endsWith(".mdx")) {
+    body = docCardsToMarkdown(body);
+    body = body.replace(/^import .*$/gm, "");
+    body = stripJsxTags(body);
+    body = body.replace(/[ \t]+\n/g, "\n").replace(/\n{3,}/g, "\n\n");
+  }
+
+  return rewriteLinks(body, id).trim();
 }
 
 function addPage(sections, sectionName, id) {
@@ -111,7 +239,7 @@ function generate() {
     "",
     "> Marten is a batteries-included web framework for Crystal. This file indexes the guides for the version under development.",
     "",
-    "These URLs are the in-development guides at https://martenframework.com/docs/next/. The latest stable guides are at https://martenframework.com/docs/. The generated API reference for this version is at https://martenframework.com/docs/api/dev/.",
+    "The full text of these guides is at https://martenframework.com/docs/llms-full.txt. These URLs are the in-development guides at https://martenframework.com/docs/next/. The latest stable guides are at https://martenframework.com/docs/. The generated API reference for this version is at https://martenframework.com/docs/api/dev/.",
     "",
   ];
 
@@ -131,32 +259,65 @@ function generate() {
   return lines.join("\n");
 }
 
+function generateFull() {
+  const sections = [];
+  walk(sidebar, null, sections);
+
+  const lines = [
+    "# Marten",
+    "",
+    "> Marten is a batteries-included web framework for Crystal. This file contains the guides for the version under development.",
+    "",
+    "The guide index is at https://martenframework.com/docs/llms.txt. These guides are the in-development guides at https://martenframework.com/docs/next/. The latest stable guides are at https://martenframework.com/docs/. The generated API reference for this version is at https://martenframework.com/docs/api/dev/.",
+    "",
+  ];
+
+  for (const section of sections) {
+    for (const id of section.items) {
+      const page = readPage(id);
+      lines.push(`# ${page.frontMatter.title}`, "");
+      lines.push(`Source: ${pageUrl(id)}`, "");
+      lines.push(prepareBody(id), "");
+    }
+  }
+
+  return lines.join("\n");
+}
+
+function checkFile(output, generated) {
+  const current = fs.existsSync(output) ? fs.readFileSync(output, "utf8") : "";
+  if (current === generated) return false;
+
+  const tmp = path.join(os.tmpdir(), `marten-${path.basename(output)}`);
+  fs.writeFileSync(tmp, generated);
+  try {
+    execFileSync("diff", ["-u", output, tmp], {stdio: "inherit"});
+  } catch (_error) {
+    // diff exits with status 1 when the files differ.
+  }
+  console.error(`\n${path.relative(ROOT, output)} is out of date. Run: node scripts/generate_llms_txt.js`);
+  return true;
+}
+
 function main() {
   const check = process.argv.includes("--check");
   if (process.argv.slice(2).some(arg => arg !== "--check")) {
     console.error("Usage: node scripts/generate_llms_txt.js [--check]");
     process.exit(2);
   }
 
-  const generated = generate();
+  const outputs = [
+    {file: OUTPUT, content: generate()},
+    {file: FULL_OUTPUT, content: generateFull()},
+  ];
 
   if (!check) {
-    fs.writeFileSync(OUTPUT, generated);
+    for (const output of outputs) fs.writeFileSync(output.file, output.content);
     return;
   }
 
-  const current = fs.existsSync(OUTPUT) ? fs.readFileSync(OUTPUT, "utf8") : "";
-  if (current === generated) return;
-
-  const tmp = path.join(os.tmpdir(), "marten-llms.txt");
-  fs.writeFileSync(tmp, generated);
-  try {
-    execFileSync("diff", ["-u", OUTPUT, tmp], {stdio: "inherit"});
-  } catch (_error) {
-    // diff exits with status 1 when the files differ.
-  }
-  console.error("\ndocs/static/llms.txt is out of date. Run: node scripts/generate_llms_txt.js");
-  process.exit(1);
+  const stale = outputs.some(output => checkFile(output.file, output.content));
+  if (stale) process.exit(1);
 }
 
 main();

scripts/generate_llms_txt.js

@@ -294,6 +294,7 @@ describe Marten::CLI::Manage::Command::New do
       agents.should contain "https://martenframework.com/docs/next/"
       agents.should contain "https://martenframework.com/docs/api/dev/index.html"
       agents.should contain "https://martenframework.com/docs/llms.txt"
+      agents.should contain "https://martenframework.com/docs/llms-full.txt"
       agents.should contain "https://martenframework.com/docs/next/getting-started/from-django"
     end
 

spec/marten/cli/manage/command/new_spec.cr

@@ -2,7 +2,7 @@
 # Marten project guide for coding agents
 
 <%- if context.edge -%>
-This project uses Marten <%= Marten::VERSION %> from the development branch. Follow the guides at <https://martenframework.com/docs/next/> and the API reference at <https://martenframework.com/docs/api/dev/index.html>. The guide index is at <https://martenframework.com/docs/llms.txt>.
+This project uses Marten <%= Marten::VERSION %> from the development branch. Follow the guides at <https://martenframework.com/docs/next/> and the API reference at <https://martenframework.com/docs/api/dev/index.html>. The guide index is at <https://martenframework.com/docs/llms.txt>. The full text of those guides is at <https://martenframework.com/docs/llms-full.txt>.
 <%- else -%>
 This project uses Marten <%= Marten::VERSION %>. Follow the guides at <https://martenframework.com/docs/> and the API reference at <https://martenframework.com/docs/api/<%= api_series %>/index.html>. These guide URLs track the latest stable release. The API URL is pinned to <%= api_series %>. When a guide and that API disagree, follow the API.
 <%- end -%>