Add agent knowledge base with size-adaptive prompt injection

Owners upload text reference files (validated: allowlisted extensions,
UTF-8 only) stored raw in agent_files under knowledge/ with derived
search chunks in agent_knowledge_chunks. Prompt assembly is stable
across turns: small corpora (≤24k chars) inject in full with per-file
[K#] citation sources the web UI renders as clickable badges; larger
corpora inject pinned KNOWLEDGE.md plus a file index and route
retrieval through the new knowledge_search tool (keyword scoring with
CJK bigrams, owner fallback for shared agents). Fork now copies and
reindexes the corpus instead of indexing identity files.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

Author: idoubi

internal/agent/context_chatbot_test.go

@@ -2,10 +2,12 @@ package agent
 
 import (
 	"context"
+	"sort"
 	"strings"
 	"testing"
 
 	"github.com/fastclaw-ai/fastclaw/internal/config"
+	"github.com/fastclaw-ai/fastclaw/internal/store"
 )
 
 // fakeMemoryStore is a deterministic in-memory MemoryStore for the
@@ -61,6 +63,25 @@ func (f *fakeMemoryStore) SaveWorkspaceFile(ctx context.Context, agentID, userID
 	return nil
 }
 
+// ListKnowledgeDocs makes the fake satisfy knowledgeDocLister so the
+// prompt's knowledge section renders from uploaded knowledge/* rows.
+// Sorted for deterministic [K#] ordering, matching the DBStore query.
+func (f *fakeMemoryStore) ListKnowledgeDocs(ctx context.Context, agentID, userID string) ([]store.KnowledgeDoc, error) {
+	prefix := agentID + "|" + userID + "|knowledge/"
+	var out []store.KnowledgeDoc
+	for key, data := range f.files {
+		if !strings.HasPrefix(key, prefix) {
+			continue
+		}
+		out = append(out, store.KnowledgeDoc{
+			Path:    strings.TrimPrefix(key, agentID+"|"+userID+"|"),
+			Content: string(data),
+		})
+	}
+	sort.Slice(out, func(i, j int) bool { return out[i].Path < out[j].Path })
+	return out, nil
+}
+
 const (
 	testAgentID = "agt_test"
 	ownerUID    = "u_owner"
@@ -112,6 +133,7 @@ func TestChatbotPrompt_EmptyChatter(t *testing.T) {
 func TestChatbotPrompt_PopulatedChatter(t *testing.T) {
 	store := newFakeMemoryStore()
 	store.put(testAgentID, ownerUID, "SOUL.md", "# DTJ Soul")
+	store.put(testAgentID, ownerUID, "knowledge/faq.md", "# FAQ\n- Plan: Pro includes web search")
 	store.put(testAgentID, chatterUID, "USER.md", "# Current Chatter\n- Name: 品冠")
 	store.put(testAgentID, chatterUID, "MEMORY.md", "# Memory Log\n- 用户在做产品")
 	cb := newChatbotBuilder(store)
@@ -128,6 +150,66 @@ func TestChatbotPrompt_PopulatedChatter(t *testing.T) {
 	mustContain(t, prompt, "用户在做产品")
 	mustContain(t, prompt, "Treat as factual and current")
 	mustNotContain(t, prompt, "(empty — nothing recorded yet for this chatter")
+
+	// Agent knowledge is owner-uploaded reference material, distinct
+	// from chatter memory and persona files. A small corpus is injected
+	// in full — no query dependence, stable across turns.
+	mustContain(t, prompt, `<agent_knowledge_base mode="full">`)
+	mustContain(t, prompt, "source_id: K1")
+	mustContain(t, prompt, "file: faq.md")
+	mustContain(t, prompt, "path: knowledge/faq.md")
+	mustContain(t, prompt, "cite it inline with the source_id")
+	mustContain(t, prompt, "Plan: Pro includes web search")
+}
+
+func TestChatbotPrompt_KnowledgeSourceIDsFollowUploadOrder(t *testing.T) {
+	store := newFakeMemoryStore()
+	store.put(testAgentID, ownerUID, "KNOWLEDGE.md", "pinned owner notes")
+	store.put(testAgentID, ownerUID, "knowledge/aaa-faq.md", "faq body")
+	store.put(testAgentID, ownerUID, "knowledge/bbb-pricing.md", "pricing body")
+	cb := newChatbotBuilder(store)
+
+	prompt := cb.BuildSystemPromptAs(chatterUID, cb.memory.WithUserID(chatterUID))
+
+	// Pinned KNOWLEDGE.md is K1, uploaded files follow in path order.
+	mustContain(t, prompt, "## [K1] KNOWLEDGE.md")
+	mustContain(t, prompt, "pinned owner notes")
+	mustContain(t, prompt, "source_id: K2\nfile: aaa-faq.md\npath: knowledge/aaa-faq.md")
+	mustContain(t, prompt, "source_id: K3\nfile: bbb-pricing.md\npath: knowledge/bbb-pricing.md")
+
+	// And the extraction side agrees with what was injected — this pair
+	// is the contract the web citation badges depend on.
+	sources := extractKnowledgeCitationSources(prompt)
+	if len(sources) != 3 {
+		t.Fatalf("extracted %d sources, want 3: %+v", len(sources), sources)
+	}
+	if sources[1].ID != "K2" || sources[1].File != "aaa-faq.md" || sources[1].Path != "knowledge/aaa-faq.md" {
+		t.Fatalf("unexpected source[1]: %+v", sources[1])
+	}
+}
+
+func TestChatbotPrompt_LargeKnowledgeSwitchesToIndexMode(t *testing.T) {
+	store := newFakeMemoryStore()
+	big := strings.Repeat("知识库内容 knowledge body. ", 2000) // ~48k chars, over the full-inject budget
+	store.put(testAgentID, ownerUID, "knowledge/aaa-handbook.md", big)
+	store.put(testAgentID, ownerUID, "knowledge/bbb-faq.md", "small faq body")
+	cb := newChatbotBuilder(store)
+
+	prompt := cb.BuildSystemPromptAs(chatterUID, cb.memory.WithUserID(chatterUID))
+
+	mustContain(t, prompt, `<agent_knowledge_base mode="index">`)
+	mustContain(t, prompt, "knowledge_search")
+	mustContain(t, prompt, "- aaa-handbook.md")
+	mustContain(t, prompt, "- bbb-faq.md")
+	// The corpus body must NOT be inlined in index mode.
+	mustNotContain(t, prompt, "small faq body")
+	if len(prompt) > 20000+len(big)/10 {
+		t.Fatalf("index-mode prompt looks like it inlined the corpus (len=%d)", len(prompt))
+	}
+	// No [K#] source blocks in index mode → no citation metadata.
+	if sources := extractKnowledgeCitationSources(prompt); len(sources) != 0 {
+		t.Fatalf("index mode should not emit citation sources, got %+v", sources)
+	}
 }
 
 func TestChatbotPrompt_NoMemorySearchEscapeHatch(t *testing.T) {

internal/agent/knowledge.go

@@ -0,0 +1,152 @@
+package agent
+
+import (
+	"context"
+	"fmt"
+	"strings"
+
+	"github.com/fastclaw-ai/fastclaw/internal/config"
+	"github.com/fastclaw-ai/fastclaw/internal/store"
+)
+
+// The knowledge base is owner-curated reference material: the pinned
+// KNOWLEDGE.md (edited as a system file) plus files uploaded under the
+// knowledge/ prefix. This module renders it into a prompt section that
+// depends only on the corpus itself — never on the current message — so
+// the system prompt stays stable across turns (prompt-cache friendly)
+// and follow-up questions keep the same context.
+//
+// Two shapes, picked by corpus size:
+//
+//	full  — the whole corpus fits the budget: every file is injected
+//	        verbatim with a per-file source id ([K1], [K2], …) the model
+//	        cites inline; the web UI turns those ids into clickable
+//	        source badges (see knowledge_citations.go).
+//	index — corpus too large to inline: the pinned KNOWLEDGE.md (capped)
+//	        plus a file index is injected, and the model is pointed at
+//	        the knowledge_search tool for on-demand retrieval.
+const (
+	// knowledgeFullInjectMaxChars is the total corpus size (runes) at or
+	// below which every knowledge file is injected verbatim. Small corpora
+	// skip retrieval entirely — 100% recall, no missed-lookup failure mode.
+	knowledgeFullInjectMaxChars = 24000
+	// knowledgePinnedMaxChars caps the always-injected KNOWLEDGE.md in
+	// index mode so a runaway pinned note can't crowd out the prompt.
+	knowledgePinnedMaxChars = 8000
+)
+
+// knowledgeDocLister is the optional MemoryStore capability exposing the
+// uploaded corpus. MemoryStoreAdapter implements it; the legacy filesystem
+// setup doesn't, in which case only the pinned KNOWLEDGE.md applies.
+type knowledgeDocLister interface {
+	ListKnowledgeDocs(ctx context.Context, agentID, userID string) ([]store.KnowledgeDoc, error)
+}
+
+func modKnowledge(p *promptCtx) string {
+	return p.cb.buildKnowledgeSection()
+}
+
+func (cb *ContextBuilder) buildKnowledgeSection() string {
+	pinned := cb.loadFileForUser("KNOWLEDGE.md", cb.userID)
+	var docs []store.KnowledgeDoc
+	if lister, ok := cb.store.(knowledgeDocLister); ok {
+		ctx := context.Background()
+		if cb.userID != "" {
+			ctx = config.WithUserID(ctx, cb.userID)
+		}
+		if got, err := lister.ListKnowledgeDocs(ctx, cb.agentID, cb.userID); err == nil {
+			docs = got
+		}
+	}
+	if pinned == "" && len(docs) == 0 {
+		return ""
+	}
+	total := len([]rune(pinned))
+	for _, doc := range docs {
+		total += len([]rune(doc.Content))
+	}
+	if total <= knowledgeFullInjectMaxChars {
+		return knowledgeFullSection(pinned, docs)
+	}
+	return knowledgeIndexSection(pinned, docs)
+}
+
+// knowledgeFullSection injects the entire corpus, one source block per
+// file. The source_id/file/path lines are parsed back out by
+// extractKnowledgeCitationSources so [K#] citations in the reply can be
+// resolved to their file.
+func knowledgeFullSection(pinned string, docs []store.KnowledgeDoc) string {
+	var blocks []string
+	id := 0
+	addBlock := func(file, path, content string) {
+		id++
+		blocks = append(blocks, fmt.Sprintf("## [K%d] %s\nsource_id: K%d\nfile: %s\npath: %s\n\n%s",
+			id, file, id, file, path, strings.TrimSpace(content)))
+	}
+	if pinned != "" {
+		addBlock("KNOWLEDGE.md", "KNOWLEDGE.md", pinned)
+	}
+	for _, doc := range docs {
+		if strings.TrimSpace(doc.Content) == "" {
+			continue
+		}
+		addBlock(knowledgeDocDisplayName(doc.Path), doc.Path, doc.Content)
+	}
+	if len(blocks) == 0 {
+		return ""
+	}
+	return "<agent_knowledge_base mode=\"full\">\n" +
+		"Reference files curated by the agent owner. Treat them as factual, current, and authoritative source material. " +
+		"Every file has a source_id like K1. When you use a fact from a file, cite it inline with the source_id, e.g. [K1]; " +
+		"if several files support a point, cite all of them, e.g. [K1][K3]. " +
+		"If the chatter's question is not covered by these files or the conversation, say what is unknown instead of inventing details.\n\n" +
+		strings.Join(blocks, "\n\n---\n\n") +
+		"\n</agent_knowledge_base>"
+}
+
+// knowledgeIndexSection lists the corpus without inlining it and points
+// the model at the knowledge_search tool.
+func knowledgeIndexSection(pinned string, docs []store.KnowledgeDoc) string {
+	var sb strings.Builder
+	sb.WriteString("<agent_knowledge_base mode=\"index\">\n")
+	sb.WriteString(fmt.Sprintf(
+		"The agent owner uploaded %d knowledge files — too large to include in full, so only an index is shown. "+
+			"Whenever the chatter asks something these files may cover, call the knowledge_search tool with focused keywords "+
+			"(retry with different terms if the first search misses) and answer from the returned snippets, mentioning the "+
+			"source file name. If the files don't cover the question, say what is unknown instead of inventing details.\n",
+		len(docs)))
+	if pinned != "" {
+		if runes := []rune(pinned); len(runes) > knowledgePinnedMaxChars {
+			pinned = string(runes[:knowledgePinnedMaxChars]) + "\n[KNOWLEDGE.md truncated — use knowledge_search for the rest]"
+		}
+		sb.WriteString("\n## Pinned notes (KNOWLEDGE.md)\n")
+		sb.WriteString(pinned)
+		sb.WriteString("\n")
+	}
+	sb.WriteString("\n## Files\n")
+	for _, doc := range docs {
+		sb.WriteString(fmt.Sprintf("- %s (%.1f KB)\n", knowledgeDocDisplayName(doc.Path), float64(len(doc.Content))/1024))
+	}
+	sb.WriteString("</agent_knowledge_base>")
+	return sb.String()
+}
+
+// knowledgeDocDisplayName strips the knowledge/ prefix and the
+// 12-hex-hash filename prefix the upload handler adds for dedup, giving
+// back the name the owner uploaded. Mirrors setup.knowledgeDisplayName.
+func knowledgeDocDisplayName(path string) string {
+	name := strings.TrimPrefix(path, "knowledge/")
+	if len(name) > 13 && name[12] == '-' && isHexString(name[:12]) {
+		return name[13:]
+	}
+	return name
+}
+
+func isHexString(s string) bool {
+	for _, r := range s {
+		if (r < '0' || r > '9') && (r < 'a' || r > 'f') {
+			return false
+		}
+	}
+	return true
+}

internal/agent/knowledge_citations.go

@@ -0,0 +1,67 @@
+package agent
+
+import (
+	"regexp"
+	"strings"
+
+	"github.com/fastclaw-ai/fastclaw/internal/provider"
+)
+
+// knowledgeCitationSource is one [K#]-citable knowledge file, parsed back
+// out of the full-mode <agent_knowledge_base> section of the system
+// prompt (see knowledgeFullSection). Attached to assistant messages as
+// metadata so the web UI can render [K#] citations as clickable badges
+// that open the source file.
+type knowledgeCitationSource struct {
+	ID   string `json:"id"`
+	File string `json:"file"`
+	Path string `json:"path"`
+}
+
+var knowledgeSourceBlockRe = regexp.MustCompile(`source_id:\s*(K\d+)\s*\nfile:\s*([^\n]+)\s*\npath:\s*([^\n]+)`)
+
+func extractKnowledgeCitationSources(systemPrompt string) []knowledgeCitationSource {
+	matches := knowledgeSourceBlockRe.FindAllStringSubmatch(systemPrompt, -1)
+	if len(matches) == 0 {
+		return nil
+	}
+	out := make([]knowledgeCitationSource, 0, len(matches))
+	for _, m := range matches {
+		out = append(out, knowledgeCitationSource{
+			ID:   strings.TrimSpace(m[1]),
+			File: strings.TrimSpace(m[2]),
+			Path: strings.TrimSpace(m[3]),
+		})
+	}
+	return out
+}
+
+func knowledgeMetadata(sources []knowledgeCitationSource) map[string]any {
+	if len(sources) == 0 {
+		return nil
+	}
+	return map[string]any{"knowledgeSources": sources}
+}
+
+func mergeMetadata(base map[string]any, extra map[string]any) map[string]any {
+	if len(base) == 0 && len(extra) == 0 {
+		return nil
+	}
+	out := make(map[string]any, len(base)+len(extra))
+	for k, v := range base {
+		out[k] = v
+	}
+	for k, v := range extra {
+		out[k] = v
+	}
+	return out
+}
+
+func firstSystemContent(messages []provider.Message) string {
+	for _, msg := range messages {
+		if msg.Role == "system" {
+			return msg.Content
+		}
+	}
+	return ""
+}