Add TQ-MSE conversation vector search

Author: XiaoConstantine

.agents/skills/sgrep/SKILL.md

@@ -21,7 +21,7 @@ Use `sgrep` for semantic and hybrid search across **code** and **agent conversat
 - When ripgrep patterns keep missing relevant code
 
 ### Conversation Search
-- Finding past discussions with **Claude Code**, **Codex CLI**, or **Cursor**
+- Finding past discussions with **Claude Code**, **Codex CLI**, **Cursor**, **OpenCode**, or **Pi**
 - Recalling how you solved a similar problem before
 - Building context from previous sessions for new tasks
 - Searching across all your coding agent interactions
@@ -31,10 +31,16 @@ Use `sgrep` for semantic and hybrid search across **code** and **agent conversat
 ```bash
 # First time only
 sgrep setup
+sgrep setup --with-rerank  # optional, only for --rerank
 
 # Index current directory; builds compact TQ-MSE chunk/file vectors by default
 sgrep index .
 
+# Optional ColBERT segment codec override
+sgrep index . --colbert-codec tqmse
+sgrep index . --colbert-codec int8
+sgrep index . --colbert-codec pq6
+
 # Legacy compatibility: also persist full SQL vectors
 sgrep index . --sql-vectors
 
@@ -63,17 +69,32 @@ sgrep --json "rate limiting"
 ## Conversation Search
 
 ```bash
-# Index conversations
+# Index conversations; refreshes compact TQ-MSE turn vectors
 sgrep conv index
+sgrep conv index --source codex
+sgrep conv index --source claude
+sgrep conv index --source opencode
+sgrep conv index --source pi
+sgrep conv index --watch
+sgrep conv index --force
 
 # Search conversations
 sgrep conv "authentication flow"
 sgrep conv "JWT refresh_token" --hybrid
 sgrep conv "database migration" --agent claude --since 7d
+sgrep conv "bug fix" --project payment-service --after 2026-01-01 --before 2026-06-01
+sgrep conv "exact phrase" --exact
+sgrep conv "auth" --json -n 1
 
-# View or export a session
+# View, export, context, and copy helpers
 sgrep conv view <session_id>
+sgrep conv view <session_id> --turn 3 --no-color
+sgrep conv export <session_id> --format markdown -o conversation.md
+sgrep conv export <session_id> --format json -o conversation.json
 sgrep conv context <session_id>
+sgrep conv context <session_id> --turns 10 --copy
+sgrep conv copy <session_id> --turn 2 --code-only
+sgrep conv status
 ```
 
 ## Semantic vs Hybrid

README.md

@@ -79,9 +79,13 @@ sgrep setup
 sgrep index .
 
 # Optional: override the default TQ-MSE ColBERT codec
+sgrep index . --colbert-codec tqmse
 sgrep index . --colbert-codec int8
 sgrep index . --colbert-codec pq6
 
+# Optional: keep full SQL vectors for legacy vector search
+sgrep index . --sql-vectors
+
 # Semantic search (quick)
 sgrep "error handling for database connections"
 
@@ -116,32 +120,43 @@ sgrep conv index --source pi
 # Watch mode (auto-index new conversations)
 sgrep conv index --watch
 
+# Re-index all cached sessions
+sgrep conv index --force
+
 # Search conversations
 sgrep conv "authentication"
 sgrep conv search "authentication" --agent codex --limit 5
 sgrep conv search "session compaction" --agent pi --limit 5
 sgrep conv "JWT token" --hybrid
 sgrep conv "database migration" --agent claude --since 7d
+sgrep conv "auth" --exact
+sgrep conv "bug fix" --project payment-service --after 2026-01-01 --before 2026-06-01
 
 # View, export, or resume a session
 sgrep conv view <session_id>
+sgrep conv view <session_id> --turn 3 --no-color
 sgrep conv export <session_id> -o conversation.md
+sgrep conv export <session_id> --format json -o conversation.json
 sgrep conv resume <session_id>
 
 # Extract context for injection into new session
 sgrep conv context <session_id>
+sgrep conv context <session_id> --turns 10 --copy
 
 # Copy to clipboard
 sgrep conv copy <session_id>
+sgrep conv copy <session_id> --turn 2 --code-only
 
 # Check index status
 sgrep conv status
 ```
 
 **Watch mode** monitors conversation directories for all agents and automatically indexes new sessions as they're created. This ensures your conversation search stays up-to-date without manual re-indexing.
 
-Conversations are stored at `~/.sgrep/conversations/conv.db`. Re-running
-`sgrep conv index` backfills missing embeddings for existing sessions.
+Conversations are stored at `~/.sgrep/conversations/conv.db` with compact
+TQ-MSE turn vectors in `~/.sgrep/conversations/turn_embeddings.tqmse`.
+Re-running `sgrep conv index` backfills missing embeddings for existing
+sessions and refreshes the compact conversation vector sidecar.
 
 Default search output includes the agent session/conversation ID so you can jump
 straight to `view`, `resume`, or external agent tooling with the exact session
@@ -237,7 +252,7 @@ sgrep --hybrid --colbert --semantic-weight 0.5 --bm25-weight 0.5 "JWT token"
 # Basic setup (embedding model only, ~130MB)
 sgrep setup
 
-# With cross-encoder reranking (~1.6GB additional)
+# With cross-encoder reranking (~636MB additional)
 sgrep setup --with-rerank
 ```
 
@@ -324,7 +339,8 @@ All data is stored in `~/.sgrep/`:
 │   └── d4e5f6/              # Hash of /path/to/repo2
 │       └── ...
 ├── conversations/
-│   └── conv.db              # Conversation search index
+│   ├── conv.db              # Conversation metadata, embeddings, and FTS
+│   └── turn_embeddings.tqmse # Compact TQ-MSE conversation turn vectors
 ├── server.pid               # Embedding server PID
 └── server.log               # Embedding server logs
 ```
@@ -361,10 +377,19 @@ when you want to re-compact the index after a long watch session.
 |---------|-------------|
 | `sgrep [query]` | Semantic search (default) |
 | `sgrep index [path]` | Index a directory |
+| `sgrep index [path] --colbert-codec tqmse/int8/pq6` | Select ColBERT segment codec |
+| `sgrep index [path] --sql-vectors` | Also store full SQL vectors for legacy search |
 | `sgrep watch [path]` | Watch and auto-index |
 | `sgrep list` | List all indexed repos |
 | `sgrep status` | Show index status |
 | `sgrep clear` | Clear index |
+| `sgrep conv [search] <query>` | Search indexed agent conversations |
+| `sgrep conv index [--source agent] [--watch] [--force]` | Index or watch conversations |
+| `sgrep conv view <session_id> [--turn N] [--no-color]` | View a conversation |
+| `sgrep conv export <session_id> --format markdown/json/html` | Export a conversation |
+| `sgrep conv context <session_id> [--turns N] [--copy]` | Extract context for a new session |
+| `sgrep conv copy <session_id> [--turn N] [--code-only]` | Copy a conversation or turn |
+| `sgrep conv status` | Show conversation index status |
 | `sgrep setup` | Download embedding model, verify llama-server |
 | `sgrep setup --with-rerank` | Also download reranker model (~636MB) |
 | `sgrep server start` | Manually start embedding server |
@@ -420,7 +445,7 @@ This creates:
 | `-c, --context` | Show code context |
 | `--json` | JSON output for agents |
 | `-q, --quiet` | Minimal output (paths only) |
-| `--threshold F` | L2 distance threshold (default: 1.5, lower = stricter) |
+| `--threshold F` | Cosine distance threshold (default: 1.5, lower = stricter) |
 | `-t, --include-tests` | Include test files in results (excluded by default) |
 | `--all-chunks` | Show all matching chunks (disable deduplication) |
 | `--hybrid` | Enable hybrid search (semantic + BM25) |
@@ -440,6 +465,8 @@ SGREP_PORT=8080                        # Embedding server port
 SGREP_DIMS=768                         # Vector dimensions
 SGREP_VECTOR_BACKEND=tqmse             # Force TQ-MSE search
 SGREP_VECTOR_BACKEND=libsql            # Force legacy SQL vector search (requires --sql-vectors index)
+SGREP_CONV_VECTOR_BACKEND=tqmse        # Force TQ-MSE conversation vector search
+SGREP_CONV_VECTOR_BACKEND=sqlite       # Force legacy SQL conversation vector scan
 ```
 
 ## How It Works

internal/cli/conv.go

@@ -572,6 +572,11 @@ func runConvIndex(cmd *cobra.Command, args []string) error {
 		}
 	}
 
+	vecCount, err := indexer.RebuildTQVectorStore(ctx)
+	if err != nil {
+		return fmt.Errorf("refresh compact TQ-MSE conversation vectors: %w", err)
+	}
+	fmt.Printf("Refreshed compact TQ-MSE conversation vectors (%d turns)\n", vecCount)
 	fmt.Printf("\nTotal: %d sessions, %d turns indexed\n", totalSessions, totalTurns)
 	return nil
 }

internal/cli/conv_watch.go

@@ -177,7 +177,12 @@ func indexConversationsOnce(ctx context.Context, indexer *conv.Indexer, parsers
 			totalTurns += result.TurnsIndexed
 		}
 	}
+	vecCount, err := indexer.RebuildTQVectorStore(ctx)
+	if err != nil {
+		return fmt.Errorf("refresh compact TQ-MSE conversation vectors: %w", err)
+	}
 	if verbose {
+		fmt.Printf("Refreshed compact TQ-MSE conversation vectors (%d turns)\n", vecCount)
 		fmt.Printf("Initial index complete: %d sessions, %d turns\n", totalSessions, totalTurns)
 	}
 	return nil
@@ -194,6 +199,7 @@ func processConvJobs(ctx context.Context, indexer *conv.Indexer, pending map[str
 	}
 	mu.Unlock()
 
+	var indexedTurns int
 	for _, job := range jobs {
 		sessions, err := job.parser.Parse(job.path)
 		if err != nil {
@@ -221,6 +227,19 @@ func processConvJobs(ctx context.Context, indexer *conv.Indexer, pending map[str
 			fmt.Printf("Indexed %d sessions (%d turns) from %s\n",
 				result.SessionsIndexed, result.TurnsIndexed, job.path)
 		}
+		indexedTurns += result.TurnsIndexed
+	}
+	if indexedTurns > 0 {
+		vecCount, err := indexer.RebuildTQVectorStore(ctx)
+		if err != nil {
+			if verbose {
+				fmt.Fprintf(os.Stderr, "Warning: refresh compact TQ-MSE conversation vectors failed: %v\n", err)
+			}
+			return
+		}
+		if verbose {
+			fmt.Printf("Refreshed compact TQ-MSE conversation vectors (%d turns)\n", vecCount)
+		}
 	}
 }
 

pkg/conv/indexer.go

@@ -119,6 +119,11 @@ func (idx *Indexer) IndexSession(ctx context.Context, session *Session) error {
 	return idx.indexSession(ctx, session)
 }
 
+// RebuildTQVectorStore refreshes the compact TQ-MSE sidecar for conversation search.
+func (idx *Indexer) RebuildTQVectorStore(ctx context.Context) (int, error) {
+	return idx.store.RebuildTQVectorStore(ctx)
+}
+
 // indexSession indexes a single session and its turns.
 func (idx *Indexer) indexSession(ctx context.Context, session *Session) error {
 	// Estimate tokens

pkg/conv/search.go

@@ -46,9 +46,12 @@ func (s *Searcher) Search(ctx context.Context, query string, opts SearchOptions)
 			return nil, fmt.Errorf("failed to generate query embedding: %w", err)
 		}
 
-		if opts.UseHybrid {
+		hasFilters := opts.Agent != AgentAll || opts.Project != "" || !opts.Since.IsZero() || !opts.Before.IsZero()
+		if opts.UseHybrid && hasFilters {
+			results, err = s.store.FilteredHybridSearch(ctx, queryEmb, searchterms.ExtractHybridSearchTerms(query), opts)
+		} else if opts.UseHybrid {
 			results, err = s.store.HybridSearch(ctx, queryEmb, searchterms.ExtractHybridSearchTerms(query), opts.Limit*2, opts.Threshold, opts.SemanticWeight, opts.BM25Weight)
-		} else if opts.Agent != AgentAll || opts.Project != "" || !opts.Since.IsZero() || !opts.Before.IsZero() {
+		} else if hasFilters {
 			results, err = s.store.FilteredSearch(ctx, queryEmb, opts)
 		} else {
 			results, err = s.store.VectorSearch(ctx, queryEmb, opts.Limit*2, opts.Threshold)

pkg/conv/search_test.go

@@ -188,6 +188,75 @@ func TestSearcherExactUsesKeywordSearchWithoutEmbedder(t *testing.T) {
 	}
 }
 
+func TestFilteredHybridSearchAppliesAgentFilter(t *testing.T) {
+	tmpDir := t.TempDir()
+	store, err := NewStore(StoreConfig{DBPath: filepath.Join(tmpDir, "test.db"), Dims: defaultDims})
+	if err != nil {
+		t.Fatalf("failed to create store: %v", err)
+	}
+	defer func() { _ = store.Close() }()
+
+	ctx := context.Background()
+	now := time.Now()
+	sessions := []*Session{
+		{
+			ID:          "claude-auth",
+			Agent:       AgentClaudeCode,
+			ProjectPath: "/repo/auth",
+			ProjectName: "auth",
+			StartedAt:   now,
+			EndedAt:     now,
+			Turns: []Turn{
+				{Index: 0, UserContent: "authentication jwt", AssistContent: "claude answer"},
+			},
+		},
+		{
+			ID:          "codex-auth",
+			Agent:       AgentCodexCLI,
+			ProjectPath: "/repo/auth",
+			ProjectName: "auth",
+			StartedAt:   now,
+			EndedAt:     now,
+			Turns: []Turn{
+				{Index: 0, UserContent: "authentication jwt", AssistContent: "codex answer"},
+			},
+		},
+	}
+	for _, session := range sessions {
+		if err := store.StoreSession(ctx, session); err != nil {
+			t.Fatalf("failed to store session: %v", err)
+		}
+	}
+
+	embedding := make([]float32, defaultDims)
+	embedding[0] = 1
+	if err := store.StoreTurnEmbeddingBatch(ctx,
+		[]string{"claude-auth:0", "codex-auth:0"},
+		[][]float32{embedding, embedding},
+	); err != nil {
+		t.Fatalf("failed to store embeddings: %v", err)
+	}
+
+	opts := DefaultSearchOptions()
+	opts.Agent = AgentCodexCLI
+	opts.Limit = 10
+	opts.Threshold = 0
+
+	results, err := store.FilteredHybridSearch(ctx, embedding, "authentication", opts)
+	if err != nil {
+		t.Fatalf("hybrid search failed: %v", err)
+	}
+	if len(results) != 1 {
+		t.Fatalf("expected 1 filtered result, got %d", len(results))
+	}
+	if results[0].SessionID != "codex-auth" {
+		t.Fatalf("expected codex-auth, got %s", results[0].SessionID)
+	}
+	if results[0].MatchType != "hybrid" {
+		t.Fatalf("expected hybrid match type, got %q", results[0].MatchType)
+	}
+}
+
 func TestConversationHit_Fields(t *testing.T) {
 	hit := ConversationHit{
 		SessionID:   "test-session",