Filter stopwords and short tokens in files backend search

Author: ellmos-ai Release Snapshot

ROADMAP.md

@@ -10,4 +10,21 @@ Future work is tracked against observable acceptance criteria:
 - Publish signed artifacts after the release repository and reporting channel
   have been verified.
 
+## Follow-ups (2026-07-31)
+
+- Normalize the files-backend ranking (for example `score / max_score` or
+  length normalization) so that `min_rank` becomes meaningful again; the
+  current `score / (score + 3)` saturation renders everything above ~300
+  as 1.00.
+- Support individual file paths in the files backend; today only directories
+  are scanned recursively and an explicit file list is silently ignored.
+- Add an FTS5 backend built on the stdlib `sqlite3` module (unicode61
+  tokenizer, bm25 mapped into (0, 1]); the backend protocol already
+  anticipates FTS5-bm25 ranking. This addresses stopword inflation, score
+  saturation and size bias structurally without new dependencies.
+- Add a forgetfulness sidecar to the index (`last_accessed` / `access_count`
+  with decay and a review threshold instead of hard deletion).
+- Optionally add an embedding-based hybrid backend that reports
+  `available() = False` when the optional packages are not installed.
+
 Roadmap items are not promises and may change.

memoryhooker/backends/files.py

@@ -8,12 +8,45 @@
 
 from __future__ import annotations
 
+import re
 from pathlib import Path
 
 from ..protocol import Hit
 
 _SNIPPET_RADIUS = 120
 
+# Mindestlaenge fuer Suchterme; kuerzere Tokens sind fast immer Rauschen.
+_MIN_TERM_LENGTH = 3
+
+# Wortzeichen (inkl. Umlaute) und Bindestrich -- Satzzeichen fallen so
+# beim Tokenisieren automatisch weg (".TOPICS," -> "TOPICS").
+_TOKEN_PATTERN = re.compile(r"[\w-]+", re.UNICODE)
+
+# Stopwoerter Deutsch + Englisch: Fuellwoerter ohne Eigeninfo. Als Suchterm
+# treffen sie per Substr-Match fast jeden Text ("fuer" in "Verfuegung")
+# und dominieren das Ranking. Wartbar als Modul-Konstante.
+_STOPWORDS = frozenset(
+    {
+        # Deutsch
+        "aber", "alle", "als", "am", "an", "auch", "auf", "aus", "bei",
+        "beim", "bin", "bis", "da", "dann", "das", "dass", "dem", "den",
+        "der", "des", "die", "dies", "diese", "dieser", "du", "durch",
+        "ein", "eine", "einem", "einen", "einer", "er", "es", "für",
+        "fuer", "habe", "haben", "hat", "ich", "ihm", "ihn", "ihr",
+        "ihre", "im", "in", "ins", "ist", "kein", "keine", "man", "mein",
+        "meine", "mit", "nach", "nicht", "noch", "nur", "ob", "oder",
+        "ohne", "sein", "seine", "sich", "sie", "sind", "so", "über",
+        "ueber", "um", "und", "uns", "vom", "von", "vor", "war", "was",
+        "weil", "wenn", "wie", "wir", "wird", "wo", "zu", "zum", "zur",
+        # Englisch
+        "the", "and", "for", "with", "this", "that", "from", "have",
+        "has", "are", "was", "were", "not", "but", "all", "can", "you",
+        "your", "its", "our", "their", "them", "they", "what", "which",
+        "when", "where", "how", "then", "there", "here", "into", "only",
+        "very", "just",
+    }
+)
+
 
 class FilesBackend:
     """Sucht in ``*.md``-Dateien unterhalb eines oder mehrerer Wurzeln."""
@@ -38,7 +71,7 @@ def search(self, query: str, limit: int = 5) -> list[Hit]:
         if not self.available():
             return []
 
-        terms = [t.lower() for t in query.split() if t.strip()]
+        terms = _terms_from_query(query)
         if not terms:
             return []
 
@@ -81,6 +114,20 @@ def search(self, query: str, limit: int = 5) -> list[Hit]:
         return hits[:limit]
 
 
+def _terms_from_query(query: str) -> list[str]:
+    """Normalisiert eine Query zu Suchtermen.
+
+    Kleinschreibung, Satzzeichen gestrippt, Mindestlaenge 3, Stopwoerter
+    raus. Bleibt kein Term uebrig (leere oder reine Fuellwort-Query),
+    liefert ``search()`` keine Treffer statt Vollrauschen.
+    """
+    return [
+        term
+        for term in _TOKEN_PATTERN.findall(query.lower())
+        if len(term) >= _MIN_TERM_LENGTH and term not in _STOPWORDS
+    ]
+
+
 def _snippet(text: str, lowered: str, terms: list[str]) -> str:
     for term in terms:
         idx = lowered.find(term)

tests/test_backends_files.py

@@ -62,3 +62,50 @@ def test_search_recurses_into_subdirectories(tmp_path: Path):
     backend = FilesBackend(tmp_path)
     hits = backend.search("gardener")
     assert hits and hits[0].source == str(Path("sub") / "nested.md")
+
+
+def test_search_ignores_stopwords(tmp_path: Path):
+    # "für" steckt als Substr in "Verfügung"/"dafür": ohne Stopwortfilter
+    # wuerde die Fuellwort-Query hier faelschlich treffen.
+    (tmp_path / "a.md").write_text(
+        "Hinweis zur Verfügung: dafür ist der Maintainer zuständig.", encoding="utf-8"
+    )
+
+    backend = FilesBackend(tmp_path)
+    assert backend.search("Rezept für Tomatensuppe") == []
+
+
+def test_search_with_keywords_still_hits(tmp_path: Path):
+    (tmp_path / "a.md").write_text(
+        "Tomatensuppe: Zwiebeln anschwitzen, Tomaten dazu.", encoding="utf-8"
+    )
+    (tmp_path / "b.md").write_text("Anderes Thema.", encoding="utf-8")
+
+    backend = FilesBackend(tmp_path)
+    hits = backend.search("Rezept für Tomatensuppe")
+    assert len(hits) == 1
+    assert hits[0].source == "a.md"
+
+
+def test_search_stopword_only_query_returns_nothing(tmp_path: Path):
+    (tmp_path / "a.md").write_text("Für das und mit dem ist alles getan.", encoding="utf-8")
+
+    backend = FilesBackend(tmp_path)
+    assert backend.search("für das und mit") == []
+
+
+def test_search_ignores_terms_shorter_than_three_chars(tmp_path: Path):
+    (tmp_path / "a.md").write_text("py py py everywhere", encoding="utf-8")
+
+    backend = FilesBackend(tmp_path)
+    assert backend.search("py") == []
+
+
+def test_search_strips_punctuation_from_terms(tmp_path: Path):
+    (tmp_path / "a.md").write_text("Die Datei heisst memoryhooker.toml", encoding="utf-8")
+
+    backend = FilesBackend(tmp_path)
+    # "?" und "." duerfen den Term nicht veraendern: "memoryhooker.toml?"
+    # als Ganzes kaeme im Text nie vor, die Tokens schon.
+    hits = backend.search("Was ist memoryhooker.toml?")
+    assert hits and hits[0].source == "a.md"