fix: improve recall archive discovery and memory validation

Author: T0UGH

pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "csep"
-version = "1.3.5"
+version = "1.3.6"
 description = "Self-evolution loop for OpenAI Codex: session reflection, session recall, and reflection-generated skills with zero Python runtime dependencies."
 readme = "README.md"
 requires-python = ">=3.11"

src/codex_self_evolution/session_recall/archive.py

@@ -200,6 +200,26 @@ def discover_transcript_for_hook_payload(
                 "candidate_count": len(meta_matches),
             }
 
+    turn_discovery = _recent_content_discovery(
+        recent_files,
+        payload,
+        key="turn_id",
+        reason="recent_turn_id",
+        min_chars=6,
+    )
+    if turn_discovery:
+        return turn_discovery
+
+    assistant_discovery = _recent_content_discovery(
+        recent_files,
+        payload,
+        key="last_assistant_message",
+        reason="recent_last_assistant_message",
+        min_chars=24,
+    )
+    if assistant_discovery:
+        return assistant_discovery
+
     cwd = str(payload.get("cwd") or "")
     if cwd:
         cwd_matches = [
@@ -427,6 +447,84 @@ def _session_meta(path: Path) -> dict[str, Any]:
     return {}
 
 
+def _recent_content_discovery(
+    recent_files: list[Path],
+    payload: dict[str, Any],
+    *,
+    key: str,
+    reason: str,
+    min_chars: int,
+) -> dict[str, Any] | None:
+    """Return a unique recent transcript whose JSON content contains a payload value."""
+    needle = _normalized_search_text(str(payload.get(key) or ""))
+    if len(needle) < min_chars:
+        return None
+    matches = [
+        path for path in recent_files
+        if _transcript_contains_normalized_text(path, needle)
+    ]
+    matches = _prefer_payload_cwd(matches, payload)
+    if len(matches) == 1:
+        return _discovered(matches[0], reason)
+    if len(matches) > 1:
+        return {
+            "status": "discovery_ambiguous",
+            "reason": reason,
+            "path": "",
+            "candidate_count": len(matches),
+        }
+    return None
+
+
+def _prefer_payload_cwd(paths: list[Path], payload: dict[str, Any]) -> list[Path]:
+    """Prefer content matches whose session metadata cwd matches the hook payload."""
+    if len(paths) <= 1:
+        return paths
+    cwd = str(payload.get("cwd") or "")
+    if not cwd:
+        return paths
+    cwd_matches = [
+        path for path in paths
+        if _same_path(str(_session_meta(path).get("cwd") or ""), cwd)
+    ]
+    return cwd_matches or paths
+
+
+def _transcript_contains_normalized_text(path: Path, needle: str) -> bool:
+    """Return whether any JSON leaf string or raw line contains normalized text."""
+    try:
+        with path.open("r", encoding="utf-8", errors="replace") as handle:
+            for raw in handle:
+                if needle in _normalized_search_text(raw):
+                    return True
+                try:
+                    entry = json.loads(raw)
+                except ValueError:
+                    continue
+                if any(needle in _normalized_search_text(value) for value in _json_leaf_strings(entry)):
+                    return True
+    except OSError:
+        return False
+    return False
+
+
+def _json_leaf_strings(value: Any) -> Iterable[str]:
+    """Yield all string leaf values from JSON-like transcript entries."""
+    if isinstance(value, str):
+        yield value
+    elif isinstance(value, dict):
+        for child in value.values():
+            yield from _json_leaf_strings(child)
+    elif isinstance(value, list):
+        for child in value:
+            yield from _json_leaf_strings(child)
+
+
+def _normalized_search_text(value: str) -> str:
+    """Normalize transcript text for robust exact-substring matching."""
+    return " ".join(value.split()).lower()
+
+
 def _same_path(left: str, right: str) -> bool:
     if not left or not right:
         return False

src/codex_self_evolution/session_reflection/validation.py

@@ -227,35 +227,86 @@ def _hot_memory_has_noop_reflection_summary(text: str) -> bool:
     """Return whether hot memory contains a per-run duplicate/no-new reflection line."""
     for line in text.splitlines():
         normalized = line.lower()
-        if _has_noop_memory_signal(normalized) and _has_reflection_review_signal(normalized):
+        if (
+            _has_reflection_review_signal(normalized)
+            and (_has_noop_memory_signal(normalized) or _has_duplicate_only_memory_signal(normalized))
+        ):
             return True
     return False
 
 
 def _memory_ref_is_noop_reflection_record(path: Path, text: str) -> bool:
     """Return whether a ref file is only a duplicate/no-new reflection ledger."""
     normalized = text.lower()
-    if not _has_noop_memory_signal(normalized):
+    if not (_has_noop_memory_signal(normalized) or _has_duplicate_only_memory_signal(normalized)):
         return False
-    if "csep-reflection-memory-skills" in path.name:
+    if _has_reflection_ledger_filename(path):
         return True
     return _has_reflection_review_signal(normalized)
 
 
+def _has_reflection_ledger_filename(path: Path) -> bool:
+    """Return whether the ref filename is generated reflection-review bookkeeping."""
+    name = path.name.lower()
+    return any(
+        marker in name
+        for marker in (
+            "csep-reflection-memory-skills",
+            "csep-memory-skills-duplicate",
+            "csep-reflection-review",
+        )
+    )
+
+
 def _has_noop_memory_signal(text: str) -> bool:
     """Return whether text says the reflection found no reusable durable candidate."""
     return any(
         token in text
         for token in (
             "未识别新增",
             "未发现新增",
+            "未出现可沉淀",
+            "未发现可沉淀",
+            "未发现可写入",
+            "未新增可",
             "无新增",
             "nothing reusable",
             "no new reusable",
         )
     )
 
 
+def _has_duplicate_only_memory_signal(text: str) -> bool:
+    """Return whether text says all candidates were duplicate or already covered."""
+    duplicate_signal = any(
+        token in text
+        for token in (
+            "duplicate",
+            "复用既有",
+            "既有",
+            "等价",
+            "already covered",
+        )
+    )
+    no_durable_signal = any(
+        token in text
+        for token in (
+            "没有新的",
+            "未产生",
+            "仅记录",
+            "仅需",
+            "无需",
+            "不写入",
+            "不产生",
+            "未创建",
+            "未新增",
+            "no durable",
+            "no new",
+        )
+    )
+    return duplicate_signal and no_durable_signal
+
+
 def _has_reflection_review_signal(text: str) -> bool:
     """Return whether text is about the CSEP memory/skill reflection review."""
     return any(

tests/test_session_recall_archive.py

@@ -180,6 +180,79 @@ def test_archive_from_hook_payload_does_not_discover_ambiguous_recent_cwd(tmp_pa
         store.close()
 
 
+def test_archive_from_hook_payload_disambiguates_recent_cwd_by_last_assistant_message(tmp_path):
+    from codex_self_evolution.session_recall.archive import archive_from_hook_payload
+
+    sessions_root = tmp_path / "sessions"
+    repo = tmp_path / "repo"
+    repo.mkdir()
+    _write_transcript(sessions_root / "one.jsonl", session_id="one", cwd=str(repo), message="other session")
+    matched = sessions_root / "two.jsonl"
+    _write_transcript(
+        matched,
+        session_id="two",
+        cwd=str(repo),
+        message="The exact final assistant sentence that appears in the Stop payload.",
+    )
+    payload = tmp_path / "payload.json"
+    payload.write_text(
+        json.dumps(
+            {
+                "session_id": "payload-session",
+                "cwd": str(repo),
+                "last_assistant_message": "The exact final assistant sentence that appears in the Stop payload.",
+            }
+        ),
+        encoding="utf-8",
+    )
+
+    result = archive_from_hook_payload(
+        payload,
+        db_path=tmp_path / "state.db",
+        sessions_root=sessions_root,
+    )
+
+    assert result["status"] == "archived"
+    assert result["session_id"] == "two"
+    assert result["discovery_reason"] == "recent_last_assistant_message"
+    assert result["discovered_transcript_path"] == str(matched.resolve())
+
+
+def test_archive_from_hook_payload_discovers_recent_turn_id(tmp_path):
+    from codex_self_evolution.session_recall.archive import archive_from_hook_payload
+
+    sessions_root = tmp_path / "sessions"
+    repo = tmp_path / "repo"
+    repo.mkdir()
+    transcript = sessions_root / "turn-match.jsonl"
+    transcript.parent.mkdir(parents=True, exist_ok=True)
+    transcript.write_text(
+        "\n".join(
+            [
+                json.dumps({"type": "session_meta", "payload": {"id": "turn-session", "cwd": str(repo)}}),
+                json.dumps({"id": "turn-abc", "role": "assistant", "content": "archive this turn"}),
+                "",
+            ]
+        ),
+        encoding="utf-8",
+    )
+    payload = tmp_path / "payload.json"
+    payload.write_text(
+        json.dumps({"session_id": "payload-session", "turn_id": "turn-abc", "cwd": str(repo)}),
+        encoding="utf-8",
+    )
+
+    result = archive_from_hook_payload(
+        payload,
+        db_path=tmp_path / "state.db",
+        sessions_root=sessions_root,
+    )
+
+    assert result["status"] == "archived"
+    assert result["session_id"] == "turn-session"
+    assert result["discovery_reason"] == "recent_turn_id"
+
+
 def test_archive_from_hook_payload_records_error_for_missing_transcript(tmp_path):
     from codex_self_evolution.session_recall.archive import archive_from_hook_payload
     from codex_self_evolution.session_recall.store import SessionRecallStore

tests/test_session_reflection_validation.py

@@ -196,6 +196,95 @@ def test_validate_receipt_flags_duplicate_only_reflection_memory_writes(tmp_path
     ]
 
 
+def test_validate_receipt_flags_duplicate_reflection_refs_without_fixed_noop_phrase(tmp_path: Path) -> None:
+    """Reflection refs that only say candidates are duplicate should not be durable memory."""
+    memory_root = tmp_path / "project" / "memory"
+    ref = memory_root / "refs" / "20260519T113111Z-csep-memory-skills-duplicate.md"
+    ref.parent.mkdir(parents=True)
+    ref.write_text(
+        "## CSEP 反思作业\n\n"
+        "本轮 `memory and skills` 复盘复用既有 `csep-reflect-session-receipt` 流程。\n\n"
+        "- memory 候选：没有新的长期 fact、rule、preference。\n"
+        "- skill 候选：workflow 路径与既有 skill 等价，判定为 duplicate。\n"
+        "- durable 变更：仅记录本轮复盘摘要和 receipt.json。\n",
+        encoding="utf-8",
+    )
+    receipt = tmp_path / "receipt.json"
+    _write_receipt(
+        receipt,
+        memory_changes=[{"path": str(ref), "action": "add"}],
+        skill_changes=[],
+    )
+
+    result = validate_receipt(
+        receipt,
+        memory_roots=[memory_root],
+        skills_root=tmp_path / "skills",
+        skill_prefix="csep-reflect-",
+    )
+
+    assert result["status"] == "partial"
+    assert result["low_value_memory_writes"] == [{"reason": "memory_reflection_noop_ref", "path": str(ref)}]
+
+
+def test_validate_receipt_flags_no_writable_memory_reflection_refs(tmp_path: Path) -> None:
+    """No-writable-memory reflection refs are duplicate ledgers even without filename markers."""
+    memory_root = tmp_path / "project" / "memory"
+    ref = memory_root / "refs" / "20260519T090154Z-39cbc123.md"
+    ref.parent.mkdir(parents=True)
+    ref.write_text(
+        "# CSEP 复盘记录\n\n"
+        "- 任务范围：memory and skills。\n"
+        "- 结果：未发现可写入长期 MEMORY 的新 fact/rule/preference。\n"
+        "- 判定：继续复用现有 workflow，命中 `csep-reflect-session-receipt`。\n"
+        "- 处理：本次无生产代码/配置改动、无新技能落地。\n",
+        encoding="utf-8",
+    )
+    receipt = tmp_path / "receipt.json"
+    _write_receipt(
+        receipt,
+        memory_changes=[{"path": str(ref), "action": "add"}],
+        skill_changes=[],
+    )
+
+    result = validate_receipt(
+        receipt,
+        memory_roots=[memory_root],
+        skills_root=tmp_path / "skills",
+        skill_prefix="csep-reflect-",
+    )
+
+    assert result["status"] == "partial"
+    assert result["low_value_memory_writes"] == [{"reason": "memory_reflection_noop_ref", "path": str(ref)}]
+
+
+def test_validate_receipt_allows_non_reflection_duplicate_memory(tmp_path: Path) -> None:
+    """Domain notes may mention duplicate behavior without being reflection noise."""
+    memory_root = tmp_path / "project" / "memory"
+    ref = memory_root / "refs" / "order-idempotency.md"
+    ref.parent.mkdir(parents=True)
+    ref.write_text(
+        "订单接口会把 duplicate request id 作为幂等重试处理，调用方可以复用同一个 request id 查询结果。\n",
+        encoding="utf-8",
+    )
+    receipt = tmp_path / "receipt.json"
+    _write_receipt(
+        receipt,
+        memory_changes=[{"path": str(ref), "action": "add"}],
+        skill_changes=[],
+    )
+
+    result = validate_receipt(
+        receipt,
+        memory_roots=[memory_root],
+        skills_root=tmp_path / "skills",
+        skill_prefix="csep-reflect-",
+    )
+
+    assert result["status"] == "succeeded"
+    assert result["low_value_memory_writes"] == []
+
+
 def test_validate_receipt_rejects_legacy_user_memory(tmp_path: Path) -> None:
     """Legacy USER.md is retained on disk but is no longer writable memory."""
     memory = tmp_path / "project" / "memory" / "USER.md"