fix: tighten synthesized skill quality gates

Author: T0UGH

plugins/codex-self-evolution/.codex-plugin/plugin.json

@@ -1,6 +1,6 @@
 {
   "name": "codex-self-evolution",
-  "version": "0.7.7",
+  "version": "0.7.8",
   "description": "Scheduled self-evolution loop with stable background memory, provider-backed stop review, explicit suggestion states, launchd compiler preflight, focused recall, and independent csep-synth skill synthesis.",
   "author": {
     "name": "T0UGH",

pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "codex-self-evolution-plugin"
-version = "0.7.7"
+version = "0.7.8"
 description = "Self-evolution loop for OpenAI Codex: Stop-hook reviewer, memory/recall compiler, and independent skill synthesis, zero runtime dependencies."
 readme = "README.md"
 requires-python = ">=3.11"

src/codex_self_evolution/__init__.py

@@ -1,3 +1,3 @@
 __all__ = ["__version__"]
 
-__version__ = "0.7.7"
+__version__ = "0.7.8"

src/codex_self_evolution/plugin_bundle/.codex-plugin/plugin.json

@@ -1,6 +1,6 @@
 {
   "name": "codex-self-evolution",
-  "version": "0.7.7",
+  "version": "0.7.8",
   "description": "Scheduled self-evolution loop with stable background memory, provider-backed stop review, explicit suggestion states, launchd compiler preflight, focused recall, and independent csep-synth skill synthesis.",
   "author": {
     "name": "T0UGH",

src/codex_self_evolution/skill_synthesis/agent.py

@@ -13,9 +13,11 @@ class SkillSynthesisAgentError(RuntimeError):
 
 
 AgentInvoker = Callable[[list[str], float], subprocess.CompletedProcess[str]]
+_EVIDENCE_KINDS = {"workflow", "fact", "rule", "preference", "duplicate", "transient"}
 
 
 def build_skill_synthesis_prompt(*, run_input_dir: Path, run_output_dir: Path, skills_root: Path) -> str:
+    """Build the bounded SOP and result contract for the synthesis agent."""
     result_path = run_output_dir / "result.json"
     return (
         "You are the codex-self-evolution skill synthesis agent.\n\n"
@@ -29,6 +31,52 @@ def build_skill_synthesis_prompt(*, run_input_dir: Path, run_output_dir: Path, s
         "4. Create, edit, or retire skills only for repeated reusable workflows.\n"
         "5. Do not create skills for one-off facts, temporary state, secrets, or account identifiers.\n"
         "6. Retired skills must use description \"Retired generated skill. Do not use.\" and csep_status: retired.\n\n"
+        "Classify every candidate before writing anything: fact|rule|preference|workflow|duplicate|transient.\n"
+        "Only workflow candidates may become active skills. Facts belong in memory, behavior rules belong in AGENTS/memory,\n"
+        "preferences belong in USER.md, transient state should be skipped, and duplicate workflows should edit or retire an existing csep-synth skill.\n\n"
+        "Active SKILL.md frontmatter must be simple single-line YAML:\n"
+        "---\n"
+        "name: csep-synth-example\n"
+        "description: Use when repeated example workflow needs concrete command guidance.\n"
+        "---\n"
+        "Do not use block scalar descriptions such as `description: |`.\n"
+        "The frontmatter name must match the csep-synth-* directory name exactly.\n"
+        "Only retired skills may contain the retired body line.\n\n"
+        "Active SKILL.md bodies must use this contract:\n"
+        "# <Skill title>\n\n"
+        "## Skill Decision\n\n"
+        "- Why this is a skill: <why the evidence is a reusable workflow>\n"
+        "- Why not memory: <why this is not a static fact, preference, or rule>\n"
+        "- Existing skill boundary: <nearest existing skill and why this is not duplicate>\n\n"
+        "## When to Use\n\n"
+        "<concrete trigger conditions>\n\n"
+        "## Inputs\n\n"
+        "<required ids, paths, links, env vars, or prerequisites>\n\n"
+        "## Workflow\n\n"
+        "<ordered tool/command steps>\n\n"
+        "## Verification\n\n"
+        "<evidence that proves success>\n\n"
+        "## Failure Handling\n\n"
+        "<when to stop, ask, skip, retire, or report uncertainty>\n\n"
+        "Write result.json with exactly this schema:\n"
+        "{\n"
+        "  \"schema_version\": 1,\n"
+        "  \"run_id\": \"<current run id if known>\",\n"
+        "  \"actions\": [\n"
+        "    {\n"
+        "      \"action\": \"create|edit|retire|skip\",\n"
+        "      \"skill_id\": \"csep-synth-example\",\n"
+        "      \"path\": \"<absolute path to SKILL.md, or empty for skip>\",\n"
+        "      \"evidence_keys\": [\"<evidence key>\"],\n"
+        "      \"reason\": \"why this action is justified\",\n"
+        "      \"evidence_kind\": \"workflow|fact|rule|preference|duplicate|transient\",\n"
+        "      \"why_skill_not_memory\": \"required for create/edit\",\n"
+        "      \"existing_skill_overlap\": \"nearest existing skill or no direct overlap\"\n"
+        "    }\n"
+        "  ],\n"
+        "  \"notes\": \"optional\"\n"
+        "}\n"
+        "Do not write legacy result shapes such as synthesized, valid_skills, invalid, retired, or skipped as top-level substitutes for actions.\n\n"
         "After writing files, return exactly DONE."
     )
 
@@ -86,6 +134,7 @@ def _subprocess_run(cmd: list[str], timeout_seconds: float) -> subprocess.Comple
 
 
 def parse_agent_result(path: Path) -> dict[str, Any]:
+    """Parse and validate the synthesis agent's result.json contract."""
     if not path.is_file():
         return {"valid": False, "actions": [], "result_missing": True, "result_invalid": False}
     try:
@@ -107,12 +156,25 @@ def parse_agent_result(path: Path) -> dict[str, Any]:
             return {"valid": False, "actions": [], "result_missing": False, "result_invalid": True, "error": "bad_action"}
         if skill_id and not re.fullmatch(r"csep-synth-[a-z0-9-]+", skill_id):
             return {"valid": False, "actions": [], "result_missing": False, "result_invalid": True, "error": "bad_skill_id"}
+        evidence_kind = str(item.get("evidence_kind") or "")
+        why_skill_not_memory = str(item.get("why_skill_not_memory") or "").strip()
+        existing_skill_overlap = str(item.get("existing_skill_overlap") or "").strip()
+        if evidence_kind and evidence_kind not in _EVIDENCE_KINDS:
+            return {"valid": False, "actions": [], "result_missing": False, "result_invalid": True, "error": "bad_evidence_kind"}
+        if action in {"create", "edit"}:
+            if not evidence_kind or not why_skill_not_memory or not existing_skill_overlap:
+                return {"valid": False, "actions": [], "result_missing": False, "result_invalid": True, "error": "missing_skill_decision"}
+            if evidence_kind != "workflow":
+                return {"valid": False, "actions": [], "result_missing": False, "result_invalid": True, "error": "non_workflow_skill"}
         parsed.append({
             "action": action,
             "skill_id": skill_id,
             "path": str(item.get("path") or ""),
             "evidence_keys": [str(v) for v in item.get("evidence_keys", []) if isinstance(v, str)],
             "reason": str(item.get("reason") or ""),
+            "evidence_kind": evidence_kind,
+            "why_skill_not_memory": why_skill_not_memory,
+            "existing_skill_overlap": existing_skill_overlap,
         })
     return {
         "valid": True,

src/codex_self_evolution/skill_synthesis/runner.py

@@ -149,10 +149,16 @@ def _run_locked(
     namespace_violations = _non_synth_changed_paths(changed_paths(all_before, all_after))
     changed_real = changed_paths(real_before, real_after) if dry_run else []
     changed_real_all = changed_paths(real_all_before, real_all_after) if dry_run else []
-    valid, invalid, retired = _validate_changed(run_id, detected)
+    reported_written = _reported_written(agent_result)
+    validation_targets = _validation_targets(
+        skills_root=skills_root,
+        changed=detected,
+        reported=reported_written,
+    )
+    valid, invalid, retired = _validate_changed(run_id, validation_targets)
     result_missing = bool(agent_result.get("result_missing"))
     result_invalid = bool(agent_result.get("result_invalid"))
-    mismatch = _reported_written(agent_result) != detected
+    mismatch = reported_written != detected
     dry_run_leak = dry_run and bool(changed_real)
     status = "success"
     if error or dry_run_leak or namespace_violations:
@@ -170,7 +176,7 @@ def _run_locked(
         "evidence_count": len(evidence),
         "new_evidence_count": sum(1 for item in evidence if not item.get("seen_before")),
         "seen_before_count": sum(1 for item in evidence if item.get("seen_before")),
-        "reported_written": _reported_written(agent_result),
+        "reported_written": reported_written,
         "detected_changed": detected,
         "changed_real_paths": changed_real_all or changed_real,
         "namespace_violations": namespace_violations,
@@ -193,21 +199,47 @@ def _run_locked(
 
 
 def _reported_written(agent_result: dict[str, Any]) -> list[str]:
+    """Return sorted SKILL.md paths the agent claims to have written."""
     out: list[str] = []
     for action in agent_result.get("actions", []):
         if action.get("action") in {"create", "edit", "retire"} and action.get("path"):
             out.append(str(action["path"]))
     return sorted(out)
 
 
+def _validation_targets(*, skills_root: Path, changed: list[str], reported: list[str]) -> list[str]:
+    """Select generated skill files that should receive post-run validation."""
+    root = skills_root.expanduser().resolve(strict=False)
+    out = {
+        str(Path(raw).expanduser())
+        for raw in [*changed, *reported]
+        if _under_root(Path(raw).expanduser(), root)
+    }
+    if skills_root.is_dir():
+        for child in sorted(skills_root.iterdir()):
+            if child.is_dir() and child.name.startswith("csep-synth-"):
+                out.add(str(child / "SKILL.md"))
+    return sorted(out)
+
+
+def _under_root(path: Path, root: Path) -> bool:
+    """Return whether path is under the current writable skills root."""
+    try:
+        return path.resolve(strict=False).is_relative_to(root)
+    except OSError:
+        return False
+
+
 def _non_synth_changed_paths(paths: list[str]) -> list[str]:
+    """Return changed skill docs outside the csep-synth namespace."""
     return sorted(
         path for path in paths
         if not Path(path).parent.name.startswith("csep-synth-")
     )
 
 
 def _validate_changed(run_id: str, changed: list[str]) -> tuple[list[dict[str, Any]], list[dict[str, Any]], list[dict[str, Any]]]:
+    """Validate generated skill docs and maintain their invalid markers."""
     valid: list[dict[str, Any]] = []
     invalid: list[dict[str, Any]] = []
     retired: list[dict[str, Any]] = []

src/codex_self_evolution/skill_synthesis/validation.py

@@ -12,13 +12,29 @@
 
 RETIRED_DESCRIPTION = "Retired generated skill. Do not use."
 RETIRED_BODY_LINE = "Do not use this skill. It is kept only for audit history."
+_BLOCK_SCALAR_MARKERS = {"|", "|-", "|+", ">", ">-", ">+"}
+_ACTIVE_DESCRIPTION_PREFIXES = ("use when ", "this skill should be used when ")
+_WORKFLOW_HEADINGS = (
+    "## when to use",
+    "## inputs",
+    "## workflow",
+    "## verification",
+    "## failure handling",
+)
+_SKILL_DECISION_MARKERS = (
+    "## skill decision",
+    "why this is a skill",
+    "why not memory",
+    "existing skill boundary",
+)
 
 
 def _now() -> str:
     return datetime.now(UTC).replace(microsecond=0).isoformat().replace("+00:00", "Z")
 
 
 def validate_synth_skill(skill_path: Path) -> dict[str, Any]:
+    """Validate one generated csep-synth SKILL.md without rewriting its body."""
     if skill_path.name != "SKILL.md":
         return {"path": str(skill_path), "valid": False, "reason": "not_skill_md"}
     if not skill_path.parent.name.startswith(SYNTH_SKILL_PREFIX):
@@ -31,17 +47,32 @@ def validate_synth_skill(skill_path: Path) -> dict[str, Any]:
     meta = _frontmatter(text)
     if not meta.get("name") or not meta.get("description"):
         return {"path": str(skill_path), "valid": False, "reason": "missing_frontmatter"}
+    if meta.get("name") != skill_path.parent.name:
+        return {"path": str(skill_path), "valid": False, "reason": "name_mismatch"}
+    description_raw = meta.get("description", "").strip()
+    if description_raw in _BLOCK_SCALAR_MARKERS:
+        return {"path": str(skill_path), "valid": False, "reason": "unsupported_description_format"}
+    status = meta.get("csep_status", "")
+    if status not in {"", "active", "retired"}:
+        return {"path": str(skill_path), "valid": False, "reason": "invalid_status"}
     if contains_secret_like_text(text):
         return {"path": str(skill_path), "valid": False, "reason": "secret_like_content"}
-    if meta.get("csep_status") == "retired":
-        if meta.get("description") != RETIRED_DESCRIPTION:
+    if status == "retired":
+        if description_raw != RETIRED_DESCRIPTION:
             return {"path": str(skill_path), "valid": False, "reason": "invalid_retired_description"}
         if RETIRED_BODY_LINE not in text:
             return {"path": str(skill_path), "valid": False, "reason": "invalid_retired_body"}
         return {"path": str(skill_path), "valid": True, "status": "retired", "reason": None}
-    description = meta.get("description", "").lower()
-    if "use" not in description or "when" not in description:
+    if RETIRED_BODY_LINE in text:
+        return {"path": str(skill_path), "valid": False, "reason": "retired_body_without_status"}
+    description = description_raw.lower()
+    if not description.startswith(_ACTIVE_DESCRIPTION_PREFIXES):
         return {"path": str(skill_path), "valid": False, "reason": "weak_description"}
+    body = text.lower()
+    if not all(marker in body for marker in _SKILL_DECISION_MARKERS):
+        return {"path": str(skill_path), "valid": False, "reason": "missing_skill_decision"}
+    if not all(heading in body for heading in _WORKFLOW_HEADINGS):
+        return {"path": str(skill_path), "valid": False, "reason": "missing_workflow_contract"}
     words = [word for word in re.split(r"\s+", text) if word]
     if len(words) < 24:
         return {"path": str(skill_path), "valid": False, "reason": "low_signal"}

tests/test_install_script.py

@@ -100,7 +100,7 @@ def test_install_script_uses_uv_tool_and_cleans_only_managed_hooks(tmp_path):
         / "cache"
         / "codex-self-evolution"
         / "codex-self-evolution"
-        / "0.7.7"
+        / "0.7.8"
         / ".codex-plugin"
         / "plugin.json"
     )

tests/test_skill_synthesis_agent.py

@@ -10,6 +10,21 @@
 )
 
 
+def _create_action(**overrides):
+    action = {
+        "action": "create",
+        "skill_id": "csep-synth-alpha",
+        "path": "/tmp/skills/csep-synth-alpha/SKILL.md",
+        "evidence_keys": ["ev1"],
+        "reason": "repeated workflow",
+        "evidence_kind": "workflow",
+        "why_skill_not_memory": "This is a reusable command workflow, not a static fact or preference.",
+        "existing_skill_overlap": "No existing skill covers this exact trigger and command sequence.",
+    }
+    action.update(overrides)
+    return action
+
+
 def test_prompt_names_workspace_and_allowed_skills_root(tmp_path: Path) -> None:
     prompt = build_skill_synthesis_prompt(
         run_input_dir=tmp_path / "run" / "input",
@@ -21,6 +36,14 @@ def test_prompt_names_workspace_and_allowed_skills_root(tmp_path: Path) -> None:
     assert str(tmp_path / "run" / "output" / "result.json") in prompt
     assert str(tmp_path / "skills" / "csep-synth-*") in prompt
     assert "Do not read files outside the input directory" in prompt
+    assert '"schema_version": 1' in prompt
+    assert '"actions": [' in prompt
+    assert "Do not write legacy result shapes" in prompt
+    assert "description: Use when" in prompt
+    assert "Classify every candidate" in prompt
+    assert "fact|rule|preference|workflow|duplicate|transient" in prompt
+    assert "why_skill_not_memory" in prompt
+    assert "existing_skill_overlap" in prompt
 
 
 def test_build_pi_command_uses_independent_provider_model() -> None:
@@ -48,11 +71,7 @@ def test_parse_agent_result_accepts_actions(tmp_path: Path) -> None:
         "run_id": "run-1",
         "actions": [
             {
-                "action": "create",
-                "skill_id": "csep-synth-alpha",
-                "path": "/tmp/skills/csep-synth-alpha/SKILL.md",
-                "evidence_keys": ["ev1"],
-                "reason": "repeated workflow",
+                **_create_action(),
             }
         ],
     }), encoding="utf-8")
@@ -61,6 +80,7 @@ def test_parse_agent_result_accepts_actions(tmp_path: Path) -> None:
 
     assert parsed["valid"] is True
     assert parsed["actions"][0]["skill_id"] == "csep-synth-alpha"
+    assert parsed["actions"][0]["evidence_kind"] == "workflow"
 
 
 def test_parse_agent_result_marks_missing_or_invalid_partial(tmp_path: Path) -> None:
@@ -75,6 +95,40 @@ def test_parse_agent_result_marks_missing_or_invalid_partial(tmp_path: Path) ->
     assert invalid["result_invalid"] is True
 
 
+def test_parse_agent_result_rejects_publishable_action_without_skill_decision(tmp_path: Path) -> None:
+    result_path = tmp_path / "result.json"
+    result_path.write_text(json.dumps({
+        "schema_version": 1,
+        "run_id": "run-1",
+        "actions": [
+            _create_action(evidence_kind="", why_skill_not_memory="", existing_skill_overlap=""),
+        ],
+    }), encoding="utf-8")
+
+    parsed = parse_agent_result(result_path)
+
+    assert parsed["valid"] is False
+    assert parsed["result_invalid"] is True
+    assert parsed["error"] == "missing_skill_decision"
+
+
+def test_parse_agent_result_rejects_publishable_non_workflow_action(tmp_path: Path) -> None:
+    result_path = tmp_path / "result.json"
+    result_path.write_text(json.dumps({
+        "schema_version": 1,
+        "run_id": "run-1",
+        "actions": [
+            _create_action(evidence_kind="rule"),
+        ],
+    }), encoding="utf-8")
+
+    parsed = parse_agent_result(result_path)
+
+    assert parsed["valid"] is False
+    assert parsed["result_invalid"] is True
+    assert parsed["error"] == "non_workflow_skill"
+
+
 def test_parse_agent_result_rejects_non_synth_skill_id(tmp_path: Path) -> None:
     result_path = tmp_path / "result.json"
     result_path.write_text(json.dumps({