Skip to content

Commit 4406016

Browse files
committed
[guard.research.ρ.D8+] Make noisy goals survive the full AI action loop
The previous 56-case smoke matrix could not distinguish JSON compliance from correct governed action selection under long, contradictory human input. A 1,024-case RTX 5080 run first measured 23 failures: stable authority-resolution errors, stochastic proposal validation failures, and excess-concurrency tails. Raise only the human goal boundary to 16,384 characters, add lossless authority attention windows, use deterministic sampling for the local structured proposal, and retain a bounded second schema regeneration. The benchmark now generates balanced run/stop cases across eight scenario families, persists exact replay coordinates, and checks classifier, final template, subject, preference, job, model, claim boundary, and Artifact digest independently. Final evidence: seed 2026081902 completed 1,024/1,024 on Qwen3.5-2B in 669.747s; all eight families were 128/128 and all eight operational checks were 1.0. The full repository's 19 gate classes and final browser approve/reject/cancel/control acceptance passed. No physical FEP execution was started.
1 parent d05961e commit 4406016

14 files changed

Lines changed: 1110 additions & 538 deletions

‎backend/README.md‎

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -125,12 +125,14 @@ bash scripts/gates.sh research
125125
# Additional durable PostgreSQL and fake-provider acceptance:
126126
DIRAC_TEST_DSN='dbname=<isolated-test-db>' bash scripts/gates.sh research
127127

128-
# Fresh seed by default; every case code-switches 4–8 languages and adds
129-
# randomized typos, dropped/repeated characters, Unicode and quotation noise.
128+
# Fresh seed and 1,024 full Method→Job→Artifact cases by default. Every case
129+
# code-switches 8–10 languages across 76–115 fragments with 36–66 randomized
130+
# typo/OCR/keyboard/Unicode/bidi/quotation/injection transformations.
130131
backend/env/bin/python -B scripts/research_loop_multilingual_benchmark.py
131132

132-
# Replay an exact failure from its persisted seed.
133-
backend/env/bin/python -B scripts/research_loop_multilingual_benchmark.py --seed <seed>
133+
# Replay exact generated cases from a persisted seed without rerunning the matrix.
134+
backend/env/bin/python -B scripts/research_loop_multilingual_benchmark.py \
135+
--cases 1024 --seed <seed> --case-indexes 20,37,69
134136
```
135137

136138
The acceptance script replaces only the expensive OpenFE RunSet with an explicitly fake,

‎backend/research/reasoner.py‎

Lines changed: 26 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -42,6 +42,13 @@
4242
MANIFEST_PATH = PROMPT_DIRECTORY / f"{PROMPT_RELEASE_ID}.manifest.json"
4343
CONTEXT_SCHEMA_PATH = ROOT / "contracts/domain/research/context-snapshot.schema.json"
4444
PROPOSAL_SCHEMA_PATH = ROOT / "contracts/domain/research/proposal.schema.json"
45+
GOAL_AUTHORITY_MARKERS = (
46+
"revision v2 [current", "what i actually mean now is:",
47+
"current human request", "final operative objective",
48+
"latest human decision", "authoritative request",
49+
"最终有效目标", "最新人工决定", "权威请求",
50+
"objectif final", "dernière décision humaine", "verbindliche anfrage",
51+
)
4552

4653

4754
def _raw_digest(path: pathlib.Path) -> str:
@@ -102,8 +109,26 @@ def build_goal_interpretation_schema(
102109
def build_goal_interpretation_messages(
103110
context: Mapping[str, Any], *, system_prompt: str,
104111
) -> tuple[str, str]:
112+
goal_intent = str(context["goal"]["intent"])
113+
folded = goal_intent.casefold()
114+
authority_hints: list[str] = []
115+
for marker in GOAL_AUTHORITY_MARKERS:
116+
cursor = 0
117+
while len(authority_hints) < 8:
118+
index = folded.find(marker.casefold(), cursor)
119+
if index < 0:
120+
break
121+
end_marker = folded.find("end final objective", index + len(marker))
122+
end = min(len(goal_intent), index + 768)
123+
if 0 <= end_marker < end:
124+
end = min(len(goal_intent), end_marker + len("end final objective"))
125+
snippet = goal_intent[index:end].strip()
126+
if snippet and snippet not in authority_hints:
127+
authority_hints.append(snippet)
128+
cursor = index + len(marker)
105129
request = {
106-
"goal_intent": context["goal"]["intent"],
130+
"goal_intent": goal_intent,
131+
"operative_attention_windows": authority_hints,
107132
"available_actions": [
108133
{
109134
"template_id": item["template_id"],

‎backend/tests/test_research_context_builder.py‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -72,6 +72,17 @@ def domain() -> dict:
7272

7373

7474
class ResearchContextBuilderTests(unittest.TestCase):
75+
def test_goal_intent_accepts_16384_characters_and_rejects_16385(self):
76+
at_limit = loop()
77+
at_limit["intent"] = "x" * 16_384
78+
built = ContextBuilder().build(at_limit, domain())
79+
self.assertEqual(len(built.document["goal"]["intent"]), 16_384)
80+
81+
over_limit = loop()
82+
over_limit["intent"] = "x" * 16_385
83+
with self.assertRaises(failures.DiracInternal):
84+
ContextBuilder().build(over_limit, domain())
85+
7586
def test_same_frozen_state_is_byte_identical_and_digest_is_self_consistent(self):
7687
builder = ContextBuilder()
7788
frozen_loop = loop()

‎backend/tests/test_research_multilingual_benchmark.py‎

Lines changed: 43 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
import json
22
import random
3+
from pathlib import Path
34

45
from scripts.research_loop_multilingual_benchmark import (
56
_apply_noise,
@@ -9,32 +10,63 @@
910

1011

1112
def test_generated_matrix_is_replayable_but_changes_with_the_seed():
12-
first = generate_cases(1729, 24)
13-
replay = generate_cases(1729, 24)
14-
fresh = generate_cases(1730, 24)
13+
first = generate_cases(1729, 32)
14+
replay = generate_cases(1729, 32)
15+
fresh = generate_cases(1730, 32)
1516

1617
assert first == replay
1718
assert [case.intent for case in first] != [case.intent for case in fresh]
18-
assert [case.target for case in first].count("run") == 12
19-
assert [case.target for case in first].count("stop") == 12
20-
assert all(4 <= len(case.languages) <= 8 for case in first)
21-
assert all(len(case.noise_operators) >= 2 for case in first)
22-
assert sum("quoted_injection_noise" in case.noise_operators for case in first) == 8
23-
assert sum("discarded_opposite_quote" in case.noise_operators for case in first) == 6
24-
assert sum("duplicate_fragment" in case.noise_operators for case in first) == 4
19+
assert [case.target for case in first].count("run") == 16
20+
assert [case.target for case in first].count("stop") == 16
21+
assert {case.scenario_family for case in first} == {
22+
"normal_noisy_request", "final_block_last", "final_block_middle",
23+
"spoken_self_correction", "versioned_ticket_thread",
24+
"nested_quotation_attack", "bidi_ocr_transcript",
25+
"implicit_scientific_request",
26+
}
27+
assert all(8 <= len(case.languages) <= 10 for case in first)
28+
assert all(72 <= case.fragment_count <= 115 for case in first)
29+
assert all(len(case.noise_operators) >= 36 for case in first)
30+
assert all(12 <= case.decoy_count <= 20 for case in first)
31+
assert all(2 <= case.nesting_depth <= 6 for case in first)
32+
assert all(len(case.intent) >= 4_000 for case in first)
33+
assert all(len(case.intent) <= 16_384 for case in first)
34+
assert all("duplicate_fragment" in case.noise_operators for case in first)
35+
36+
37+
def test_goal_contracts_accept_ten_x_inputs_without_widening_other_text():
38+
root = Path(__file__).resolve().parents[2]
39+
registry = json.loads(
40+
(root / "contracts/commands/registry.json").read_text(encoding="utf-8"))
41+
commands = {item["id"]: item for item in registry["commands"]}
42+
assert commands["research.loop.create"]["input_schema"]["properties"][
43+
"intent"]["maxLength"] == 16_384
44+
assert commands["research.loop.control"]["input_schema"]["properties"][
45+
"revised_intent"]["maxLength"] == 16_384
46+
47+
context_schema = json.loads((
48+
root / "contracts/domain/research/context-snapshot.schema.json"
49+
).read_text(encoding="utf-8"))
50+
assert context_schema["properties"]["goal"]["properties"]["intent"] == {
51+
"$ref": "#/$defs/text16384"
52+
}
53+
assert context_schema["$defs"]["text4096"]["maxLength"] == 4_096
2554

2655

2756
def test_every_noise_operator_really_changes_text():
2857
source = "Continue the planned calcul with scientific références 计算停止."
2958
for index, operator in enumerate([
3059
"known_typo", "repeat_character", "drop_character", "zero_width",
3160
"space_collapse", "case_noise", "unicode_nfd", "punctuation_burst",
61+
"keyboard_neighbor", "word_transposition", "token_duplication",
62+
"ocr_confusion", "homoglyph", "bidi_marks", "stray_markup",
63+
"dictation_filler", "newline_fragmentation",
3264
]):
3365
assert _apply_noise(source, operator, random.Random(index)) != source
3466

3567

3668
def test_mixed_unicode_goal_survives_the_frozen_context_byte_for_byte():
37-
case = generate_cases(8675309, 2)[0]
69+
case = generate_cases(8675309, 16)[0]
3870
document, encoded = build_context(case)
3971

4072
assert document["goal"]["intent"] == case.intent

‎backend/tests/test_research_reasoner_method.py‎

Lines changed: 19 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -213,6 +213,7 @@ def test_multilingual_goal_interpreter_is_small_bounded_and_constrains_proposal(
213213
request = json.loads(user.removeprefix(
214214
"JSON goal interpretation input:\n"))
215215
self.assertEqual(request["goal_intent"], context["goal"]["intent"])
216+
self.assertEqual(request["operative_attention_windows"], [])
216217
interpreted, result = interpret_goal(
217218
OpenAICompatibleChatProvider(), self.profile, context)
218219
self.assertEqual(interpreted["selected_template_id"],
@@ -228,6 +229,23 @@ def test_multilingual_goal_interpreter_is_small_bounded_and_constrains_proposal(
228229
{"const": "fep.run_selected_edge.v1"})
229230
self.assertEqual(self.server.last_payload["temperature"], 0)
230231

232+
def test_goal_interpreter_attention_window_copies_current_revision_verbatim(self):
233+
context = copy.deepcopy(self.context)
234+
context["goal"]["intent"] = (
235+
"REVISION v1 [OBSOLETE]: stop. "
236+
"REVISION v2 [CURRENT, supersedes v1] — 继续研究 C2→C7. "
237+
"DISCARDED: stop stop stop."
238+
)
239+
_system, user = build_goal_interpretation_messages(
240+
context, system_prompt="Return JSON only.")
241+
request = json.loads(user.removeprefix(
242+
"JSON goal interpretation input:\n"))
243+
self.assertEqual(len(request["operative_attention_windows"]), 1)
244+
self.assertTrue(request["operative_attention_windows"][0].startswith(
245+
"REVISION v2 [CURRENT"))
246+
self.assertNotIn(
247+
"REVISION v1", request["operative_attention_windows"][0])
248+
231249
def test_invalid_first_proposal_regenerates_once_with_bounded_error(self):
232250
self.server.mode = "invalid-then-valid"
233251
_, job = self.run_job(self.payload("reasoner-regenerate"))
@@ -240,7 +258,7 @@ def test_repeated_invalid_proposal_fails_with_typed_terminal_error(self):
240258
_, job = self.run_job(self.payload("reasoner-invalid"))
241259
self.assertEqual(job["state"], "failed", job)
242260
self.assertEqual(job["error_code"], "MODEL_OUTPUT_INVALID")
243-
self.assertEqual(self.server.attempts, 2)
261+
self.assertEqual(self.server.attempts, 3)
244262

245263
def test_same_request_key_replays_job_without_second_provider_call(self):
246264
payload = self.payload("reasoner-idempotent")

‎contracts/commands/registry.json‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -41,7 +41,7 @@
4141
"input_object_kinds": ["program","campaign"], "output_object_kinds": ["mission","run"],
4242
"errors": ["INVALID_PARAMETERS","NOT_FOUND","IDEMPOTENCY_CONFLICT","PROVIDER_UNAVAILABLE","UNSUPPORTED","DB_UNAVAILABLE","INTERNAL"],
4343
"handler": "handlers:research_loop_create",
44-
"input_schema": {"type":"object","additionalProperties":false,"required":["request_key","program_ref","campaign_ref","intent","autonomy_class","provider_profile_id","data_classification","budget","policy"],"properties":{"request_key":{"type":"string","minLength":1,"maxLength":256},"program_ref":{"type":"object","additionalProperties":false,"required":["kind","id"],"properties":{"kind":{"const":"program"},"id":{"type":"string","format":"uuid"}}},"campaign_ref":{"type":"object","additionalProperties":false,"required":["kind","id"],"properties":{"kind":{"const":"campaign"},"id":{"type":"string","format":"uuid"}}},"intent":{"type":"string","minLength":1,"maxLength":4096},"autonomy_class":{"enum":["A0","A1","A2","A3"]},"provider_profile_id":{"type":"string","minLength":1,"maxLength":128},"data_classification":{"enum":["public","internal","partner_confidential","restricted","regulated"]},"budget":{"type":"object","additionalProperties":false,"required":["max_reasoner_calls","max_iterations","max_fep_runsets","max_gpu_hours","max_external_cost"],"properties":{"max_reasoner_calls":{"type":"integer","minimum":1,"maximum":128},"max_iterations":{"type":"integer","minimum":1,"maximum":128},"max_fep_runsets":{"type":"integer","minimum":0,"maximum":32},"max_gpu_hours":{"type":"number","minimum":0},"max_external_cost":{"type":"number","minimum":0}}},"policy":{"type":"object","additionalProperties":false,"required":["auto_risk_classes","per_action_risk_classes","human_only_risk_classes","stop_on_campaign_stale","stop_on_open_identity_conflict","max_same_subject_actions","cloud_egress_approved"],"properties":{"auto_risk_classes":{"type":"array","items":{"enum":["R0","R1","R2","R3","R4"]},"uniqueItems":true},"per_action_risk_classes":{"type":"array","items":{"enum":["R0","R1","R2","R3","R4"]},"uniqueItems":true},"human_only_risk_classes":{"type":"array","items":{"enum":["R0","R1","R2","R3","R4"]},"uniqueItems":true},"stop_on_campaign_stale":{"type":"boolean"},"stop_on_open_identity_conflict":{"type":"boolean"},"max_same_subject_actions":{"type":"integer","minimum":1,"maximum":32},"cloud_egress_approved":{"type":"boolean"}}}}},
44+
"input_schema": {"type":"object","additionalProperties":false,"required":["request_key","program_ref","campaign_ref","intent","autonomy_class","provider_profile_id","data_classification","budget","policy"],"properties":{"request_key":{"type":"string","minLength":1,"maxLength":256},"program_ref":{"type":"object","additionalProperties":false,"required":["kind","id"],"properties":{"kind":{"const":"program"},"id":{"type":"string","format":"uuid"}}},"campaign_ref":{"type":"object","additionalProperties":false,"required":["kind","id"],"properties":{"kind":{"const":"campaign"},"id":{"type":"string","format":"uuid"}}},"intent":{"type":"string","minLength":1,"maxLength":16384},"autonomy_class":{"enum":["A0","A1","A2","A3"]},"provider_profile_id":{"type":"string","minLength":1,"maxLength":128},"data_classification":{"enum":["public","internal","partner_confidential","restricted","regulated"]},"budget":{"type":"object","additionalProperties":false,"required":["max_reasoner_calls","max_iterations","max_fep_runsets","max_gpu_hours","max_external_cost"],"properties":{"max_reasoner_calls":{"type":"integer","minimum":1,"maximum":128},"max_iterations":{"type":"integer","minimum":1,"maximum":128},"max_fep_runsets":{"type":"integer","minimum":0,"maximum":32},"max_gpu_hours":{"type":"number","minimum":0},"max_external_cost":{"type":"number","minimum":0}}},"policy":{"type":"object","additionalProperties":false,"required":["auto_risk_classes","per_action_risk_classes","human_only_risk_classes","stop_on_campaign_stale","stop_on_open_identity_conflict","max_same_subject_actions","cloud_egress_approved"],"properties":{"auto_risk_classes":{"type":"array","items":{"enum":["R0","R1","R2","R3","R4"]},"uniqueItems":true},"per_action_risk_classes":{"type":"array","items":{"enum":["R0","R1","R2","R3","R4"]},"uniqueItems":true},"human_only_risk_classes":{"type":"array","items":{"enum":["R0","R1","R2","R3","R4"]},"uniqueItems":true},"stop_on_campaign_stale":{"type":"boolean"},"stop_on_open_identity_conflict":{"type":"boolean"},"max_same_subject_actions":{"type":"integer","minimum":1,"maximum":32},"cloud_egress_approved":{"type":"boolean"}}}}},
4545
"output_schema": {"type":"object","additionalProperties":false,"required":["mission_ref","run_ref","state","stage","version","created"],"properties":{"mission_ref":{"type":"object","additionalProperties":false,"required":["kind","id"],"properties":{"kind":{"const":"mission"},"id":{"type":"string","format":"uuid"}}},"run_ref":{"type":"object","additionalProperties":false,"required":["kind","id"],"properties":{"kind":{"const":"run"},"id":{"type":"string","format":"uuid"}}},"state":{"type":"string"},"stage":{"type":"string"},"version":{"type":"integer","minimum":1},"created":{"type":"boolean"}}}
4646
},
4747
{
@@ -77,7 +77,7 @@
7777
"idempotency_policy": "idempotent", "job_policy": "forbidden", "provenance_policy": "required",
7878
"input_object_kinds": ["run"], "output_object_kinds": ["run"],
7979
"errors": ["INVALID_PARAMETERS","NOT_FOUND","PROVIDER_UNAVAILABLE","UNSUPPORTED","DB_UNAVAILABLE","INTERNAL"], "handler": "handlers:research_loop_control",
80-
"input_schema": {"type":"object","additionalProperties":false,"required":["run_ref","expected_version","action","rationale"],"properties":{"run_ref":{"type":"object","additionalProperties":false,"required":["kind","id"],"properties":{"kind":{"const":"run"},"id":{"type":"string","format":"uuid"}}},"expected_version":{"type":"integer","minimum":1},"action":{"enum":["pause","resume","retry","cancel","revise_intent","change_provider"]},"rationale":{"type":"string","minLength":1,"maxLength":4096},"revised_intent":{"type":"string","minLength":1,"maxLength":4096},"provider_profile_id":{"type":"string","minLength":1,"maxLength":128}}},
80+
"input_schema": {"type":"object","additionalProperties":false,"required":["run_ref","expected_version","action","rationale"],"properties":{"run_ref":{"type":"object","additionalProperties":false,"required":["kind","id"],"properties":{"kind":{"const":"run"},"id":{"type":"string","format":"uuid"}}},"expected_version":{"type":"integer","minimum":1},"action":{"enum":["pause","resume","retry","cancel","revise_intent","change_provider"]},"rationale":{"type":"string","minLength":1,"maxLength":4096},"revised_intent":{"type":"string","minLength":1,"maxLength":16384},"provider_profile_id":{"type":"string","minLength":1,"maxLength":128}}},
8181
"output_schema": {"type":"object","additionalProperties":false,"required":["run_ref","state","stage","version"],"properties":{"run_ref":{"type":"object"},"state":{"type":"string"},"stage":{"type":"string"},"version":{"type":"integer"}}}
8282
},
8383
{

‎contracts/domain/research/ai-provider-profile.schema.json‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -70,7 +70,7 @@
7070
"max_request_bytes": {"type": "integer", "minimum": 1024, "maximum": 262144},
7171
"max_response_bytes": {"type": "integer", "minimum": 1024, "maximum": 262144},
7272
"max_provider_attempts": {"type": "integer", "minimum": 1, "maximum": 5},
73-
"max_schema_regenerations": {"type": "integer", "minimum": 0, "maximum": 1}
73+
"max_schema_regenerations": {"type": "integer", "minimum": 0, "maximum": 2}
7474
}
7575
},
7676
"data_policy": {

‎contracts/domain/research/context-snapshot.schema.json‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,7 @@
2121
"type": "object", "additionalProperties": false,
2222
"required": ["intent", "constraints", "success_definition", "revised_at"],
2323
"properties": {
24-
"intent": {"$ref": "#/$defs/text4096"},
24+
"intent": {"$ref": "#/$defs/text16384"},
2525
"constraints": {"$ref": "#/$defs/textList64"},
2626
"success_definition": {"$ref": "#/$defs/textList64"},
2727
"revised_at": {"type": ["string", "null"], "format": "date-time"}
@@ -138,6 +138,7 @@
138138
},
139139
"$defs": {
140140
"sha256": {"type": "string", "pattern": "^sha256:[0-9a-f]{64}$"},
141+
"text16384": {"type": "string", "minLength": 1, "maxLength": 16384},
141142
"text4096": {"type": "string", "minLength": 1, "maxLength": 4096},
142143
"textList64": {
143144
"type": "array", "maxItems": 64,

‎contracts/research/prompts/fep-action-proposal-v1.manifest.json‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2,8 +2,8 @@
22
"schema_version": "1.0",
33
"prompt_release_id": "fep-action-proposal-v1",
44
"system_prompt_sha256": "sha256:aa60a8b4de6799f31f597b0333f3ecf7246c5430998362e911c08392f0a997a8",
5-
"goal_interpreter_prompt_sha256": "sha256:aee08f32f9e8a1554195d58ea7cd66531345eb1a3dd86815016dec266c492266",
6-
"context_schema_sha256": "sha256:dc017a3b31ce51dd521d797bf66a9aa9d5f71b924f56cf7f6ff64fbf46e9225b",
5+
"goal_interpreter_prompt_sha256": "sha256:3cf61b416dc3c97442507ceb9dfb4925560e784dd13811699f36660df3e2c0e3",
6+
"context_schema_sha256": "sha256:1edaf78a7e52330945bd1cbd2a3c67d910841508b43998a5125d55ef7e898f17",
77
"proposal_schema_sha256": "sha256:3f9fd14c992326b82ee2091f3daba4da1f654964bdf5d6918d3337ecd1523d18",
88
"action_catalog_sha256": "sha256:9be8409e7b3b39a7b1f959dc8624ba25009b54715940c1555539f5ad7d8007d5"
99
}

0 commit comments

Comments
 (0)