From 976bdcfec4276b0d38bdb9700167b54301bfc059 Mon Sep 17 00:00:00 2001 From: FatosLocos Date: Sat, 18 Jul 2026 15:10:58 +0200 Subject: [PATCH] Record sealed ECP-7 deep decoder batch --- README.md | 10 +-- config/ecp7-b12-development.yaml | 15 ++++ docs/AI_AGENT_START.md | 32 +++++---- docs/development-log-ecp7.md | 69 +++++++++++++++++++ docs/research-design-ecp7.md | 37 ++++++++++ evidence/ecp7-development/README.md | 5 ++ .../ecp7-development/batch12-manifest.json | 43 ++++++++++++ src/ai_taal/models.py | 22 +++++- tests/conftest.py | 5 ++ tests/test_config.py | 23 +++++++ tests/test_models.py | 35 ++++++++++ 11 files changed, 275 insertions(+), 21 deletions(-) create mode 100644 config/ecp7-b12-development.yaml create mode 100644 evidence/ecp7-development/batch12-manifest.json diff --git a/README.md b/README.md index 1e88ecf..4c82960 100644 --- a/README.md +++ b/README.md @@ -26,15 +26,16 @@ The protocol uses no words or alphabet, but four meaning-free local symbols. For See [`docs/results-ecp6.md`](docs/results-ecp6.md) for the conclusion, [`docs/protocol-specification-ecp6.md`](docs/protocol-specification-ecp6.md) for the wire format and [`evidence/ecp6/report.md`](evidence/ecp6/report.md) for the compact confirmatory evidence. ECP-7 development now tests how much of that result depends on the explicit -one-factor-per-slot architecture. Its first eleven sealed batches are valid +one-factor-per-slot architecture. Its first twelve sealed batches are valid development results but none passes the full gate. Batch 10 remains strongest, extending the position-aware design to 15,000 optimization steps. It reaches 82.08% train exactness, 72.98% validation and 75.15% translator validation while using 12,358–12,906 hard messages per sender. The translator threshold now passes, but the registered train, validation and injectivity thresholds do not. Batch -11's utilization-weight decay regressed all principal measures. The ECP-6 -positive controls remain perfect and the ECP-7 confirmatory test remains sealed. -See [`docs/development-log-ecp7.md`](docs/development-log-ecp7.md). +11's utilization-weight decay regressed all principal measures, and Batch 12's +deeper decoder regressed further. The ECP-6 positive controls remain perfect and +the ECP-7 confirmatory test remains sealed. See +[`docs/development-log-ecp7.md`](docs/development-log-ecp7.md). ## Build on this work @@ -85,6 +86,7 @@ Do not translate those artifacts in place. New experiment configurations and all - [`config/ecp7-b9-development.yaml`](config/ecp7-b9-development.yaml) — generic position-aware MLP decoding on the parallel sender. - [`config/ecp7-b10-development.yaml`](config/ecp7-b10-development.yaml) — the Batch 9 design with an isolated extended optimization horizon. - [`config/ecp7-b11-development.yaml`](config/ecp7-b11-development.yaml) — late utilization-weight decay on the Batch 10 base. +- [`config/ecp7-b12-development.yaml`](config/ecp7-b12-development.yaml) — one additional generic shared decoder layer on the Batch 10 base. - [`config/ecp7-positive-control-development.yaml`](config/ecp7-positive-control-development.yaml) — ECP-6 positive control on the ECP-7 split. - [`config/ecp0.yaml`](config/ecp0.yaml) — machine-readable configuration from step 1. - [`config/ecp1.yaml`](config/ecp1.yaml) — frozen configuration of the population trial. diff --git a/config/ecp7-b12-development.yaml b/config/ecp7-b12-development.yaml new file mode 100644 index 0000000..7ec0251 --- /dev/null +++ b/config/ecp7-b12-development.yaml @@ -0,0 +1,15 @@ +extends: ecp7-b10-development.yaml + +experiment: + title: ECP-7 extended optimization with a deep position-aware decoder + arm: extended_optimization_deep_position_aware_decoder + objective: >- + Test whether one additional generic shared hidden layer lets receivers and + the isolated translator resolve the residual color and size distinctions + without introducing factor-specific paths or bindings. + +agents: + receiver: + family: deep_position_aware_mlp_receiver + translator: + family: deep_position_aware_mlp_receiver diff --git a/docs/AI_AGENT_START.md b/docs/AI_AGENT_START.md index d0fa4de..83fabbc 100644 --- a/docs/AI_AGENT_START.md +++ b/docs/AI_AGENT_START.md @@ -41,7 +41,7 @@ python3.12 -m venv .venv .venv/bin/ecp6 --config config/ecp6.yaml validate ``` -Expected baseline: 79 passing tests and split sizes `14336/1024/1024`. +Expected baseline: 82 passing tests and split sizes `14336/1024/1024`. ## 4. Reproduce ECP-6 @@ -71,7 +71,7 @@ This reruns a known experiment. It must not be presented as a new independent co ## 5. Continue ECP-7 scientifically -Do not modify ECP-0 through ECP-6 or the completed ECP-7 Batch 1 through Batch 11 +Do not modify ECP-0 through ECP-6 or the completed ECP-7 Batch 1 through Batch 12 configs. ECP7-B1-I collapsed to 130–139 hard messages. ECP7-B2-I improved its soft objective but collapsed further to 85–104 hard messages. ECP7-B3-I applied that loss to straight-through hard messages and improved to 585–972 messages, @@ -95,12 +95,14 @@ schedule and extended population optimization to 15,000 steps. It reached 12,358–12,906 messages. The translator gate passed, but train, validation and injectivity did not. ECP7-B11-I decayed late utilization pressure and regressed to 79.09% train, 72.22% validation and 72.41% translator validation. B10 remains -the strongest base. The confirmatory ECP-7 test is still sealed. +the strongest base. ECP7-B12-I added one shared decoder layer and regressed +further to 71.53% train, 47.98% validation and 63.04% translator validation. +The confirmatory ECP-7 test is still sealed. Continue with this sequence: 1. Read `docs/research-design-ecp7.md` and `docs/development-log-ecp7.md`. -2. Define exactly one ECP7-B12 intervention and its failure criterion. +2. Define exactly one ECP7-B13 intervention and its failure criterion. 3. Register the variant and immutable configuration hashes before training. 4. Add tests for every new invariant. 5. Use only `smoke` and `develop`; keep the ECP-7 test split sealed. @@ -113,23 +115,23 @@ Continue with this sequence: ## 6. Recommended next experiment -Batch 10 is the strongest weak-structure result. Batch 11 showed that reducing -late utilization pressure harms task accuracy, code use and sender agreement. -B10's one-hidden-layer population decoder and fresh translator both retain a -size weakness, while color is the other residual population error. The most -informative next step is one generic decoder-capacity change. A clean ECP7-B12 -progression is: +Batch 10 is the strongest weak-structure result. Batch 11 ruled out reduced late +utilization pressure, and Batch 12 ruled out a deeper shared decoder. B10 still +generates all four message positions from one shallow shared sender context. +The most informative next step is the symmetric sender-capacity test. A clean +ECP7-B13 progression is: - keep the same world and bit budget; - return to the complete B10 training and loss schedule; -- preserve the flattened position-aware input and shared representation; -- add exactly one second shared hidden layer to both receiver and translator; +- keep the B10 one-hidden-layer receiver and translator; +- add exactly one second shared hidden layer to the bounded parallel sender + before all four slot heads; - measure injectivity, validation composition and new-reader induction; - rerun the ECP-6 architecture as the frozen positive control. -Do not add a factor-specific input, binding matrix, slot selection or loss -change. Do not change the sender, hidden width, data, temperature, duration, -validation cadence or development thresholds, and do not combine decoder depth +Do not add a factor-specific sender path, binding matrix, slot selection or loss +change. Do not change the decoder, hidden width, data, temperature, duration, +validation cadence or development thresholds, and do not combine sender depth with variable-length messages or negotiation. That would make its effect uninterpretable. diff --git a/docs/development-log-ecp7.md b/docs/development-log-ecp7.md index 77557e8..ae452ae 100644 --- a/docs/development-log-ecp7.md +++ b/docs/development-log-ecp7.md @@ -787,3 +787,72 @@ complete B10 schedule and add exactly one second generic shared hidden layer to both the position-aware receiver and isolated translator. It must preserve the same flattened positional input and must not add factor-specific paths, binding, loss, duration, data or threshold changes. + +## Batch 12 preregistration + +Date: July 18, 2026
+Seed: `11`
+Maximum population steps: `15,000`
+Temperature schedule steps: `5,000`, then hold at `0.8`
+Decoder change: one additional shared `hidden_dim × hidden_dim` layer
+Split-SHA-256: `4947058c75ab07cb43a87eb82776b12cb2a7e2eeba7114de110d3b852cbc64cd`
+Test unsealed: **no** + +ECP7-B12-I returns to every Batch 10 training and loss setting, then changes +only the receiver and translator depth. The four factor heads still consume one +shared representation of all four embedded positions. There is no binding or +factor-specific input path. The unchanged positive control is rerun, and no +alternative decoder variant is admitted into this batch. + +## Batch 12 results + +Positive-control run: `runs/ecp7-batch12-control-development/20260718T130109Z-ecp7-development`
+Intervention run: `runs/ecp7-batch12-intervention-development/20260718T130109Z-ecp7-development`
+Test unsealed: **no** + +| Metric | Positive control | ECP7-B12-I | +|---|---:|---:| +| Population train, mean | 100% | 71.5297% | +| Population train, worst link | 100% | 66.6016% | +| Population validation, mean | 100% | 47.9797% | +| Population validation, worst link | 100% | 39.7461% | +| Universal translator, validation | 100% | 63.0371% | +| Exact sender-message agreement | 100% | 88.64% | +| Unique messages per sender | 15,360 | 10,119–11,404 | +| Collision meanings per sender | 0 | 3,956–5,241 | +| Message entropy | 13.91 bits | 13.11–13.33 bits | +| Development gate | pass | **fail** | + +The positive control passed every gate at 100%. ECP7-B12-I failed train, +validation, translator and injectivity. It does not authorize confirmatory +access. + +Relative to B10, the second decoder layer reduced train exactness from 82.0836% +to 71.5297%, validation from 72.9797% to 47.9797%, and translator validation from +75.1465% to 63.0371%. Code use and sender agreement also fell. The added generic +decoder depth therefore made the emergent protocol harder, not easier, to learn. + +Population validation factor accuracies were +`[77.07%, 86.05%, 100%, 77.05%]`; universal-translator factor accuracies were +`[97.17%, 87.50%, 100%, 75.90%]`. The deeper shared representation did not +resolve the residual factors consistently across receivers and a new reader. + +The selected checkpoint was step 10,000 with task loss `0.1539` and utilization +loss `-0.7153`. Validation subsequently regressed, so the registered patience +rule stopped population training at step 13,000. + +Both sealed analyses report 65 matching artifact hashes, 16,384 validation-only +episodes, no confirmatory-test keys, valid local alphabets, and exactly 14 +declared bits for every logged message. + +## Batch 12 decision + +ECP7-B12-I is rejected. Decoder depth is not the B10 bottleneck. Batch 10 remains +the strongest weak-structure base, and the confirmatory split remains sealed. + +The remaining capacity hypothesis is sender-side: B10 generates all four slots +from one shallow shared context, and receiver depth has now been ruled out. A +future ECP7-B13 may return to the complete B10 model and add exactly one second +generic shared hidden layer to the bounded parallel sender before all four slot +heads. It must retain the B10 one-layer receiver and translator and must not add +factor-specific sender paths, loss, width, duration, data or threshold changes. diff --git a/docs/research-design-ecp7.md b/docs/research-design-ecp7.md index 362c93f..4756d9b 100644 --- a/docs/research-design-ecp7.md +++ b/docs/research-design-ecp7.md @@ -519,3 +519,40 @@ All existing validity and development gates remain unchanged. Failure is recorded without confirmatory test access. No alternative final weight, decay shape, decay boundary, duration, architecture, loss, optimizer or seed may be tried inside Batch 11. + +## Batch 12 preregistration — Deep position-aware decoder + +Status: **preregistered for sealed development**
+Preregistered: July 18, 2026
+Configuration: `config/ecp7-b12-development.yaml`
+Raw configuration SHA-256: `4cc37488453b756fb43b4782d0e3e93cf051815a367d1852047fcc34fa29d5ce`
+Effective configuration SHA-256: `798a6380ab144e5db63c8177d3dd70edf76e66b660b3cbb355f5beabe86b27b2` + +Batch 11 showed that reducing late utilization pressure worsened task accuracy, +code use and sender agreement. Batch 10 therefore remains the base. Its shared +one-hidden-layer decoder and independently trained translator both retained a +size weakness, while color was the other residual population error. Batch 12 +tests whether the generic decoder is still too shallow for the entangled code. + +ECP7-B12-I returns to the complete B10 training and loss schedule. It keeps the +shared symbol embedding, flattened four-position input and first generic hidden +projection, then adds exactly one second `hidden_dim × hidden_dim` projection +with `tanh`. All four factor heads read that same second shared representation. +The isolated translator uses the identical deeper family. + +The decoder still has no factor-specific input path, factor-specific hidden +projection, binding matrix, slot selector or hard factor-to-slot assignment. +Hidden width, sender, losses, optimizer, data, 15,000-step duration, original +5,000-step temperature annealing, selection window and thresholds remain +unchanged from B10. Smoke runs are mechanical only and may not authorize +tuning. + +The batch contains exactly two seed-11 runs: + +1. the unchanged ECP-6 positive control; +2. ECP7-B12-I with the deeper shared position-aware receiver and translator. + +All existing validity and development gates remain unchanged. Failure is +recorded without confirmatory test access. No alternative depth, activation, +width, residual connection, sender, loss, duration, optimizer or seed may be +tried inside Batch 12. diff --git a/evidence/ecp7-development/README.md b/evidence/ecp7-development/README.md index c94758a..2027746 100644 --- a/evidence/ecp7-development/README.md +++ b/evidence/ecp7-development/README.md @@ -63,6 +63,11 @@ Batch 11 decayed code-utilization weight after step 5,000. It regressed to 11,893–12,323 messages. Batch 10 therefore remains the strongest base. Exact identities and hashes are in `batch11-manifest.json`. +Batch 12 added one generic shared hidden layer to the receiver and translator. +It regressed sharply to 71.53% train exactness, 47.98% validation and 63.04% +translator validation. Decoder depth is therefore rejected as the B10 +bottleneck. Exact identities and hashes are in `batch12-manifest.json`. + Reproduce the development arms with: ```bash diff --git a/evidence/ecp7-development/batch12-manifest.json b/evidence/ecp7-development/batch12-manifest.json new file mode 100644 index 0000000..f3b25c6 --- /dev/null +++ b/evidence/ecp7-development/batch12-manifest.json @@ -0,0 +1,43 @@ +{ + "batch": "ECP7-B12", + "confirmatory_test_unsealed": false, + "control": { + "artifact_hashes": { + "effective_config.yaml": "4bfb976293f117d3ae35f790e4fd6371628be95d91febb334bc275dc5d6b66d4", + "posthoc-analysis.json": "2d92231db6e55352d15ae772fa8bee116a1c725db1aa31216a4cb0b5aa9ec270", + "seed-11/metrics.json": "b32329ddd65833a0f0814144ed7c6b0cdffca1100829c61638c9d5ea3a58cff6", + "summary.json": "34f1828bc95891a08bf234032a08696c495af55c27e26b607a2e0557e25f2e6b" + }, + "development_gate_passed": true, + "population_train_mean_exact_match": 1.0, + "population_validation_mean_exact_match": 1.0, + "run_id": "20260718T130109Z-ecp7-development", + "universal_translator_validation_exact_match": 1.0, + "unique_messages_per_sender": [15360, 15360, 15360, 15360] + }, + "intervention": { + "artifact_hashes": { + "effective_config.yaml": "27017456afb7c224537735095dc1b031bc66fae8e7cf195ba088db9b208f3b82", + "posthoc-analysis.json": "b6714ce0414acc540ba386c0ecfb0ea9d6789a413743713dc83c560a3e310812", + "seed-11/metrics.json": "b03f9b13b0b03e3a1dd3d9bb81015e4f1ccde1e38eccd793bc76de813f217d51", + "summary.json": "1dd591a537f57015bff6792bbcbad70563292f16ff66e174bc20df73d757f518" + }, + "best_code_utilization_loss": -0.7153493165969849, + "best_step": 10000, + "collision_meanings_per_sender": [4171, 5241, 4380, 3956], + "development_gate_passed": false, + "exact_sender_message_agreement": 0.8863823784722223, + "message_entropy_bits_per_sender": [13.30507490700547, 13.107133180956183, 13.260048045675191, 13.333167991315648], + "population_train_mean_exact_match": 0.7152971540178572, + "population_train_worst_link_exact_match": 0.666015625, + "population_validation_factor_accuracy": [0.77069091796875, 0.8604736328125, 1.0, 0.7705078125], + "population_validation_mean_exact_match": 0.47979736328125, + "population_validation_worst_link_exact_match": 0.3974609375, + "run_id": "20260718T130109Z-ecp7-development", + "universal_translator_validation_exact_match": 0.63037109375, + "unique_messages_per_sender": [11189, 10119, 10980, 11404] + }, + "seed": 11, + "split_artifact_sha256": "676195997ee0c77d5c9565bae045dbb938b432a2e4e74b666233b0a41f924718", + "split_sha256": "4947058c75ab07cb43a87eb82776b12cb2a7e2eeba7114de110d3b852cbc64cd" +} diff --git a/src/ai_taal/models.py b/src/ai_taal/models.py index 53ae08b..30c7aa9 100644 --- a/src/ai_taal/models.py +++ b/src/ai_taal/models.py @@ -746,7 +746,7 @@ def __init__(self, spec: ModelSpec) -> None: nn.Linear(spec.hidden_dim, size) for size in spec.factor_sizes ) - def forward(self, message: Tensor) -> tuple[Tensor, ...]: + def _message_representation(self, message: Tensor) -> Tensor: if message.ndim == 2: embedded = self.symbol_embedding(message) elif message.ndim == 3: @@ -756,7 +756,23 @@ def forward(self, message: Tensor) -> tuple[Tensor, ...]: "Message must have shape [batch,length] or [batch,length,vocab]." ) flattened = embedded.reshape(embedded.shape[0], -1) - representation = torch.tanh(self.message_projection(flattened)) + return torch.tanh(self.message_projection(flattened)) + + def forward(self, message: Tensor) -> tuple[Tensor, ...]: + representation = self._message_representation(message) + return tuple(head(representation) for head in self.factor_heads) + + +class DeepPositionAwareMLPReceiver(PositionAwareMLPReceiver): + """Add one generic shared hidden layer before all factor heads.""" + + def __init__(self, spec: ModelSpec) -> None: + super().__init__(spec) + self.hidden_projection = nn.Linear(spec.hidden_dim, spec.hidden_dim) + + def forward(self, message: Tensor) -> tuple[Tensor, ...]: + representation = self._message_representation(message) + representation = torch.tanh(self.hidden_projection(representation)) return tuple(head(representation) for head in self.factor_heads) @@ -825,6 +841,8 @@ def make_receiver(spec: ModelSpec) -> ReceiverModel: return Receiver(spec) if spec.receiver_family == "position_aware_mlp_receiver": return PositionAwareMLPReceiver(spec) + if spec.receiver_family == "deep_position_aware_mlp_receiver": + return DeepPositionAwareMLPReceiver(spec) if spec.receiver_family == "factorized_permutation_slot_receiver": return FactorizedPermutationSlotReceiver(spec) raise ValueError(f"Unknown receiver architecture: {spec.receiver_family}") diff --git a/tests/conftest.py b/tests/conftest.py index dba600f..ad79aad 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -98,3 +98,8 @@ def ecp7_b10_config(): @pytest.fixture(scope="session") def ecp7_b11_config(): return load_config(PROJECT_ROOT / "config" / "ecp7-b11-development.yaml") + + +@pytest.fixture(scope="session") +def ecp7_b12_config(): + return load_config(PROJECT_ROOT / "config" / "ecp7-b12-development.yaml") diff --git a/tests/test_config.py b/tests/test_config.py index 4c70e04..a2a43eb 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -432,3 +432,26 @@ def test_code_utilization_weight_decay_must_be_bounded(ecp7_b11_config): ] = 1.0 with pytest.raises(ConfigError, match="below its initial weight"): validate_config(excessive_final) + + +def test_ecp7_b12_changes_only_the_shared_decoder_depth( + ecp7_b10_config, ecp7_b12_config +): + assert ecp7_b12_config["world"] == ecp7_b10_config["world"] + assert ecp7_b12_config["dataset"] == ecp7_b10_config["dataset"] + assert ecp7_b12_config["channel"] == ecp7_b10_config["channel"] + assert ecp7_b12_config["training"] == ecp7_b10_config["training"] + assert ecp7_b12_config["agents"]["sender"] == ecp7_b10_config["agents"][ + "sender" + ] + assert ecp7_b12_config["agents"]["population"] == ecp7_b10_config["agents"][ + "population" + ] + assert ecp7_b12_config["agents"]["receiver"] == { + "family": "deep_position_aware_mlp_receiver" + } + b10_translator = deepcopy(ecp7_b10_config["agents"]["translator"]) + b12_translator = deepcopy(ecp7_b12_config["agents"]["translator"]) + assert b10_translator.pop("family") == "position_aware_mlp_receiver" + assert b12_translator.pop("family") == "deep_position_aware_mlp_receiver" + assert b12_translator == b10_translator diff --git a/tests/test_models.py b/tests/test_models.py index 46d8d8d..7c7912d 100644 --- a/tests/test_models.py +++ b/tests/test_models.py @@ -5,6 +5,7 @@ from ai_taal.models import ( BoundedAutoregressiveSender, BoundedParallelSender, + DeepPositionAwareMLPReceiver, CommunicationSystem, FactorizedPermutationSlotReceiver, InjectivePermutationSlotSender, @@ -345,3 +346,37 @@ def test_position_aware_mlp_receiver_checkpoint_round_trip( ) assert exit_code == 0 assert output_path.read_text(encoding="utf-8").startswith("[[") + + +def test_deep_position_aware_receiver_adds_one_shared_layer(ecp7_b12_config): + receiver = DeepPositionAwareMLPReceiver(ModelSpec.from_config(ecp7_b12_config)) + message = torch.tensor([[0, 1, 2, 3], [15, 15, 7, 7]]) + + outputs = receiver(message) + + assert [output.shape for output in outputs] == [ + (2, 16), + (2, 16), + (2, 8), + (2, 8), + ] + assert receiver.hidden_projection.in_features == receiver.spec.hidden_dim + assert receiver.hidden_projection.out_features == receiver.spec.hidden_dim + assert not hasattr(receiver, "binding_matrix") + assert not hasattr(receiver, "factor_projections") + + +def test_deep_position_aware_receiver_checkpoint_round_trip( + ecp7_b12_config, tmp_path +): + receiver = DeepPositionAwareMLPReceiver(ModelSpec.from_config(ecp7_b12_config)) + path = tmp_path / "deep-position-aware-receiver.pt" + save_agent_checkpoint(path, receiver, kind="receiver") + + loaded = load_agent_checkpoint(path, expected_kind="receiver") + + assert isinstance(loaded, DeepPositionAwareMLPReceiver) + message = torch.tensor([[0, 1, 2, 3], [15, 15, 7, 7]]) + expected = tuple(output.detach() for output in receiver(message)) + actual = loaded(message) + assert all(torch.equal(left, right) for left, right in zip(actual, expected))