From aec0be4a62f9d2f86b1552c256d0837040f65b5f Mon Sep 17 00:00:00 2001 From: FatosLocos Date: Sat, 18 Jul 2026 14:10:00 +0200 Subject: [PATCH] Record sealed ECP-7 context-invariance batch --- README.md | 9 +-- config/ecp7-b8-development.yaml | 18 +++++ docs/AI_AGENT_START.md | 26 +++---- docs/development-log-ecp7.md | 72 +++++++++++++++++++ docs/research-design-ecp7.md | 41 +++++++++++ evidence/ecp7-development/README.md | 4 ++ .../ecp7-development/batch8-manifest.json | 41 +++++++++++ src/ai_taal/training.py | 4 +- tests/conftest.py | 5 ++ tests/test_config.py | 31 ++++++++ tests/test_training.py | 21 ++++++ 11 files changed, 254 insertions(+), 18 deletions(-) create mode 100644 config/ecp7-b8-development.yaml create mode 100644 evidence/ecp7-development/batch8-manifest.json diff --git a/README.md b/README.md index 60f41de..16b5e9c 100644 --- a/README.md +++ b/README.md @@ -26,13 +26,13 @@ The protocol uses no words or alphabet, but four meaning-free local symbols. For See [`docs/results-ecp6.md`](docs/results-ecp6.md) for the conclusion, [`docs/protocol-specification-ecp6.md`](docs/protocol-specification-ecp6.md) for the wire format and [`evidence/ecp6/report.md`](evidence/ecp6/report.md) for the compact confirmatory evidence. ECP-7 development now tests how much of that result depends on the explicit -one-factor-per-slot architecture. Its first seven sealed batches are valid +one-factor-per-slot architecture. Its first eight sealed batches are valid negative results. Straight-through hard utilization remains the best weakly structured population-validation result at 1.92%. A parallel joint-context sender increased code use to 3,118–3,415 messages and train exactness to 9.60%, -but shape remained below chance and validation stayed at 0.51%. The ECP-6 -positive controls remained perfect. The ECP-7 confirmatory test remains sealed. -See +but shape remained below chance and validation stayed at 0.51%. Adding relaxed +context invariance reduced that gain. The ECP-6 positive controls remained +perfect. The ECP-7 confirmatory test remains sealed. See [`docs/development-log-ecp7.md`](docs/development-log-ecp7.md). ## Build on this work @@ -80,6 +80,7 @@ Do not translate those artifacts in place. New experiment configurations and all - [`config/ecp7-b5-development.yaml`](config/ecp7-b5-development.yaml) — straight-through sender consensus on the Batch 3 base. - [`config/ecp7-b6-development.yaml`](config/ecp7-b6-development.yaml) — normalized minimax factor reconstruction on the Batch 3 base. - [`config/ecp7-b7-development.yaml`](config/ecp7-b7-development.yaml) — bounded parallel generation from one shared joint context. +- [`config/ecp7-b8-development.yaml`](config/ecp7-b8-development.yaml) — algebraic context invariance on the parallel sender. - [`config/ecp7-positive-control-development.yaml`](config/ecp7-positive-control-development.yaml) — ECP-6 positive control on the ECP-7 split. - [`config/ecp0.yaml`](config/ecp0.yaml) — machine-readable configuration from step 1. - [`config/ecp1.yaml`](config/ecp1.yaml) — frozen configuration of the population trial. diff --git a/config/ecp7-b8-development.yaml b/config/ecp7-b8-development.yaml new file mode 100644 index 0000000..a2b8b5a --- /dev/null +++ b/config/ecp7-b8-development.yaml @@ -0,0 +1,18 @@ +extends: ecp7-b7-development.yaml + +experiment: + title: ECP-7 parallel sender with context-invariant transitions + arm: parallel_sender_algebraic_consistency + objective: >- + Test whether making the same atomic meaning change produce the same relaxed + message displacement across training contexts converts the parallel + sender's memorized distinctions into a compositional protocol. + +training: + algebraic_consistency: + enabled: true + weight: 1.0 + warmup_steps: 800 + relaxed_temperature: 1.0 + batch_size: 32 + quadruple_pool_size: 8192 diff --git a/docs/AI_AGENT_START.md b/docs/AI_AGENT_START.md index dcec9df..460307f 100644 --- a/docs/AI_AGENT_START.md +++ b/docs/AI_AGENT_START.md @@ -41,7 +41,7 @@ python3.12 -m venv .venv .venv/bin/ecp6 --config config/ecp6.yaml validate ``` -Expected baseline: 68 passing tests and split sizes `14336/1024/1024`. +Expected baseline: 70 passing tests and split sizes `14336/1024/1024`. ## 4. Reproduce ECP-6 @@ -71,7 +71,7 @@ This reruns a known experiment. It must not be presented as a new independent co ## 5. Continue ECP-7 scientifically -Do not modify ECP-0 through ECP-6 or the completed ECP-7 Batch 1 through Batch 7 +Do not modify ECP-0 through ECP-6 or the completed ECP-7 Batch 1 through Batch 8 configs. ECP7-B1-I collapsed to 130–139 hard messages. ECP7-B2-I improved its soft objective but collapsed further to 85–104 hard messages. ECP7-B3-I applied that loss to straight-through hard messages and improved to 585–972 messages, @@ -84,12 +84,14 @@ remained near chance and validation reached only 0.46%. All paired ECP-6 positive controls stayed perfect. ECP7-B7-I replaced autoregression with a parallel joint-context sender. It increased train exactness to 9.60% and used 3,118–3,415 messages, but shape stayed below chance and validation reached only -0.51%. The confirmatory ECP-7 test is still sealed. +0.51%. ECP7-B8-I added context-invariant atomic transitions, but regressed to +857–1,135 messages and 0.32% validation. The confirmatory ECP-7 test is still +sealed. Continue with this sequence: 1. Read `docs/research-design-ecp7.md` and `docs/development-log-ecp7.md`. -2. Define exactly one ECP7-B8 intervention and its failure criterion. +2. Define exactly one ECP7-B9 intervention and its failure criterion. 3. Register the variant and immutable configuration hashes before training. 4. Add tests for every new invariant. 5. Use only `smoke` and `develop`; keep the ECP-7 test split sealed. @@ -103,20 +105,20 @@ Continue with this sequence: ## 6. Recommended next experiment Batch 7 is the strongest optimization result but still memorizes rather than -composes: train exactness and code-space use increased sharply, while held-out -color-shape exactness remained low. The most informative next step is one -context-invariance objective on the parallel sender. A clean ECP7-B8 -progression is: +composes. Batch 8 context invariance suppressed that gain. B7 messages change +when shape changes, and its fresh universal translator decodes shape better +than its jointly trained recurrent receivers. The most informative next step is +one generic decoder change. A clean ECP7-B9 progression is: - keep the same world and bit budget; - keep the joint autoregressive sender and generic isolated receiver; -- keep the B7 sender and receiver, then add the existing algebraic consistency - loss for identical atomic factor transitions across different contexts; +- keep the B7 sender and losses, then replace the GRU final-state decoder with + one position-aware MLP over all four embedded message slots; - measure injectivity, validation composition and new-reader induction; - rerun the ECP-6 architecture as the frozen positive control. -Do not restore the Batch 4, Batch 5 or Batch 6 terms, and do not combine -context-invariance with receiver changes, architectural resizing, longer +Do not restore the Batch 4 through Batch 6 or Batch 8 terms, and do not combine +the receiver change with sender changes, architectural resizing, longer training, variable-length messages or negotiation. That would make its effect uninterpretable. diff --git a/docs/development-log-ecp7.md b/docs/development-log-ecp7.md index 6a34d2b..b1c4175 100644 --- a/docs/development-log-ecp7.md +++ b/docs/development-log-ecp7.md @@ -490,3 +490,75 @@ context-invariance objective: the existing algebraic consistency loss for the same atomic factor transition in two different training contexts. This directly targets composition rather than memorization and assigns no factor to a message slot. It must not restore B4–B6 losses or change the receiver simultaneously. + +## Batch 8 preregistration + +Date: July 18, 2026
+Seed: `11`
+Maximum population steps: `5,000`
+Split-SHA-256: `4947058c75ab07cb43a87eb82776b12cb2a7e2eeba7114de110d3b852cbc64cd`
+Test unsealed: **no** + +ECP7-B8-I retains the complete Batch 7 architecture and loss set, then activates +one context-invariance term. It matches two training-only instances of the same +atomic factor transition and minimizes the squared difference between their +relaxed message displacements. Its fixed construction and hyperparameters are +recorded in `docs/research-design-ecp7.md`. No factor is assigned to a channel +slot. The unchanged positive control is rerun. Smoke is mechanical only and +cannot authorize tuning. + +## Batch 8 results + +Positive-control run: `runs/ecp7-batch8-control-development/20260718T120358Z-ecp7-development`
+Intervention run: `runs/ecp7-batch8-intervention-development/20260718T120358Z-ecp7-development`
+Test unsealed: **no** + +| Metric | Positive control | ECP7-B8-I | +|---|---:|---:| +| Population train, mean | 100% | 0.5349% | +| Population train, worst link | 100% | 0.4464% | +| Population validation, mean | 100% | 0.3235% | +| Population validation, worst link | 100% | 0.0977% | +| Universal translator, validation | 100% | 0.6836% | +| Exact sender-message agreement | 100% | 1.18% | +| Unique messages per sender | 15,360 | 857–1,135 | +| Collision meanings per sender | 0 | 14,225–14,503 | +| Message entropy | 13.91 bits | 8.23–8.92 bits | +| Development gate | pass | **fail** | + +The positive control again validated the batch at 100%. ECP7-B8-I failed the +population, validation, translator and injectivity gates. It stopped at step +2,200 under the unchanged early-stopping rule, with step 600 selected. + +At that checkpoint, algebraic consistency loss was `0.05265` at its scheduled +weight of `0.75`, and the hard-utilization loss was `-0.7321`. The surrogate +could be reduced, but it did not create a useful discrete compositional code. +Compared with B7, training exactness fell from 9.5965% to 0.5349%, unique +messages fell from 3,118–3,415 to 857–1,135, population validation fell from +0.5066% to 0.3235%, and translator validation fell from 2.7588% to 0.6836%. + +Population validation factor accuracies were approximately +`[7.63%, 6.57%, 89.04%, 96.64%]`. The context objective therefore suppressed +the strong B7 color signal and left shape near chance. Universal-translator +factor accuracies of `[14.60%, 10.30%, 98.12%, 99.02%]` again show that a fresh +decoder extracts more color and shape information than the jointly trained +population receivers, but far below the registered gate. + +Both sealed analyses report 65 matching artifact hashes, 16,384 validation-only +episodes, no confirmatory-test keys, valid local alphabets, and exactly 14 +declared bits for every logged message. + +## Batch 8 decision + +ECP7-B8-I is recorded as a negative development result and is not retained. +Relaxed transition invariance competed with the discrete task and utilization +objectives instead of converting B7's distinctions into composition. The +confirmatory split remains sealed. + +B7 remains the strongest optimization base. Its shape minimal pairs change +approximately 1.11–1.13 message positions and concentrate about 64% of that +change in one position, while its post-hoc universal translator decodes shape +better than the jointly trained receivers. A future ECP7-B9 may keep the B7 +sender and losses but replace the recurrent final-state receiver with one +generic position-aware MLP over all four embedded slots. This tests decoder +access without factor-to-slot binding and must not add another loss. diff --git a/docs/research-design-ecp7.md b/docs/research-design-ecp7.md index f4c4159..020e301 100644 --- a/docs/research-design-ecp7.md +++ b/docs/research-design-ecp7.md @@ -356,3 +356,44 @@ All existing validity and development gates remain unchanged. Failure is recorded without confirmatory test access. No receiver change, hidden-size change, extra loss, longer duration or alternative parallel architecture may be tried inside Batch 7. + +## Batch 8 preregistration — Algebraic context invariance + +Status: **preregistered for sealed development**
+Preregistered: July 18, 2026
+Configuration: `config/ecp7-b8-development.yaml`
+Raw configuration SHA-256: `3eac0d25bf636d4a1aeb6161ad479a4a78f0a3e88788a0274c154c21ff192829`
+Effective configuration SHA-256: `6e8f9ea0e968693bd9f67a4ca1180ab447c6e2d6c2fbc6b9f9a66582aa02fa3c` + +Batch 7 greatly improved training exactness and code-space use, but those +distinctions did not compose across the held-out color-shape matching. Batch 8 +retains the entire B7 architecture and loss set, then activates one existing +context-invariance objective that was previously disabled. + +Every training quadruple contains meanings `A`, `B`, `A'`, and `B'`. The +transitions `A→A'` and `B→B'` change the same factor value in two different +contexts, and all four meanings belong to the training split. For relaxed +message distributions `m`, the objective is: + +`L_alg = mean ||(m(A') - m(A)) - (m(B') - m(B))||²` + +This encourages the same atomic semantic change to produce the same message +displacement regardless of the other factor values. It uses factor identities +only to construct matching transitions. It does not specify a target message, +symbol, channel slot, factor-to-slot binding or codebook. + +The loss weight is fixed at `1.0` with an 800-step linear warmup, relaxed +temperature `1.0`, 32 quadruples per update and a deterministic pool of 8,192 +training-only quadruples. These values match the dormant inherited settings; +only `enabled` and `weight` change from B7. The parallel sender, generic +receiver, hard-utilization loss, channel, split, training duration, selection +and translator remain unchanged. + +The batch contains exactly two seed-11 runs: + +1. the unchanged ECP-6 positive control; +2. ECP7-B8-I with algebraic context invariance on the B7 parallel sender. + +All existing validity and development gates remain unchanged. Failure is +recorded without confirmatory test access. No alternative quadruple sampling, +weight, warmup, receiver, architecture or duration may be tried inside Batch 8. diff --git a/evidence/ecp7-development/README.md b/evidence/ecp7-development/README.md index 7deb59c..437c387 100644 --- a/evidence/ecp7-development/README.md +++ b/evidence/ecp7-development/README.md @@ -43,6 +43,10 @@ sender. It reached 3,118–3,415 messages, 9.60% train exactness and 45.64% colo accuracy, but shape remained below chance and validation stayed at 0.51%. Exact identities and hashes are in `batch7-manifest.json`. +Batch 8 added algebraic context invariance to Batch 7. The relaxed surrogate +improved, but the discrete protocol regressed to 857–1,135 messages and 0.32% +validation. Exact identities and hashes are in `batch8-manifest.json`. + Reproduce the development arms with: ```bash diff --git a/evidence/ecp7-development/batch8-manifest.json b/evidence/ecp7-development/batch8-manifest.json new file mode 100644 index 0000000..ff447e4 --- /dev/null +++ b/evidence/ecp7-development/batch8-manifest.json @@ -0,0 +1,41 @@ +{ + "batch": "ECP7-B8", + "confirmatory_test_unsealed": false, + "control": { + "artifact_hashes": { + "effective_config.yaml": "4bfb976293f117d3ae35f790e4fd6371628be95d91febb334bc275dc5d6b66d4", + "posthoc-analysis.json": "f22ba2bf1042a97deb5274f03fab22d70678d1418986927764ced363634e0846", + "seed-11/metrics.json": "e9bde51415a531c0ad32f7caf921048840dff9c1f6d55b1ad4635292ff668b60", + "summary.json": "ad2301a93c46c175d2b38687dfe7b66e8b75a14e02b8971fdf18a8a30d3273ae" + }, + "development_gate_passed": true, + "population_train_mean_exact_match": 1.0, + "population_validation_mean_exact_match": 1.0, + "run_id": "20260718T120358Z-ecp7-development", + "universal_translator_validation_exact_match": 1.0, + "unique_messages_per_sender": [15360, 15360, 15360, 15360] + }, + "intervention": { + "artifact_hashes": { + "effective_config.yaml": "15fa04bb33b321b2ca8c29f01da84ff8bbcdd70e85ecbe96c08d6a51cca84cef", + "posthoc-analysis.json": "4f1af42cbd75044cd7149a729a4869d81b55a3c3e8b929da0a08570d0e608c60", + "seed-11/metrics.json": "945cf063cb1300029175a590452d7e2f39e1ff068b6eb423892357f15581c745", + "summary.json": "7fefb6774ba89d3300aaa2138640057484181591940ce918c27349a5e1fa17ff" + }, + "best_algebraic_consistency_loss": 0.05264976993203163, + "best_code_utilization_loss": -0.7321186661720276, + "best_step": 600, + "development_gate_passed": false, + "exact_sender_message_agreement": 0.011827256944444446, + "message_entropy_bits_per_sender": [8.229443275271354, 8.409431758360435, 8.922917128759757, 8.81365004262187], + "population_train_mean_exact_match": 0.005349295479910714, + "population_validation_factor_accuracy": [0.0762939453125, 0.065673828125, 0.890380859375, 0.9664306640625], + "population_validation_mean_exact_match": 0.00323486328125, + "run_id": "20260718T120358Z-ecp7-development", + "universal_translator_validation_exact_match": 0.0068359375, + "unique_messages_per_sender": [857, 895, 1123, 1135] + }, + "seed": 11, + "split_artifact_sha256": "676195997ee0c77d5c9565bae045dbb938b432a2e4e74b666233b0a41f924718", + "split_sha256": "4947058c75ab07cb43a87eb82776b12cb2a7e2eeba7114de110d3b852cbc64cd" +} diff --git a/src/ai_taal/training.py b/src/ai_taal/training.py index bf7ac69..fe5b3b4 100644 --- a/src/ai_taal/training.py +++ b/src/ai_taal/training.py @@ -19,7 +19,7 @@ PopulationSystem, Receiver, ReceiverModel, - Sender, + SenderModel, make_receiver, make_sender, ) @@ -756,7 +756,7 @@ def build_algebraic_quadruples( def algebraic_consistency_loss( - sender: Sender, quadruples: Tensor, *, temperature: float + sender: SenderModel, quadruples: Tensor, *, temperature: float ) -> Tensor: """Make the same factor change context-invariant in message distribution.""" if quadruples.ndim != 3 or quadruples.shape[1:] != (4, 4): diff --git a/tests/conftest.py b/tests/conftest.py index 4fc2ca7..56aa74e 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -78,3 +78,8 @@ def ecp7_b6_config(): @pytest.fixture(scope="session") def ecp7_b7_config(): return load_config(PROJECT_ROOT / "config" / "ecp7-b7-development.yaml") + + +@pytest.fixture(scope="session") +def ecp7_b8_config(): + return load_config(PROJECT_ROOT / "config" / "ecp7-b8-development.yaml") diff --git a/tests/test_config.py b/tests/test_config.py index 9aa0dc5..e2485a2 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -297,3 +297,34 @@ def test_ecp7_b7_changes_only_the_sender_generation_architecture( assert "joint_message_collision" not in b7_training assert "sender_message_consensus" not in b7_training assert "factor_minimax" not in b7_training + + +def test_ecp7_b8_adds_only_algebraic_context_invariance( + ecp7_b7_config, ecp7_b8_config +): + assert ecp7_b8_config["world"] == ecp7_b7_config["world"] + assert ecp7_b8_config["dataset"] == ecp7_b7_config["dataset"] + assert ecp7_b8_config["channel"] == ecp7_b7_config["channel"] + assert ecp7_b8_config["agents"] == ecp7_b7_config["agents"] + b7_training = deepcopy(ecp7_b7_config["training"]) + b8_training = deepcopy(ecp7_b8_config["training"]) + assert b7_training.pop("algebraic_consistency") == { + "enabled": False, + "weight": 0.0, + "warmup_steps": 800, + "relaxed_temperature": 1.0, + "batch_size": 32, + "quadruple_pool_size": 8192, + } + assert b8_training.pop("algebraic_consistency") == { + "enabled": True, + "weight": 1.0, + "warmup_steps": 800, + "relaxed_temperature": 1.0, + "batch_size": 32, + "quadruple_pool_size": 8192, + } + assert b8_training == b7_training + assert "joint_message_collision" not in b8_training + assert "sender_message_consensus" not in b8_training + assert "factor_minimax" not in b8_training diff --git a/tests/test_training.py b/tests/test_training.py index 3f565a3..3b09cd6 100644 --- a/tests/test_training.py +++ b/tests/test_training.py @@ -4,6 +4,7 @@ from ai_taal.models import ( BoundedAutoregressiveSender, + BoundedParallelSender, FactorizedPermutationSlotReceiver, ModelSpec, PopulationSystem, @@ -85,6 +86,26 @@ def test_algebraic_consistency_loss_backpropagates(ecp2_config): assert any(parameter.grad is not None for parameter in sender.parameters()) +def test_algebraic_consistency_supports_the_bounded_parallel_sender( + ecp7_b8_config, +): + split = build_splits(ecp7_b8_config) + sender = BoundedParallelSender(ModelSpec.from_config(ecp7_b8_config)) + quadruples = build_algebraic_quadruples( + split.train, + factor_sizes=sender.spec.factor_sizes, + sample_count=8, + seed=456, + ) + + loss = algebraic_consistency_loss(sender, quadruples, temperature=1.0) + loss.backward() + + assert torch.isfinite(loss) + assert loss.item() >= 0.0 + assert any(parameter.grad is not None for parameter in sender.parameters()) + + def test_slot_binding_consensus_loss_backpropagates(ecp2_config): config = deepcopy(ecp2_config) config["agents"]["sender"]["family"] = "learned_permutation_slot_sender"