From c63e5e32622fc56753cea1513d9f0f7f2fe31fb3 Mon Sep 17 00:00:00 2001 From: FatosLocos Date: Sat, 18 Jul 2026 22:00:33 +0200 Subject: [PATCH 1/2] Preregister ECP-8 Batch 1 capacity study --- AGENTS.md | 4 +- README.md | 13 +++ config/ecp8-base-development.yaml | 64 +++++++++++++ config/ecp8-control-development.yaml | 60 ++++++++++++ config/ecp8-development.yaml | 21 +++++ config/ecp8-positive-control-development.yaml | 8 ++ docs/AI_AGENT_START.md | 22 +++-- docs/development-log-ecp8.md | 30 ++++++ docs/research-design-ecp8.md | 94 +++++++++++++++++++ tests/conftest.py | 17 ++++ tests/test_config.py | 33 +++++++ tests/test_population.py | 31 ++++++ 12 files changed, 387 insertions(+), 10 deletions(-) create mode 100644 config/ecp8-base-development.yaml create mode 100644 config/ecp8-control-development.yaml create mode 100644 config/ecp8-development.yaml create mode 100644 config/ecp8-positive-control-development.yaml create mode 100644 docs/development-log-ecp8.md create mode 100644 docs/research-design-ecp8.md diff --git a/AGENTS.md b/AGENTS.md index d35cfb5..c65fa81 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -41,6 +41,6 @@ python3.12 -m venv .venv - Keep existing ECP-0 through ECP-6 configurations reproducible. - Document new experiments in three stages: preregistration, sealed development log, confirmatory results. - Write new configurations, documentation, reports and user-facing output in English. -- When proposing ECP-7, prefer a single falsifiable intervention over several simultaneous architecture changes. +- For ECP-8, preserve the preregistered three-arm capacity isolation and do not combine it with another architecture, loss, split, or training change. -The recommended next research question is whether compositional structure still emerges when the explicit one-factor-per-slot architectural bias is weakened. +The active research question is whether two surplus unallocated channel bits can make the strongest weak-structure system injective without restoring a factor-slot binding. diff --git a/README.md b/README.md index 3ed9025..2100073 100644 --- a/README.md +++ b/README.md @@ -73,6 +73,14 @@ configuration was frozen and its confirmatory split was never opened. See The compact synthesis is in [`docs/ecp7-development-conclusion.md`](docs/ecp7-development-conclusion.md). +ECP-8 Batch 1 is now preregistered for sealed development. It tests whether the +strongest generic ECP-7 architecture becomes injective when its four positions +may all use the complete 16-symbol vocabulary: 16 fixed bits instead of the +14-bit lower bound. A factorized positive control and a direct 14-bit +weak-structure control use the same fresh split. No ECP-8 training or +confirmatory metric existed when the configurations, hashes and gates were +registered. See [`docs/research-design-ecp8.md`](docs/research-design-ecp8.md). + ## Build on this work A new developer or fresh AI agent can start with no previous chat context. First read [`AGENTS.md`](AGENTS.md) for the binding research rules and then follow [`docs/AI_AGENT_START.md`](docs/AI_AGENT_START.md) for installation, architecture overview, reproduction and proper setup of ECP-7. @@ -112,6 +120,8 @@ Do not translate those artifacts in place. New experiment configurations and all - [`docs/research-design-ecp7.md`](docs/research-design-ecp7.md) — preregistered weak-structure question, paired control and sealed split. - [`docs/development-log-ecp7.md`](docs/development-log-ecp7.md) — the first negative weak-structure batch and integrity checks. - [`docs/ecp7-development-conclusion.md`](docs/ecp7-development-conclusion.md) — sealed development synthesis and closure after 29 variants. +- [`docs/research-design-ecp8.md`](docs/research-design-ecp8.md) — preregistered surplus-capacity question, fresh split and three-arm gate. +- [`docs/development-log-ecp8.md`](docs/development-log-ecp8.md) — sealed ECP-8 development record. - [`config/ecp7-development.yaml`](config/ecp7-development.yaml) — sealed weak-structure intervention configuration. - [`config/ecp7-b2-development.yaml`](config/ecp7-b2-development.yaml) — factor-agnostic soft code-utilization intervention. - [`config/ecp7-b3-development.yaml`](config/ecp7-b3-development.yaml) — the same utilization formula on straight-through hard messages. @@ -142,6 +152,9 @@ Do not translate those artifacts in place. New experiment configurations and all - [`config/ecp7-b28-development.yaml`](config/ecp7-b28-development.yaml) — Batch 25 with one zero-initialized shared residual sender interaction. - [`config/ecp7-b29-development.yaml`](config/ecp7-b29-development.yaml) — Batch 28 with residual parameters frozen through step 30,000. - [`config/ecp7-positive-control-development.yaml`](config/ecp7-positive-control-development.yaml) — ECP-6 positive control on the ECP-7 split. +- [`config/ecp8-positive-control-development.yaml`](config/ecp8-positive-control-development.yaml) — factorized 14-bit positive control on the fresh ECP-8 split. +- [`config/ecp8-control-development.yaml`](config/ecp8-control-development.yaml) — paired 14-bit weak-structure control. +- [`config/ecp8-development.yaml`](config/ecp8-development.yaml) — four-global-position 16-bit intervention. - [`config/ecp0.yaml`](config/ecp0.yaml) — machine-readable configuration from step 1. - [`config/ecp1.yaml`](config/ecp1.yaml) — frozen configuration of the population trial. - [`config/ecp3.yaml`](config/ecp3.yaml) — frozen configuration of the first successful base model. diff --git a/config/ecp8-base-development.yaml b/config/ecp8-base-development.yaml new file mode 100644 index 0000000..3312272 --- /dev/null +++ b/config/ecp8-base-development.yaml @@ -0,0 +1,64 @@ +extends: ecp6-development.yaml + +experiment: + id: ecp8 + title: ECP-8 shared sealed development base + status: development_test_sealed + frozen_at_utc: null + parent_experiment: ecp7 + arm: shared_sealed_base + objective: >- + Test whether two surplus unallocated channel bits let a weak-structure + population learn an injective compositional protocol without restoring a + factor-to-slot binding. + +reproducibility: + dataset_seed: 18072058 + +dataset: + split_strategy: dual_explicit_fresh_color_shape_matchings + train_meanings: 14336 + validation_meanings: 1024 + compositional_test_meanings: 1024 + holdout_factors: [color, shape] + held_out_factor_pairs: 16 + validation_held_out_factor_pairs: 16 + variants_per_held_out_pair: 64 + excluded_factor_pairs: [] + explicit_validation_factor_pairs: + - {color: 0, shape: 12} + - {color: 1, shape: 13} + - {color: 2, shape: 14} + - {color: 3, shape: 15} + - {color: 4, shape: 0} + - {color: 5, shape: 1} + - {color: 6, shape: 2} + - {color: 7, shape: 3} + - {color: 8, shape: 4} + - {color: 9, shape: 5} + - {color: 10, shape: 6} + - {color: 11, shape: 7} + - {color: 12, shape: 8} + - {color: 13, shape: 9} + - {color: 14, shape: 10} + - {color: 15, shape: 11} + explicit_test_factor_pairs: + - {color: 0, shape: 13} + - {color: 1, shape: 14} + - {color: 2, shape: 15} + - {color: 3, shape: 0} + - {color: 4, shape: 1} + - {color: 5, shape: 2} + - {color: 6, shape: 3} + - {color: 7, shape: 4} + - {color: 8, shape: 5} + - {color: 9, shape: 6} + - {color: 10, shape: 7} + - {color: 11, shape: 8} + - {color: 12, shape: 9} + - {color: 13, shape: 10} + - {color: 14, shape: 11} + - {color: 15, shape: 12} + +outcome_thresholds: + paired_control_required: true diff --git a/config/ecp8-control-development.yaml b/config/ecp8-control-development.yaml new file mode 100644 index 0000000..6aae510 --- /dev/null +++ b/config/ecp8-control-development.yaml @@ -0,0 +1,60 @@ +extends: ecp7-b25-development.yaml + +experiment: + id: ecp8 + title: ECP-8 14-bit weak-structure paired control + status: development_test_sealed + frozen_at_utc: null + parent_experiment: ecp7 + arm: local_capacity_14_bit_control + objective: >- + Measure the strongest ECP-7 weak-structure system on the fresh ECP-8 split + before changing only its channel capacity allocation. + +reproducibility: + dataset_seed: 18072058 + +dataset: + split_strategy: dual_explicit_fresh_color_shape_matchings + train_meanings: 14336 + validation_meanings: 1024 + compositional_test_meanings: 1024 + holdout_factors: [color, shape] + held_out_factor_pairs: 16 + validation_held_out_factor_pairs: 16 + variants_per_held_out_pair: 64 + excluded_factor_pairs: [] + explicit_validation_factor_pairs: + - {color: 0, shape: 12} + - {color: 1, shape: 13} + - {color: 2, shape: 14} + - {color: 3, shape: 15} + - {color: 4, shape: 0} + - {color: 5, shape: 1} + - {color: 6, shape: 2} + - {color: 7, shape: 3} + - {color: 8, shape: 4} + - {color: 9, shape: 5} + - {color: 10, shape: 6} + - {color: 11, shape: 7} + - {color: 12, shape: 8} + - {color: 13, shape: 9} + - {color: 14, shape: 10} + - {color: 15, shape: 11} + explicit_test_factor_pairs: + - {color: 0, shape: 13} + - {color: 1, shape: 14} + - {color: 2, shape: 15} + - {color: 3, shape: 0} + - {color: 4, shape: 1} + - {color: 5, shape: 2} + - {color: 6, shape: 3} + - {color: 7, shape: 4} + - {color: 8, shape: 5} + - {color: 9, shape: 6} + - {color: 10, shape: 7} + - {color: 11, shape: 8} + - {color: 12, shape: 9} + - {color: 13, shape: 10} + - {color: 14, shape: 11} + - {color: 15, shape: 12} diff --git a/config/ecp8-development.yaml b/config/ecp8-development.yaml new file mode 100644 index 0000000..9139754 --- /dev/null +++ b/config/ecp8-development.yaml @@ -0,0 +1,21 @@ +extends: ecp8-control-development.yaml + +experiment: + title: ECP-8 global-capacity intervention development + arm: global_capacity_16_bit_intervention + objective: >- + Test whether four unrestricted 16-symbol positions eliminate hard-code + collisions in the unchanged weak-structure architecture and training setup. + +channel: + type: discrete_fixed_length + vocabulary_size: 16 + symbol_min: 0 + symbol_max: 15 + message_length: 4 + bits_per_symbol: 4 + bits_per_message: 16 + factor_alphabet_sizes: null + bits_per_symbol_by_factor: null + slot_alphabet_sizes: null + bits_per_symbol_by_slot: null diff --git a/config/ecp8-positive-control-development.yaml b/config/ecp8-positive-control-development.yaml new file mode 100644 index 0000000..d3758d0 --- /dev/null +++ b/config/ecp8-positive-control-development.yaml @@ -0,0 +1,8 @@ +extends: ecp8-base-development.yaml + +experiment: + title: ECP-8 factorized positive-control development + arm: ecp6_factorized_positive_control + objective: >- + Verify that the frozen ECP-6 architecture solves the fresh ECP-8 + validation matching at the information-theoretic 14-bit lower bound. diff --git a/docs/AI_AGENT_START.md b/docs/AI_AGENT_START.md index 4960b71..20f914e 100644 --- a/docs/AI_AGENT_START.md +++ b/docs/AI_AGENT_START.md @@ -41,7 +41,7 @@ python3.12 -m venv .venv .venv/bin/ecp6 --config config/ecp6.yaml validate ``` -Expected baseline: 133 passing tests and split sizes `14336/1024/1024`. +Expected baseline: 136 passing tests and split sizes `14336/1024/1024`. ## 4. Reproduce ECP-6 @@ -69,7 +69,7 @@ Then verify hashes, channel constraints and topographic null models: This reruns a known experiment. It must not be presented as a new independent confirmatory test. -## 5. Continue ECP-7 scientifically +## 5. Preserve the completed ECP-7 series Do not modify ECP-0 through ECP-6 or the completed ECP-7 Batch 1 through Batch 29 configs. ECP7-B1-I collapsed to 130–139 hard messages. ECP7-B2-I improved its @@ -192,12 +192,18 @@ ECP-7 development is closed after 29 sealed variants. Do not continue a residual schedule, collision coefficient, replay route or horizon search, and do not open the unused ECP-7 confirmatory split. -Start ECP-8 with a separate preregistration that chooses exactly one new -question. Plausible directions include learned compositional structure without -fixed slots, variable-length coding efficiency, interactive negotiation, or -transfer across altered synthetic worlds. These are different hypotheses: -select one, define its channel and baselines, generate a fresh deterministic -split, and register the development and confirmatory gates before training. +ECP-8 Batch 1 is a separate preregistration that selects the learned-structure +direction through one narrow capacity question. Read +`docs/research-design-ecp8.md` and +`docs/development-log-ecp8.md`. Three seed-11 arms were registered before +training: a factorized 14-bit positive control, the strongest weak-structure +14-bit paired control, and the identical weak system with four unrestricted +16-symbol positions. Do not change their split, seed, architecture, training, +channel or gates, and never add `--unseal-test` during development. + +After all three sealed runs, record the result even if it is negative. A passed +development gate only permits a later freeze decision; it does not itself +authorize confirmatory test access. ## 7. Definition of done for any contribution diff --git a/docs/development-log-ecp8.md b/docs/development-log-ecp8.md new file mode 100644 index 0000000..67d1aa5 --- /dev/null +++ b/docs/development-log-ecp8.md @@ -0,0 +1,30 @@ +# ECP-8 Sealed Development Log + +The ECP-8 confirmatory split is sealed. This log records every development arm, +including negative results, before any possible confirmatory access. + +## Batch 1 preregistration — Global channel capacity + +Preregistered: July 18, 2026 at 19:58:58 UTC
+Seed: `11`
+Test unsealed: **no**
+Split SHA-256: `241acd2ee42ff658ddf0c8c897d4e4df0f52a18098b3ef68c009c74c2134592a` + +| Arm | Configuration | Raw SHA-256 | Effective SHA-256 | +|---|---|---|---| +| ECP8-B1-P | `config/ecp8-positive-control-development.yaml` | `e996bf66c30b3e699b27046e9824c13c7ecf3f2cb9831c4b9f0528d17b7b5f05` | `3a8e55ee262418f1005405afec3eeee7ebaaad164cb83274bd95c95e2c0939c3` | +| ECP8-B1-C | `config/ecp8-control-development.yaml` | `9e3c865fcec9698bb95fcfc3ad429a354e26ffd39f2533dd7dc836dfdd3ab34a` | `fcaf5e8f39df2349cb3dedefcc233e5a08d813763a5b644d6a2ed2c050748f06` | +| ECP8-B1-I | `config/ecp8-development.yaml` | `f6219110ebfe8f26248d9dbd646d80e1033f63d618a2524f7b96668ebd7fdaea` | `0572857aa786e0a41995aedd1cee8c2bb6dbdbc47f46025411bb8c06615b9693` | + +ECP8-B1-C reruns the strongest ECP-7 weak-structure setup on the fresh ECP-8 +split. ECP8-B1-I changes only the local channel capacities: `[16,16,8,8]` +becomes `[16,16,16,16]`, increasing fixed payload size from 14 to 16 bits. +The sender still has one generic joint context and four position heads; neither +sender nor receiver receives a factor-slot binding. + +The full decision gate is fixed in `docs/research-design-ecp8.md`. In addition +to the existing train, validation and translator thresholds, the intervention +must be injective over all 15,360 accessible meanings, improve the paired +control's minimum codebook, and preserve both mean and worst-link validation. +Only these three seed-11 runs are admitted. The confirmatory test remains +unauthorized regardless of a single development result. diff --git a/docs/research-design-ecp8.md b/docs/research-design-ecp8.md new file mode 100644 index 0000000..f7f9eda --- /dev/null +++ b/docs/research-design-ecp8.md @@ -0,0 +1,94 @@ +# Research Design ECP-8 — Surplus Capacity Without Factor Slots + +Status: **preregistered for sealed development**
+Preregistered: July 18, 2026 at 19:58:58 UTC
+Positive-control configuration: `config/ecp8-positive-control-development.yaml`
+Paired-control configuration: `config/ecp8-control-development.yaml`
+Intervention configuration: `config/ecp8-development.yaml` + +## Question + +Can two surplus, semantically unallocated channel bits eliminate hard-message +collisions in the strongest weak-structure ECP-7 system, without restoring a +factor-to-slot sender or receiver binding? + +This is a new research phase, not ECP7-B30. ECP-7 established that the tested +generic 14-bit systems remain non-injective. ECP-8 isolates whether the exact +information-theoretic boundary itself is too restrictive for learning: the +weak sender and receiver, optimization, world and population stay fixed while +the four position alphabets change from `[16,16,8,8]` to `[16,16,16,16]`. + +## World and fresh sealed split + +The uniform world remains `16 × 16 × 8 × 8 = 16,384` meanings with 14 bits of +source entropy. The split is newly generated for ECP-8 and uses color-shape +matchings not used by ECP-6 or ECP-7: + +| Split | Meanings | Matching | +|---|---:|---| +| Train | 14,336 | all pairs outside both sealed matchings | +| Validation | 1,024 | `shape = color + 12 mod 16` | +| Confirmatory test | 1,024 | `shape = color + 13 mod 16` | + +Dataset seed: `18072058`
+Split SHA-256: `241acd2ee42ff658ddf0c8c897d4e4df0f52a18098b3ef68c009c74c2134592a` + +Only train and validation may be used during development. The confirmatory +matching must not appear in metrics, episodes, selection or post-hoc analysis. + +## Preregistered arms + +1. **Factorized positive control.** The ECP-6 factorized population uses its + minimal 14-bit channel on the fresh split. This validates the split and + implementation independently of the weak-structure hypothesis. +2. **Weak-structure paired control.** The ECP7-B25 bounded-parallel sender, + position-aware receiver, training schedule and 14-bit local capacities are + rerun unchanged on the fresh split. +3. **Global-capacity intervention.** The paired control changes only the + channel to four unrestricted positions over the same 16-symbol vocabulary. + Every position can use symbols `0..15`, for a fixed 16-bit message. + +The global positions retain order, but no position has a factor identity, +factor-local input, factor-specific decoder or restricted capacity. Sixteen +bits provide four times as many possible messages as the 16,384-meaning world, +with a 14.29% payload overhead over the source lower bound. + +## Development execution + +Batch 1 contains exactly three seed-11 development runs, one for each arm. All +population and translator training parameters are inherited unchanged. The +weak arms use at most 45,000 population steps, hard-symbol evaluation, the +registered B25 replay route and validation-only checkpoint selection. No loss, +architecture, optimizer, replay, horizon, temperature, seed, split or +translator variant may be added inside this batch. + +The positive control is valid only if it reaches 100% mean and worst-link train +and validation exactness, 100% translator validation, and 15,360 distinct hard +messages per sender over the accessible train-plus-validation meanings. + +The 16-bit intervention passes its development gate only if all conditions hold: + +1. population train mean is at least 97%; +2. population train worst link is at least 95%; +3. population validation mean and worst link are each at least 80%; +4. universal-translator validation is at least 70%; +5. every sender uses exactly 15,360 distinct hard messages over accessible + meanings; +6. its minimum sender codebook is larger than the paired control's minimum; +7. its mean and worst-link validation do not fall below the paired control; +8. channel, artifact-hash, isolation and sealed-test audits all pass. + +Passing development would permit freezing at most one ECP-8 design in a later +commit. It does not authorize test access by itself. Failure is published as a +sealed development result and does not justify an unregistered coefficient or +capacity search. + +## Frozen development identities + +| Arm | Raw file SHA-256 | Effective configuration SHA-256 | +|---|---|---| +| Positive control | `e996bf66c30b3e699b27046e9824c13c7ecf3f2cb9831c4b9f0528d17b7b5f05` | `3a8e55ee262418f1005405afec3eeee7ebaaad164cb83274bd95c95e2c0939c3` | +| 14-bit paired control | `9e3c865fcec9698bb95fcfc3ad429a354e26ffd39f2533dd7dc836dfdd3ab34a` | `fcaf5e8f39df2349cb3dedefcc233e5a08d813763a5b644d6a2ed2c050748f06` | +| 16-bit intervention | `f6219110ebfe8f26248d9dbd646d80e1033f63d618a2524f7b96668ebd7fdaea` | `0572857aa786e0a41995aedd1cee8c2bb6dbdbc47f46025411bb8c06615b9693` | + +These identities and thresholds were recorded before any ECP-8 training run. diff --git a/tests/conftest.py b/tests/conftest.py index 57fb1e8..99438d8 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -188,3 +188,20 @@ def ecp7_b28_config(): @pytest.fixture(scope="session") def ecp7_b29_config(): return load_config(PROJECT_ROOT / "config" / "ecp7-b29-development.yaml") + + +@pytest.fixture(scope="session") +def ecp8_positive_control_config(): + return load_config( + PROJECT_ROOT / "config" / "ecp8-positive-control-development.yaml" + ) + + +@pytest.fixture(scope="session") +def ecp8_control_config(): + return load_config(PROJECT_ROOT / "config" / "ecp8-control-development.yaml") + + +@pytest.fixture(scope="session") +def ecp8_config(): + return load_config(PROJECT_ROOT / "config" / "ecp8-development.yaml") diff --git a/tests/test_config.py b/tests/test_config.py index 48f67ba..84303a0 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -1022,3 +1022,36 @@ def test_sender_residual_activation_is_late_bounded_and_architecture_specific( invalid_sender["agents"]["sender"]["family"] = "bounded_parallel_sender" with pytest.raises(ConfigError, match="requires the residual parallel sender"): validate_config(invalid_sender) + + +def test_ecp8_arms_share_one_world_and_fresh_split( + ecp8_positive_control_config, ecp8_control_config, ecp8_config +): + arms = (ecp8_positive_control_config, ecp8_control_config, ecp8_config) + for arm in arms: + assert arm["experiment"]["id"] == "ecp8" + assert arm["experiment"]["status"] == "development_test_sealed" + assert arm["world"] == ecp8_positive_control_config["world"] + assert arm["reproducibility"] == ecp8_positive_control_config["reproducibility"] + assert arm["dataset"] == ecp8_positive_control_config["dataset"] + + +def test_ecp8_intervention_changes_only_weak_structure_channel_capacity( + ecp8_control_config, ecp8_config +): + assert ecp8_config["world"] == ecp8_control_config["world"] + assert ecp8_config["dataset"] == ecp8_control_config["dataset"] + assert ecp8_config["agents"] == ecp8_control_config["agents"] + assert ecp8_config["training"] == ecp8_control_config["training"] + assert ecp8_config["translator_training"] == ecp8_control_config[ + "translator_training" + ] + assert ecp8_control_config["channel"]["type"] == ( + "slot_local_discrete_fixed_length" + ) + assert ecp8_control_config["channel"]["slot_alphabet_sizes"] == [16, 16, 8, 8] + assert ecp8_control_config["channel"]["bits_per_message"] == 14 + assert ecp8_config["channel"]["type"] == "discrete_fixed_length" + assert ecp8_config["channel"]["slot_alphabet_sizes"] is None + assert ecp8_config["channel"]["bits_per_symbol"] == 4 + assert ecp8_config["channel"]["bits_per_message"] == 16 diff --git a/tests/test_population.py b/tests/test_population.py index 31ae686..cbe1cb9 100644 --- a/tests/test_population.py +++ b/tests/test_population.py @@ -120,3 +120,34 @@ def test_ecp7_uses_new_matching_for_intervention_and_positive_control( assert len(intervention_split.train) == 14_336 assert len(intervention_split.validation) == 1_024 assert len(intervention_split.compositional_test) == 1_024 + + +def test_ecp8_uses_one_fresh_matching_for_all_three_arms( + ecp6_config, + ecp7_config, + ecp8_positive_control_config, + ecp8_control_config, + ecp8_config, +): + prior_splits = (build_splits(ecp6_config), build_splits(ecp7_config)) + positive_split = build_splits(ecp8_positive_control_config) + control_split = build_splits(ecp8_control_config) + intervention_split = build_splits(ecp8_config) + + prior_pairs = { + pair + for split in prior_splits + for pair in ( + *split.held_out_validation_color_shape_pairs, + *split.held_out_color_shape_pairs, + ) + } + ecp8_pairs = set(positive_split.held_out_validation_color_shape_pairs) | set( + positive_split.held_out_color_shape_pairs + ) + assert prior_pairs.isdisjoint(ecp8_pairs) + assert positive_split.manifest() == control_split.manifest() + assert positive_split.manifest() == intervention_split.manifest() + assert len(intervention_split.train) == 14_336 + assert len(intervention_split.validation) == 1_024 + assert len(intervention_split.compositional_test) == 1_024 From 52d8be4c82a4de9742a06adc2f0c53da198f8e50 Mon Sep 17 00:00:00 2001 From: FatosLocos Date: Sat, 18 Jul 2026 22:17:48 +0200 Subject: [PATCH 2/2] Record ECP-8 Batch 1 capacity result --- AGENTS.md | 4 +- README.md | 17 +-- docs/AI_AGENT_START.md | 10 +- docs/development-log-ecp8.md | 69 +++++++++++ docs/research-design-ecp8.md | 18 ++- evidence/ecp8-development/README.md | 20 +++ .../ecp8-development/batch1-manifest.json | 114 ++++++++++++++++++ src/ai_taal/analysis.py | 26 +++- tests/test_analysis.py | 61 +++++++++- 9 files changed, 320 insertions(+), 19 deletions(-) create mode 100644 evidence/ecp8-development/README.md create mode 100644 evidence/ecp8-development/batch1-manifest.json diff --git a/AGENTS.md b/AGENTS.md index c65fa81..7f31529 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -41,6 +41,6 @@ python3.12 -m venv .venv - Keep existing ECP-0 through ECP-6 configurations reproducible. - Document new experiments in three stages: preregistration, sealed development log, confirmatory results. - Write new configurations, documentation, reports and user-facing output in English. -- For ECP-8, preserve the preregistered three-arm capacity isolation and do not combine it with another architecture, loss, split, or training change. +- Preserve the completed ECP-8 Batch 1 capacity isolation and its sealed confirmatory split. Any Batch 2 must keep the 16-bit arm fixed and preregister exactly one structure-induction mechanism. -The active research question is whether two surplus unallocated channel bits can make the strongest weak-structure system injective without restoring a factor-slot binding. +ECP-8 Batch 1 found that surplus capacity nearly removes collisions but does not reach injectivity or compositional validation. Do not continue a channel-size search. diff --git a/README.md b/README.md index 2100073..2ad1f4b 100644 --- a/README.md +++ b/README.md @@ -73,13 +73,15 @@ configuration was frozen and its confirmatory split was never opened. See The compact synthesis is in [`docs/ecp7-development-conclusion.md`](docs/ecp7-development-conclusion.md). -ECP-8 Batch 1 is now preregistered for sealed development. It tests whether the -strongest generic ECP-7 architecture becomes injective when its four positions -may all use the complete 16-symbol vocabulary: 16 fixed bits instead of the -14-bit lower bound. A factorized positive control and a direct 14-bit -weak-structure control use the same fresh split. No ECP-8 training or -confirmatory metric existed when the configurations, hashes and gates were -registered. See [`docs/research-design-ecp8.md`](docs/research-design-ecp8.md). +ECP-8 Batch 1 is a valid negative sealed-development result. It tested whether +the strongest generic ECP-7 architecture becomes injective when all four +positions may use the complete 16-symbol vocabulary: 16 fixed bits instead of +the 14-bit lower bound. Compared with the paired 14-bit control, mean train +exactness rose from 79.55% to 98.76%, worst-link validation from 64.84% to +73.34%, and the minimum codebook from 12,048 to 15,095 messages. The arm still +had 225–265 collisions per sender and failed both 80% validation thresholds. +The ECP-8 confirmatory split remains sealed. See +[`docs/development-log-ecp8.md`](docs/development-log-ecp8.md). ## Build on this work @@ -155,6 +157,7 @@ Do not translate those artifacts in place. New experiment configurations and all - [`config/ecp8-positive-control-development.yaml`](config/ecp8-positive-control-development.yaml) — factorized 14-bit positive control on the fresh ECP-8 split. - [`config/ecp8-control-development.yaml`](config/ecp8-control-development.yaml) — paired 14-bit weak-structure control. - [`config/ecp8-development.yaml`](config/ecp8-development.yaml) — four-global-position 16-bit intervention. +- [`evidence/ecp8-development/batch1-manifest.json`](evidence/ecp8-development/batch1-manifest.json) — compact sealed Batch 1 evidence and artifact hashes. - [`config/ecp0.yaml`](config/ecp0.yaml) — machine-readable configuration from step 1. - [`config/ecp1.yaml`](config/ecp1.yaml) — frozen configuration of the population trial. - [`config/ecp3.yaml`](config/ecp3.yaml) — frozen configuration of the first successful base model. diff --git a/docs/AI_AGENT_START.md b/docs/AI_AGENT_START.md index 20f914e..5302f6d 100644 --- a/docs/AI_AGENT_START.md +++ b/docs/AI_AGENT_START.md @@ -41,7 +41,7 @@ python3.12 -m venv .venv .venv/bin/ecp6 --config config/ecp6.yaml validate ``` -Expected baseline: 136 passing tests and split sizes `14336/1024/1024`. +Expected baseline: 138 passing tests and split sizes `14336/1024/1024`. ## 4. Reproduce ECP-6 @@ -205,6 +205,14 @@ After all three sealed runs, record the result even if it is negative. A passed development gate only permits a later freeze decision; it does not itself authorize confirmatory test access. +Batch 1 is complete and negative. The 16-bit arm reached 98.76% mean train, +74.46% mean validation, 73.34% worst-link validation and 81.05% translator +validation with 15,095–15,135 unique messages. It failed injectivity and both +80% validation gates; the confirmatory split remains sealed. Do not search +larger channels. At most one Batch 2 structure-induction intervention is +justified at the unchanged 16-bit capacity. Preregister it before training and +do not combine it with a slot binding or another objective change. + ## 7. Definition of done for any contribution - full tests pass; diff --git a/docs/development-log-ecp8.md b/docs/development-log-ecp8.md index 67d1aa5..f33f8d1 100644 --- a/docs/development-log-ecp8.md +++ b/docs/development-log-ecp8.md @@ -28,3 +28,72 @@ must be injective over all 15,360 accessible meanings, improve the paired control's minimum codebook, and preserve both mean and worst-link validation. Only these three seed-11 runs are admitted. The confirmatory test remains unauthorized regardless of a single development result. + +## Batch 1 results + +Positive-control run: `runs/ecp8-batch1-positive-control-development/20260718T200136Z-ecp8-development`
+Paired-control run: `runs/ecp8-batch1-control-development/20260718T200256Z-ecp8-development`
+Intervention run: `runs/ecp8-batch1-intervention-development/20260718T200637Z-ecp8-development`
+Test unsealed: **no** + +| Metric | Positive control | 14-bit control | 16-bit intervention | +|---|---:|---:|---:| +| Population train, mean | 100% | 79.5476% | 98.7566% | +| Population train, worst link | 100% | 78.6551% | 98.6886% | +| Population validation, mean | 100% | 70.2698% | 74.4629% | +| Population validation, worst link | 100% | 64.8438% | 73.3398% | +| Universal translator, validation | 100% | 75.1953% | 81.0547% | +| Exact sender-message agreement | 100% | 93.74% | 77.56% | +| Unique messages per sender | 15,360 | 12,048–12,255 | 15,095–15,135 | +| Collision meanings per sender | 0 | 3,105–3,312 | 225–265 | +| Message entropy | 13.91 bits | 13.45–13.48 bits | 13.87–13.88 bits | +| Declared payload | 14 bits | 14 bits | 16 bits | +| Registered development gate | pass | fail | **fail** | + +The positive control passes every validity requirement at 100%, proving that +the fresh split remains fully solvable under the factorized 14-bit protocol. +The paired weak-structure control selects step 7,400 and stops at the registered +15,000-step minimum. The 16-bit intervention keeps improving, selects step +16,200 and stops at step 21,200. + +The extra two bits have a large isolated effect. Mean train exactness improves +by 19.21 percentage points and worst-link train by 20.03 points. Mean +validation improves by 4.19 points, worst-link validation by 8.50 points and +translator validation by 5.86 points. The minimum sender codebook gains 3,047 +occupied messages and the final all-link hard-error replay pool falls from +1,953 to 119. + +Capacity is nevertheless insufficient. The intervention retains 225–265 hard +collisions per sender, so none is injective. Mean and worst-link validation are +74.46% and 73.34%, below their registered 80% thresholds. The intervention +passes both train thresholds, the translator threshold and both paired +non-regression checks, but fails injectivity and both validation thresholds. + +The error pattern also distinguishes memorization from learned compositional +structure. The 16-bit arm reaches near-perfect training and perfect texture +validation, while held-out color and shape remain at 94.66% and 79.77% factor +accuracy. Sender agreement falls from 93.74% to 77.56%, and sender topographic +correlations fall from 0.761–0.771 to 0.693–0.700. The surplus code space lets +independent senders separate most training meanings, but it does not induce a +shared compositional organization for the withheld relation. + +All three post-hoc analyses verify 65 artifact hashes, 16,384 validation-only +episodes, absent confirmatory-test metrics and valid hard messages. The global +channel audit checks every intervention message for four symbols in `0..15` +and exactly 16 declared bits. + +## Batch 1 decision + +ECP8-B1-I is a valid negative development result. Two surplus unallocated bits +substantially reduce the hard-code bottleneck and improve every joint accuracy +metric, but they do not produce an injective or sufficiently compositional +protocol. The pure-capacity hypothesis is rejected, and the ECP-8 confirmatory +split remains sealed. + +Do not continue with an 18-bit, 20-bit or longer fixed message search. A Batch +2, if attempted, must keep the 16-bit channel and Batch 1 training fixed while +preregistering exactly one structure-induction mechanism. The most direct +candidate is the previously defined context-invariant transition objective, +because Batch 1 now supplies the code capacity that was missing when that +objective collapsed in ECP7-B8. It must be tested without a slot binding or +additional simultaneous change. diff --git a/docs/research-design-ecp8.md b/docs/research-design-ecp8.md index f7f9eda..1d1b1e3 100644 --- a/docs/research-design-ecp8.md +++ b/docs/research-design-ecp8.md @@ -1,6 +1,6 @@ # Research Design ECP-8 — Surplus Capacity Without Factor Slots -Status: **preregistered for sealed development**
+Status: **sealed development active; Batch 1 valid negative**
Preregistered: July 18, 2026 at 19:58:58 UTC
Positive-control configuration: `config/ecp8-positive-control-development.yaml`
Paired-control configuration: `config/ecp8-control-development.yaml`
@@ -92,3 +92,19 @@ capacity search. | 16-bit intervention | `f6219110ebfe8f26248d9dbd646d80e1033f63d618a2524f7b96668ebd7fdaea` | `0572857aa786e0a41995aedd1cee8c2bb6dbdbc47f46025411bb8c06615b9693` | These identities and thresholds were recorded before any ECP-8 training run. + +## Batch 1 outcome + +The factorized positive control passed at 100%. Relative to the direct 14-bit +weak-structure control, the 16-bit intervention increased mean train exactness +from 79.55% to 98.76%, mean validation from 70.27% to 74.46%, worst-link +validation from 64.84% to 73.34% and the minimum sender codebook from 12,048 to +15,095. Universal-translator validation increased from 75.20% to 81.05%. + +The intervention still used only 15,095–15,135 unique messages for 15,360 +accessible meanings and did not reach either 80% validation threshold. It +therefore failed the registered injectivity and validation gates. This is a +valid negative sealed-development result; no confirmatory access is authorized. +See `docs/development-log-ecp8.md` and +`evidence/ecp8-development/batch1-manifest.json` for the full decision and +compact artifact identities. diff --git a/evidence/ecp8-development/README.md b/evidence/ecp8-development/README.md new file mode 100644 index 0000000..79cdbef --- /dev/null +++ b/evidence/ecp8-development/README.md @@ -0,0 +1,20 @@ +# ECP-8 sealed development evidence + +ECP-8 Batch 1 tests whether two surplus, semantically unallocated channel bits +solve the hard-code bottleneck of the strongest weak-structure ECP-7 system. +The factorized 14-bit positive control, weak-structure 14-bit paired control and +weak-structure 16-bit intervention use one fresh deterministic split and seed +11. The confirmatory split was never opened. + +The positive control is perfect. The 16-bit intervention raises mean train +exactness from 79.55% to 98.76%, mean validation from 70.27% to 74.46% and +worst-link validation from 64.84% to 73.34%. It also raises the minimum sender +codebook from 12,048 to 15,095 of 15,360 accessible meanings. However, every +sender still has 225–265 hard-code collisions and validation remains below the +registered 80% mean and worst-link thresholds. Batch 1 is therefore a valid +negative development result. + +`batch1-manifest.json` records the preregistration commit, configuration and +split identities, compact metrics, gate checks and hashes of the local run +artifacts. The large checkpoints, message matrices and episode logs remain +under ignored local `runs/` directories. diff --git a/evidence/ecp8-development/batch1-manifest.json b/evidence/ecp8-development/batch1-manifest.json new file mode 100644 index 0000000..b3791e0 --- /dev/null +++ b/evidence/ecp8-development/batch1-manifest.json @@ -0,0 +1,114 @@ +{ + "batch": "ECP8-B1", + "confirmatory_test_unsealed": false, + "configurations": { + "intervention": { + "effective_sha256": "0572857aa786e0a41995aedd1cee8c2bb6dbdbc47f46025411bb8c06615b9693", + "path": "config/ecp8-development.yaml", + "raw_sha256": "f6219110ebfe8f26248d9dbd646d80e1033f63d618a2524f7b96668ebd7fdaea" + }, + "paired_control": { + "effective_sha256": "fcaf5e8f39df2349cb3dedefcc233e5a08d813763a5b644d6a2ed2c050748f06", + "path": "config/ecp8-control-development.yaml", + "raw_sha256": "9e3c865fcec9698bb95fcfc3ad429a354e26ffd39f2533dd7dc836dfdd3ab34a" + }, + "positive_control": { + "effective_sha256": "3a8e55ee262418f1005405afec3eeee7ebaaad164cb83274bd95c95e2c0939c3", + "path": "config/ecp8-positive-control-development.yaml", + "raw_sha256": "e996bf66c30b3e699b27046e9824c13c7ecf3f2cb9831c4b9f0528d17b7b5f05" + } + }, + "intervention": { + "artifact_hashes": { + "effective_config.yaml": "86cce5f1f679b6cbfd781f015c58283d50c42511dbd26ddb7c0732f8cc63c638", + "posthoc-analysis.json": "fd73d82668a43bdcf5ce750e79d40d004a721530445d0143e39d1235699144e4", + "seed-11/metrics.json": "dc654f204698f196ed19828bac77c866ef9b93492d16639163c404b1dd5bec5f", + "summary.json": "9f2712e7599daaddfe506a43b57d295f423fe92146d018f72c4aba7ce5b15df5" + }, + "best_step": 16200, + "channel_audit_applicable": true, + "channel_bits": 16, + "collision_meanings_per_sender": [265, 241, 225, 237], + "development_gate_passed": false, + "final_step": 21200, + "message_entropy_bits_per_sender": [13.872139655666405, 13.875510387275181, 13.877544574286759, 13.875294025782184], + "population_train_mean_exact_match": 0.9875662667410714, + "population_train_worst_link_exact_match": 0.9868861607142857, + "population_validation_factor_accuracy": [0.94659423828125, 0.79766845703125, 0.9976806640625, 1.0], + "population_validation_mean_exact_match": 0.74462890625, + "population_validation_worst_link_exact_match": 0.7333984375, + "run_id": "20260718T200637Z-ecp8-development", + "sender_message_agreement": 0.7755642361111111, + "universal_translator_validation_exact_match": 0.810546875, + "unique_messages_per_sender": [15095, 15119, 15135, 15123] + }, + "paired_control": { + "artifact_hashes": { + "effective_config.yaml": "c25d3fc2ce87917e98c57d6bff3588dabdcee98cc4e1f06b44a0f944204fb815", + "posthoc-analysis.json": "1a03b75fc568abdddb1513e1514da7cfabf01b4ba4e6b37341ef10d5995b3b80", + "seed-11/metrics.json": "1545cbf6df5419e5c1dbe54d2e0d70fbf0bba587613a2ab64571ffd52eb4e8e1", + "summary.json": "e6de108c96d285730710cf6cb6f86a2ca629d036de5335e3cca845b09e0d6df3" + }, + "best_step": 7400, + "channel_audit_applicable": true, + "channel_bits": 14, + "collision_meanings_per_sender": [3162, 3105, 3122, 3312], + "development_gate_passed": false, + "final_step": 15000, + "message_entropy_bits_per_sender": [13.473864279689561, 13.482229528712717, 13.480163426011318, 13.453976865953427], + "population_train_mean_exact_match": 0.7954755510602678, + "population_train_worst_link_exact_match": 0.7865513392857143, + "population_validation_factor_accuracy": [0.9853515625, 0.85198974609375, 0.96044921875, 0.8804931640625], + "population_validation_mean_exact_match": 0.70269775390625, + "population_validation_worst_link_exact_match": 0.6484375, + "run_id": "20260718T200256Z-ecp8-development", + "sender_message_agreement": 0.9374023437500001, + "universal_translator_validation_exact_match": 0.751953125, + "unique_messages_per_sender": [12198, 12255, 12238, 12048] + }, + "paired_effect": { + "minimum_unique_message_gain": 3047, + "population_train_mean_difference": 0.1920907156808036, + "population_train_worst_link_difference": 0.2003348214285714, + "population_validation_mean_difference": 0.04193115234375, + "population_validation_worst_link_difference": 0.0849609375, + "sender_message_agreement_difference": -0.161838107638889, + "universal_translator_validation_difference": 0.05859375 + }, + "positive_control": { + "artifact_hashes": { + "effective_config.yaml": "ef6a5c4715e81f974bc3d428388f0dee39ac5b9ddee74a3c3eafda7fb94f94ff", + "posthoc-analysis.json": "8e8417e31b3ce13fa71a26788509c80845538d13f6c29489c106a65932166e54", + "seed-11/metrics.json": "7fdd6bfeb93cb3f9623c49f536dc4504b8b6232aee7f0f25c411f12a4e048bc2", + "summary.json": "d79ebad4528e51fc91f928081d0ec15dee9e53e7f3097e82faad9c20ed40c66f" + }, + "channel_audit_applicable": true, + "channel_bits": 14, + "development_gate_passed": true, + "population_train_mean_exact_match": 1.0, + "population_validation_mean_exact_match": 1.0, + "population_validation_worst_link_exact_match": 1.0, + "run_id": "20260718T200136Z-ecp8-development", + "universal_translator_validation_exact_match": 1.0, + "unique_messages_per_sender": [15360, 15360, 15360, 15360] + }, + "preregistration_commit": "c63e5e32622fc56753cea1513d9f0f7f2fe31fb3", + "registered_intervention_gate": { + "all_artifact_hashes_match": true, + "all_channel_messages_valid": true, + "all_senders_injective": false, + "all_test_metrics_absent": true, + "all_validation_only_episodes": true, + "minimum_codebook_improved_over_control": true, + "population_train_mean_at_least_0_97": true, + "population_train_worst_link_at_least_0_95": true, + "population_validation_mean_at_least_0_80": false, + "population_validation_mean_preserved_over_control": true, + "population_validation_worst_link_at_least_0_80": false, + "population_validation_worst_link_preserved_over_control": true, + "universal_translator_validation_at_least_0_70": true + }, + "seed": 11, + "split_artifact_sha256": "b686b8d705a2387f9681c4d409c85c83bf437a361b62e551977001ceae8de506", + "split_sha256": "241acd2ee42ff658ddf0c8c897d4e4df0f52a18098b3ef68c009c74c2134592a" +} diff --git a/src/ai_taal/analysis.py b/src/ai_taal/analysis.py index 8256330..3e4dee9 100644 --- a/src/ai_taal/analysis.py +++ b/src/ai_taal/analysis.py @@ -162,7 +162,7 @@ def _analyze_population_development_run( ) for sender_id, (semantic, messages) in sender_matrices.items() } - channel_audit = _factor_local_channel_audit( + channel_audit = _discrete_channel_audit( episode_path, metrics, config ) protocols = metrics["sender_protocols"] @@ -481,7 +481,7 @@ def _analyze_population_run( ) for sender_id, (semantic, messages) in sender_matrices.items() } - factor_local_audit = _factor_local_channel_audit( + factor_local_audit = _discrete_channel_audit( metrics_path.parent / "episodes.jsonl", metrics, config ) rows.append( @@ -680,11 +680,12 @@ def _contains_mapping_key(value: Any, key: str) -> bool: return False -def _factor_local_channel_audit( +def _discrete_channel_audit( path: Path, metrics: dict[str, Any], config: dict[str, Any] ) -> dict[str, Any]: channel = config["channel"] if channel["type"] not in { + "discrete_fixed_length", "factor_local_discrete_fixed_length", "slot_local_discrete_fixed_length", }: @@ -693,10 +694,20 @@ def _factor_local_channel_audit( factor_sizes = channel["factor_alphabet_sizes"] bindings = metrics["sender_binding_diagnostics"]["hard_factor_by_slot"] alphabet_sizes_by_slot = None - else: + elif channel["type"] == "slot_local_discrete_fixed_length": factor_sizes = None bindings = None alphabet_sizes_by_slot = channel["slot_alphabet_sizes"] + else: + factor_sizes = None + bindings = None + alphabet_sizes_by_slot = [channel["vocabulary_size"]] * channel[ + "message_length" + ] + message_length = channel.get( + "message_length", + len(factor_sizes or alphabet_sizes_by_slot), + ) violations = [] checked = 0 with path.open("r", encoding="utf-8") as handle: @@ -710,7 +721,12 @@ def _factor_local_channel_audit( violations.append( {"line": line_number, "reason": "incorrect_channel_bits"} ) - for slot_index, symbol in enumerate(message["symbols"]): + symbols = message["symbols"] + if len(symbols) != message_length: + violations.append( + {"line": line_number, "reason": "incorrect_message_length"} + ) + for slot_index, symbol in enumerate(symbols[:message_length]): factor_index = ( factor_by_slot[slot_index] if factor_by_slot is not None diff --git a/tests/test_analysis.py b/tests/test_analysis.py index 0bb9e17..e71f2af 100644 --- a/tests/test_analysis.py +++ b/tests/test_analysis.py @@ -1,7 +1,7 @@ import json from ai_taal.analysis import ( - _factor_local_channel_audit, + _discrete_channel_audit, _paired_sign_flip_test, _population_episode_matrices, ) @@ -66,7 +66,7 @@ def test_slot_local_channel_audit_checks_each_position_capacity(tmp_path): } } - audit = _factor_local_channel_audit(path, {}, config) + audit = _discrete_channel_audit(path, {}, config) assert audit["applicable"] is True assert audit["valid"] is True @@ -91,7 +91,62 @@ def test_slot_local_channel_audit_rejects_a_symbol_outside_its_slot(tmp_path): } } - audit = _factor_local_channel_audit(path, {}, config) + audit = _discrete_channel_audit(path, {}, config) assert audit["valid"] is False assert audit["violations"][0]["slot"] == 2 + + +def test_global_fixed_length_channel_audit_checks_full_vocabulary(tmp_path): + episode = { + "message": { + "sender_id": "sender-0", + "channel_bits": 16, + "symbols": [15, 8, 15, 8], + } + } + path = tmp_path / "episodes.jsonl" + path.write_text(json.dumps(episode) + "\n", encoding="utf-8") + config = { + "channel": { + "type": "discrete_fixed_length", + "bits_per_message": 16, + "message_length": 4, + "vocabulary_size": 16, + } + } + + audit = _discrete_channel_audit(path, {}, config) + + assert audit["applicable"] is True + assert audit["valid"] is True + assert audit["slot_alphabet_sizes"] == [16, 16, 16, 16] + + +def test_global_fixed_length_channel_audit_rejects_invalid_message(tmp_path): + episode = { + "message": { + "sender_id": "sender-0", + "channel_bits": 15, + "symbols": [0, 16, 2], + } + } + path = tmp_path / "episodes.jsonl" + path.write_text(json.dumps(episode) + "\n", encoding="utf-8") + config = { + "channel": { + "type": "discrete_fixed_length", + "bits_per_message": 16, + "message_length": 4, + "vocabulary_size": 16, + } + } + + audit = _discrete_channel_audit(path, {}, config) + + assert audit["valid"] is False + assert {violation["reason"] for violation in audit["violations"]} == { + "incorrect_channel_bits", + "incorrect_message_length", + "symbol_outside_local_alphabet", + }