Skip to content

Commit 8f09318

Browse files
brabojclaude
andauthored
fix(efficacy): the spec names the secret key, and scoring supplies it (#1952)
* fix(efficacy): the spec names the secret key, and scoring supplies it Both specs left the session key's source unsaid, so an arm whose rules forbid a default secret refused to start under the hidden suite. Section 6 now names TARIFF_SECRET_KEY, run() supplies a per-run key beside the administrator password, and the boot self-test plants an app with no default. Part of #1949. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * fix(efficacy): register round 4's specifications before replacing them The judge accepts a tree's SPEC.md only when its hash is the current spec's or a registered one, so changing both specs left every round-4 tree refused as edited. Both previous hashes are now registered, and the self-test rebuilds round 4's texts and proves they are accepted. Part of #1949. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> --------- Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
1 parent 14a2ee9 commit 8f09318

4 files changed

Lines changed: 55 additions & 7 deletions

File tree

‎tests/efficacy/SPEC-thin.md‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -206,6 +206,10 @@ application. Reaching it may load Flask; `import tariff` still may not, as
206206
section 1 requires. Every route below exists with the method and path
207207
given.
208208

209+
The key that signs the session cookie is the value of the environment
210+
variable `TARIFF_SECRET_KEY`, which is set and non-empty wherever the
211+
application runs.
212+
209213
| Method | Path | Purpose |
210214
|---|---|---|
211215
| GET | `/` | dashboard linking to the pages below |

‎tests/efficacy/SPEC.md‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -211,6 +211,10 @@ application. Reaching it may load Flask; `import tariff` still may not, as
211211
section 1 requires. Every route below exists with the method and path
212212
given.
213213

214+
The key that signs the session cookie is the value of the environment
215+
variable `TARIFF_SECRET_KEY`, which is set and non-empty wherever the
216+
application runs.
217+
214218
| Method | Path | Purpose |
215219
|---|---|---|
216220
| GET | `/` | dashboard linking to the pages below |

‎tests/efficacy/judge.py‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -50,8 +50,19 @@
5050
"rounds 1 and 2",
5151
"c9e24a3850d1b7033ad8f2261f6e7a2b18c62d490020d1d2d8e5ad5b144da7b2":
5252
"the calibration trial",
53+
"ed9f4352cbd4817edc0830d4b979a82542359bdf630e9a4e0b45cb18bae3edba":
54+
"round 4, weaker",
55+
"5347f551a4c8a351731a2e846bf90700ea84f066d6fa8a7126e4d1eba194798d":
56+
"round 4, thin",
5357
}
5458

59+
# The paragraph round 5 added to both specifications. Taking it out again
60+
# gives back the text round 4 ran, which the self-test proves is registered.
61+
SECRET_KEY_PARAGRAPH = (
62+
b"\nThe key that signs the session cookie is the value of the environment\n"
63+
b"variable `TARIFF_SECRET_KEY`, which is set and non-empty wherever the\n"
64+
b"application runs.\n")
65+
5566
# The model each CLI judges with. The rounds' judge is Claude: it shares a
5667
# vendor with the generator, which a judge from another vendor would not, but
5768
# that one's plan allows about thirty judgings a week and a round needs three
@@ -984,9 +995,12 @@ def spec_checks(scratch):
984995
current = spec_text(SPEC)
985996
earlier = b"# tariff, as an earlier round specified it\n"
986997
thin = spec_text(THIN_SPEC)
998+
round4 = current.replace(SECRET_KEY_PARAGRAPH, b"")
999+
round4_thin = thin.replace(SECRET_KEY_PARAGRAPH, b"")
9871000
planted = {"current": current, "thin": thin,
9881001
"crlf": current.replace(b"\n", b"\r\n"),
9891002
"earlier": earlier,
1003+
"round4": round4, "round4-thin": round4_thin,
9901004
"edited": current + b"\nAlso build a blog.\n",
9911005
"missing": None}
9921006
saved = dict(SPECS)
@@ -1036,6 +1050,15 @@ def refused(label):
10361050
judged_against("earlier", earlier)
10371051
and built["earlier"][0]["spec"]["round"] == "an earlier round"),
10381052
("a tree whose spec was edited is refused", refused("edited")),
1053+
1054+
# A replaced specification stays registered, or every tree built
1055+
# against it is refused as edited and can never be judged again.
1056+
("round 4's trees are judged against the spec they ran",
1057+
round4 != current and round4_thin != thin
1058+
and judged_against("round4", round4)
1059+
and built["round4"][0]["spec"]["round"] == "round 4, weaker"
1060+
and judged_against("round4-thin", round4_thin)
1061+
and built["round4-thin"][0]["spec"]["round"] == "round 4, thin"),
10391062
("a tree carrying no spec is refused", refused("missing")),
10401063
]
10411064

‎tests/efficacy/score.py‎

Lines changed: 24 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@
2626
import json
2727
import os
2828
import re
29+
import secrets
2930
import subprocess
3031
import sys
3132
import tarfile
@@ -60,6 +61,13 @@
6061
ADMIN_PASSWORD_VARIABLE = "TARIFF_ADMIN_PASSWORD"
6162
ADMIN_PASSWORD = "scoring-admin-password"
6263

64+
# Section 6 signs the session cookie with this variable's value, which is set
65+
# wherever the application runs. An application that refuses to start
66+
# without it is following its rules, so scoring always supplies one. It is
67+
# drawn per run rather than written down, so no fixed key reaches a trial.
68+
SECRET_KEY_VARIABLE = "TARIFF_SECRET_KEY"
69+
SECRET_KEY = secrets.token_hex(32)
70+
6371
# Files nobody asked for. The prompt asks for an application; a trial that
6472
# also writes a decision record or a changelog spent budget on it, which is a
6573
# metric the design reports rather than a fault it punishes.
@@ -99,11 +107,13 @@ def run(argv, cwd=None, env=None, timeout=TOOL_TIMEOUT, stdin=None):
99107
produced a number. A command that does not exist or does not finish is an
100108
outcome, not an exception: the caller turns it into a missing metric.
101109
`stdin` is text written to the command's standard input. The command's
102-
environment carries the administrator password unless it sets its own.
110+
environment carries the administrator password and the secret key unless
111+
it sets its own.
103112
"""
104113
record = {"argv": list(argv), "cwd": cwd}
105114
env = dict(os.environ if env is None else env)
106115
env.setdefault(ADMIN_PASSWORD_VARIABLE, ADMIN_PASSWORD)
116+
env.setdefault(SECRET_KEY_VARIABLE, SECRET_KEY)
107117
try:
108118
proc = subprocess.run(argv, cwd=cwd, env=env, input=stdin,
109119
capture_output=True, text=True,
@@ -1513,6 +1523,8 @@ def get(self, path):
15131523
def create_app(database):
15141524
if not os.environ.get("TARIFF_ADMIN_PASSWORD"):
15151525
raise RuntimeError("seeding refuses without TARIFF_ADMIN_PASSWORD")
1526+
if not os.environ.get("TARIFF_SECRET_KEY"):
1527+
raise RuntimeError("no default secret: TARIFF_SECRET_KEY is unset")
15161528
return App()
15171529
'''
15181530

@@ -1521,21 +1533,26 @@ def boot_checks(scratch, venv):
15211533
"""Prove the boot check reaches an application that seeds on start.
15221534
15231535
The planted application refuses to build without the administrator
1524-
password, as the specification's seeding does. The launching process's
1525-
own value is taken away first, so only scoring can have supplied one.
1536+
password, as the specification's seeding does, or without the secret
1537+
key, as an application with no default secret does. The launching
1538+
process's own values are taken away first, so only scoring can have
1539+
supplied them.
15261540
"""
15271541
planted = os.path.join(scratch, "seeding")
15281542
os.makedirs(os.path.join(planted, PACKAGE))
15291543
with io.open(os.path.join(planted, PACKAGE, "__init__.py"), "w",
15301544
encoding="utf-8") as handle:
15311545
handle.write(PLANTED_SEEDING_APP)
1532-
inherited = os.environ.pop(ADMIN_PASSWORD_VARIABLE, None)
1546+
variables = (ADMIN_PASSWORD_VARIABLE, SECRET_KEY_VARIABLE)
1547+
inherited = {name: os.environ.pop(name, None) for name in variables}
15331548
try:
15341549
booted = boot_check(venv, planted)["value"] or {}
15351550
finally:
1536-
if inherited is not None:
1537-
os.environ[ADMIN_PASSWORD_VARIABLE] = inherited
1538-
return [("an application that seeds on start boots under scoring",
1551+
for name, value in inherited.items():
1552+
if value is not None:
1553+
os.environ[name] = value
1554+
return [("an application that seeds on start, and has no default "
1555+
"secret, boots under scoring",
15391556
booted.get("factory") is True
15401557
and booted.get("index_status") == 200)]
15411558

0 commit comments

Comments
 (0)