From f5cea8184ce590aef3fb1e62021a5c27051486fc Mon Sep 17 00:00:00 2001 From: Timothy Wayne Gregg Date: Fri, 13 Nov 2026 22:57:13 -0500 Subject: [PATCH 01/19] feat(worker): pin immutable external web profile identity --- integrations/profiles/external-web-v1.json | 1 + integrations/worker/profiles.py | 109 +++++++++++++++++++++ test_external_profiles.py | 102 +++++++++++++++++++ 3 files changed, 212 insertions(+) create mode 100644 integrations/profiles/external-web-v1.json create mode 100644 integrations/worker/profiles.py create mode 100644 test_external_profiles.py diff --git a/integrations/profiles/external-web-v1.json b/integrations/profiles/external-web-v1.json new file mode 100644 index 0000000..aca80ae --- /dev/null +++ b/integrations/profiles/external-web-v1.json @@ -0,0 +1 @@ +{"allowed_paths":["/","/robots.txt"],"allowed_services":[{"port":80,"scheme":"http"},{"port":443,"scheme":"https"}],"checks":[{"check_id":"robots_policy","expected_data":"Public robots permission for the selected service root.","max_body_bytes":262144,"max_header_bytes":32768,"method":"GET","path":"/robots.txt","reporting":"Permission outcome and bounded status only; no raw robots content.","request_mode":"acquire","sensitivity":"Stop on possible sensitive-data or compromise indicators; discard the body.","timeout_seconds":10},{"check_id":"public_header_triage","expected_data":"Public homepage headers and already-returned HTML markers, classified by the existing Checker.","max_body_bytes":262144,"max_header_bytes":32768,"method":"GET","path":"/","reporting":"Reported PHP and WordPress version evidence remains uncertain; exploitability is not tested. No title, asset URLs or HTML are retained.","request_mode":"acquire","sensitivity":"Stop on possible sensitive-data or compromise indicators before inspection; discard the body.","timeout_seconds":10},{"check_id":"security_header_presence","expected_data":"Presence of HSTS, CSP, frame-control and content-type options in public root response headers.","max_body_bytes":0,"max_header_bytes":32768,"method":"HEAD","path":"/","reporting":"Presence observations only, not a conclusion that configuration is secure.","request_mode":"acquire_after_robots_permission","sensitivity":"Inspect headers for stop indicators; never read or retain a HEAD response body.","timeout_seconds":10},{"check_id":"tls_certificate","expected_data":"Verified certificate hostname/chain, negotiated protocol and certificate expiry from one TLS-only connection.","max_body_bytes":0,"max_header_bytes":0,"method":"TLS","path":null,"reporting":"Connection verification and public expiry metadata only; no cipher sweep or security certification.","request_mode":"selected_https_service_only","sensitivity":"No application data, raw certificate or session keys are retained.","timeout_seconds":10}],"coverage_statement":"Bounded public external observations only; no exploitability test, compromise diagnosis or clean bill of health.","forbidden_operations":["application_write_methods","credentials_or_forms","exploit_payloads","secret_file_retrieval","suspected_shell_paths","executable_debug_probes","arbitrary_paths_or_options","access_restriction_retries","unapproved_redirect_origins","concurrent_target_connections"],"limits":{"active_seconds":600,"aggregate_body_bytes":5242880,"concurrent_connections":1,"http_transactions":100,"min_interval_ms":1000,"redirect_hops":3,"request_timeout_seconds":10,"response_body_bytes":262144,"response_header_bytes":32768,"tls_only_checks":20,"transient_retries_per_check":2},"manual_remainder":[{"check_id":"runtime_and_backport_confirmation","description":"Provider/runtime evidence and backport confirmation require reviewed corroboration; advertised banners are insufficient.","scope":"manual_review","status":"not_assessed"},{"check_id":"internal_configuration_review","description":"Internal configuration, authenticated access and filesystem inspection are not authorized by a website hostname.","scope":"separate_authorization_required","status":"not_assessed"},{"check_id":"compromise_investigation","description":"A possible indicator stops this profile; investigation needs separately reviewed scope and authority.","scope":"separate_authorization_required","status":"not_assessed"}],"profile_id":"external-web-v1","schema_version":1} diff --git a/integrations/worker/profiles.py b/integrations/worker/profiles.py new file mode 100644 index 0000000..c52ca03 --- /dev/null +++ b/integrations/worker/profiles.py @@ -0,0 +1,109 @@ +"""Immutable execution-profile identity; no network or activation side effects.""" + +from __future__ import annotations + +import hashlib +import json +import os +from pathlib import Path +import stat + + +PROFILE_ID = "external-web-v1" +PROFILE_SHA256 = "1f2a19c959570157ee9df788141ae1e21b8f67aba8c6ce99b768efbd42eb7a37" +MANIFEST_PATH = Path(__file__).resolve().parents[1] / "profiles" / (PROFILE_ID + ".json") +MAX_MANIFEST_BYTES = 32 * 1024 +_SERVICE_ORDER = {("http", 80): 0, ("https", 443): 1} + + +class ProfileError(ValueError): + """A fixed, log-safe refusal category, never untrusted profile contents.""" + + +def canonical_json(value) -> bytes: + return (json.dumps(value, ensure_ascii=True, sort_keys=True, separators=(",", ":"), allow_nan=False) + "\n").encode("ascii") + + +def _pairs(pairs): + value = {} + for key, item in pairs: + if key in value: + raise ProfileError("profile_duplicate_key") + value[key] = item + return value + + +def _read_manifest(path: Path) -> bytes: + descriptor = -1 + try: + if path.is_symlink(): + raise ProfileError("profile_file_invalid") + descriptor = os.open(path, os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0)) + before = os.fstat(descriptor) + if not stat.S_ISREG(before.st_mode) or before.st_nlink != 1 or not 0 < before.st_size <= MAX_MANIFEST_BYTES: + raise ProfileError("profile_file_invalid") + raw = os.read(descriptor, MAX_MANIFEST_BYTES + 1) + after = os.fstat(descriptor) + if (before.st_dev, before.st_ino, before.st_size, before.st_mtime_ns, before.st_ctime_ns) != ( + after.st_dev, after.st_ino, after.st_size, after.st_mtime_ns, after.st_ctime_ns + ) or len(raw) != before.st_size: + raise ProfileError("profile_file_changed") + return raw + except OSError as exc: + raise ProfileError("profile_unavailable") from exc + finally: + if descriptor >= 0: + os.close(descriptor) + + +def profile_manifest(profile_id: str = PROFILE_ID, *, path: Path | None = None) -> dict: + """Return a fresh copy of the one pinned manifest, never mutable configuration. + + The optional path supports installation layouts and offline tests. It does not + permit another manifest: exact bytes must match this release's pinned digest. + A changed behavior requires a new profile version and renewed acceptance. + """ + if type(profile_id) is not str or profile_id != PROFILE_ID: + raise ProfileError("profile_unknown") + raw = _read_manifest(MANIFEST_PATH if path is None else Path(path)) + if hashlib.sha256(raw).hexdigest() != PROFILE_SHA256: + raise ProfileError("profile_digest_mismatch") + try: + value = json.loads(raw.decode("ascii"), object_pairs_hook=_pairs, + parse_constant=lambda _value: (_ for _ in ()).throw(ProfileError("profile_nonfinite"))) + if canonical_json(value) != raw or type(value) is not dict or value.get("schema_version") != 1 or value.get("profile_id") != PROFILE_ID: + raise ProfileError("profile_schema_invalid") + except (UnicodeError, json.JSONDecodeError, TypeError, ValueError) as exc: + if isinstance(exc, ProfileError): + raise + raise ProfileError("profile_schema_invalid") from exc + return value + + +def profile_digest(profile_id: str = PROFILE_ID) -> str: + profile_manifest(profile_id) + return PROFILE_SHA256 + + +def validate_services(value) -> tuple[tuple[str, int], ...]: + """Require the exact selected service list, without adding HTTPS or HTTP.""" + if type(value) is not list or not 1 <= len(value) <= 2: + raise ProfileError("profile_services_invalid") + result = [] + for service in value: + if type(service) is not dict or set(service) != {"scheme", "port"}: + raise ProfileError("profile_services_invalid") + scheme, port = service["scheme"], service["port"] + if type(scheme) is not str or type(port) is not int or (scheme, port) not in _SERVICE_ORDER: + raise ProfileError("profile_services_invalid") + result.append((scheme, port)) + if len(set(result)) != len(result) or result != sorted(result, key=_SERVICE_ORDER.__getitem__): + raise ProfileError("profile_services_invalid") + return tuple(result) + + +def validate_binding(profile_id, digest, services) -> tuple[tuple[str, int], ...]: + """Validate a frozen claim/acceptance binding before any target connection.""" + if type(digest) is not str or digest != profile_digest(profile_id): + raise ProfileError("profile_binding_mismatch") + return validate_services(services) diff --git a/test_external_profiles.py b/test_external_profiles.py new file mode 100644 index 0000000..412fe5e --- /dev/null +++ b/test_external_profiles.py @@ -0,0 +1,102 @@ +"""Local-only identity and scope refusal regressions for the worker profile.""" + +import hashlib +import json +import os +from pathlib import Path +import tempfile +import unittest +from unittest import mock + +from integrations.worker import profiles as P + + +class ProfileTests(unittest.TestCase): + def test_pinned_bytes_and_canonical_identity(self): + manifest = P.profile_manifest() + self.assertEqual(P.canonical_json(manifest), P.MANIFEST_PATH.read_bytes()) + self.assertEqual(hashlib.sha256(P.canonical_json(manifest)).hexdigest(), P.profile_digest()) + self.assertEqual(manifest["limits"]["aggregate_body_bytes"], 5 * 1024 * 1024) + self.assertEqual(manifest["limits"]["response_header_bytes"], 32768) + self.assertEqual({c["method"] for c in manifest["checks"]}, {"GET", "HEAD", "TLS"}) + + def test_returned_object_cannot_change_future_identity(self): + manifest = P.profile_manifest() + manifest["limits"]["http_transactions"] = 100000 + self.assertEqual(P.profile_manifest()["limits"]["http_transactions"], 100) + + def test_unknown_or_non_scalar_profile_refused(self): + for value in ("r2-header-v1", "external-web-v2", "", None, True, [P.PROFILE_ID]): + with self.subTest(value=value), self.assertRaises(P.ProfileError): + P.profile_manifest(value) + + def test_missing_manifest_is_not_a_fallback(self): + with tempfile.TemporaryDirectory() as directory, self.assertRaises(P.ProfileError): + P.profile_manifest(path=Path(directory) / "absent") + + def test_changed_schema_limits_or_whitespace_refused(self): + for change in ("schema", "limits", "extra", "space"): + with self.subTest(change=change), tempfile.TemporaryDirectory() as directory: + value = P.profile_manifest() + if change == "schema": value["schema_version"] = 2 + if change == "limits": value["limits"]["http_transactions"] = 101 + if change == "extra": value["unknown"] = True + raw = P.canonical_json(value) if change != "space" else json.dumps(value).encode() + path = Path(directory) / "manifest.json" + path.write_bytes(raw) + with self.assertRaises(P.ProfileError): P.profile_manifest(path=path) + + def test_symlink_or_hardlink_refused(self): + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) / "manifest.json" + path.write_bytes(P.MANIFEST_PATH.read_bytes()) + hard = Path(directory) / "hard.json" + os.link(path, hard) + with self.assertRaises(P.ProfileError): P.profile_manifest(path=hard) + hard.unlink() + if hasattr(os, "O_NOFOLLOW"): + link = Path(directory) / "link.json" + link.symlink_to(path) + with self.assertRaises(P.ProfileError): P.profile_manifest(path=link) + + def test_empty_or_oversize_source_refused(self): + for raw in (b"", b"x" * (P.MAX_MANIFEST_BYTES + 1)): + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) / "manifest.json" + path.write_bytes(raw) + with self.assertRaises(P.ProfileError): P.profile_manifest(path=path) + + def test_change_during_read_refused(self): + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) / "manifest.json" + path.write_bytes(P.MANIFEST_PATH.read_bytes()) + original_read = os.read + def change(descriptor, size): + raw = original_read(descriptor, size) + path.write_bytes(raw + b" ") + return raw + with mock.patch.object(P.os, "read", side_effect=change), self.assertRaises(P.ProfileError): + P.profile_manifest(path=path) + + def test_exact_selected_services_are_not_expanded(self): + https = [{"scheme": "https", "port": 443}] + self.assertEqual(P.validate_binding(P.PROFILE_ID, P.PROFILE_SHA256, https), (("https", 443),)) + http = [{"scheme": "http", "port": 80}] + self.assertEqual(P.validate_binding(P.PROFILE_ID, P.PROFILE_SHA256, http), (("http", 80),)) + + def test_noncanonical_or_unapproved_services_refused(self): + http, https = {"scheme": "http", "port": 80}, {"scheme": "https", "port": 443} + for services in ([], [https, http], [http, http], [dict(http, hostname="other.test")], + [{"scheme": "https", "port": True}], [{"scheme": "HTTPS", "port": 443}], + [{"scheme": "https", "port": 8443}], (https,), ["https:443"]): + with self.subTest(services=services), self.assertRaises(P.ProfileError): + P.validate_services(services) + + def test_binding_digest_must_match_current_pinned_profile(self): + for digest in ("0" * 64, P.PROFILE_SHA256.upper(), None, True): + with self.subTest(digest=digest), self.assertRaises(P.ProfileError): + P.validate_binding(P.PROFILE_ID, digest, [{"scheme": "https", "port": 443}]) + + +if __name__ == "__main__": + unittest.main() From 3c513c1fa67f341ade83c3cdcd24153ae7224945 Mon Sep 17 00:00:00 2001 From: Timothy Wayne Gregg Date: Fri, 13 Nov 2026 23:03:23 -0500 Subject: [PATCH 02/19] feat(worker): bind automatic profile claims and durable callbacks --- integrations/worker/protocol.py | 30 +++++- integrations/worker/runner.py | 123 +++++++++++++++++++-- integrations/worker/scanner.py | 25 ++++- integrations/worker/service.py | 18 +++- integrations/worker/spool.py | 67 +++++++++++- test_external_worker_wire.py | 183 ++++++++++++++++++++++++++++++++ 6 files changed, 429 insertions(+), 17 deletions(-) create mode 100644 test_external_worker_wire.py diff --git a/integrations/worker/protocol.py b/integrations/worker/protocol.py index cc5c41d..b2c9e0b 100644 --- a/integrations/worker/protocol.py +++ b/integrations/worker/protocol.py @@ -15,6 +15,7 @@ from urllib.request import Request from .network import PinnedOpener, RequestOrigin +from .profiles import validate_binding API_SITE_URL = "https://complete.tech" @@ -55,6 +56,20 @@ class JobClaim: expires_at: datetime lease: str lease_until: datetime + profile_id: str = "r2-header-v1" + profile_sha256: str = "" + services: tuple[tuple[str, int], ...] = () + + +def profile_fields(claim: JobClaim) -> dict: + """Additive frozen binding; legacy messages remain byte-for-byte unchanged.""" + if claim.profile_id == "r2-header-v1": + if claim.profile_sha256 or claim.services: + raise ValueError("profile_binding_mismatch") + return {} + services = [{"scheme": scheme, "port": port} for scheme, port in claim.services] + validate_binding(claim.profile_id, claim.profile_sha256, services) + return {"profile_id": claim.profile_id, "profile_sha256": claim.profile_sha256, "services": services} @dataclass(frozen=True) @@ -103,7 +118,7 @@ def validate_claim(value, *, now: datetime | None = None) -> JobClaim | None: if data is None: return None required = {"id", "target", "scope_hash", "terms_hash", "expires_at", "lease", "lease_until"} - if not isinstance(data, dict) or set(data) != required: + if not isinstance(data, dict) or set(data) not in (required, required | {"profile_id", "profile_sha256", "services"}): raise APIError("invalid_job_schema") job_id = data["id"] if isinstance(job_id, bool) or not isinstance(job_id, int) or not 1 <= job_id <= MAX_JOB_ID: @@ -135,7 +150,16 @@ def validate_claim(value, *, now: datetime | None = None) -> JobClaim | None: raise APIError("stale_claim") if lease_until > expires_at or (lease_until - now).total_seconds() > MAX_LEASE_SECONDS: raise APIError("invalid_lease_bounds") - return JobClaim(job_id, target, scope_hash, terms_hash, expires_at, lease, lease_until) + profile_id, digest, services = "r2-header-v1", "", () + if "profile_id" in data: + try: + profile_id, digest = data["profile_id"], data["profile_sha256"] + services = validate_binding(profile_id, digest, data["services"]) + if (_scope.initial_scheme, _scope.initial_port) not in services: + raise ValueError("profile_primary_service_missing") + except (TypeError, ValueError) as exc: + raise APIError("invalid_profile_binding") from exc + return JobClaim(job_id, target, scope_hash, terms_hash, expires_at, lease, lease_until, profile_id, digest, services) def validate_control_task(value, *, now: datetime | None = None) -> ControlTask | None: @@ -225,7 +249,7 @@ def admit(self, claim: JobClaim, kind: str) -> dict: A short timeout keeps a slow store from stalling the supervisor; a timeout is a refusal. """ - if kind not in {"http", "probe", "control", "control_probe"}: + if kind not in {"http", "tls", "probe", "control", "control_probe"}: raise APIError("invalid_admission_kind") return self.post( "/ct-scan/v1/admit", diff --git a/integrations/worker/runner.py b/integrations/worker/runner.py index 098b2e2..6989b44 100644 --- a/integrations/worker/runner.py +++ b/integrations/worker/runner.py @@ -19,6 +19,8 @@ from .admission import AdmissionPump from .network import RequestOrigin +from .protocol import profile_fields +from .profiles import PROFILE_ID, validate_binding MAX_SCANNER_STDOUT = 32 * 1024 @@ -113,7 +115,7 @@ def __init__(self, *, repo_root=None, popen=subprocess.Popen, monotonic=time.mon self._popen = popen self._monotonic = monotonic - def run(self, target: str, *, timeout: float = MAX_WALL_SECONDS, stop_event=None, gate=None, task=None) -> dict: + def run(self, target: str, *, timeout: float = MAX_WALL_SECONDS, stop_event=None, gate=None, task=None, claim=None) -> dict: # Fail closed by construction: without an admission gate there is no scan. if gate is None: raise ScannerFailure("admission_guard_required") @@ -132,6 +134,13 @@ def run(self, target: str, *, timeout: float = MAX_WALL_SECONDS, stop_event=None if task not in {None, "control_dns", "control_https"}: raise ScannerFailure("invalid_scanner_task") message = {"target": normalized} if task is None else {"target": normalized, "task": task} + if claim is not None: + if task is not None or claim.target != target: + raise ScannerFailure("scanner_profile_mismatch") + try: + message.update(profile_fields(claim)) + except ValueError as exc: + raise ScannerFailure("scanner_profile_mismatch") from exc payload = json.dumps(message, separators=(",", ":")).encode("utf-8") if len(payload) > MAX_SCANNER_INPUT: raise ScannerFailure("scanner_input_too_large") @@ -170,7 +179,7 @@ def run(self, target: str, *, timeout: float = MAX_WALL_SECONDS, stop_event=None child_sock.close() raise ScannerFailure("scanner_start_failed") from exc child_sock.close() - pump = AdmissionPump(parent_sock, gate, kind="http" if task is None else "control") + pump = AdmissionPump(parent_sock, gate, kind="external" if claim is not None and claim.profile_id == PROFILE_ID else "http" if task is None else "control") try: buffers = {"stdout": bytearray(), "stderr": bytearray()} @@ -272,7 +281,7 @@ def run(self, target: str, *, timeout: float = MAX_WALL_SECONDS, stop_event=None raise ScannerFailure(child_error or "scanner_process_failed") if task is not None: return self._parse_control_success(bytes(buffers["stdout"]), task) - return self._parse_success(bytes(buffers["stdout"]), target, scope) + return self._parse_success(bytes(buffers["stdout"]), target, scope, claim=claim) finally: # Let an in-flight liveness probe finish before the socket and the API client are reused. pump.wait_probe(5.0) @@ -331,16 +340,18 @@ def _parse_control_success(cls, output: bytes, task: str): return control @classmethod - def _parse_success(cls, output: bytes, target: str, scope: RequestOrigin): + def _parse_success(cls, output: bytes, target: str, scope: RequestOrigin, *, claim=None): parsed = cls._parse_json_line(output) if set(parsed) != {"ok", "result"} or type(parsed.get("ok")) is not bool or parsed["ok"] is not True: raise ScannerFailure("scanner_invalid_output") result = parsed["result"] - cls.validate_result(result, target, scope) + cls.validate_result(result, target, scope, claim=claim) return result @staticmethod - def validate_result(value, target: str, scope: RequestOrigin): + def validate_result(value, target: str, scope: RequestOrigin, *, claim=None): + if claim is not None and claim.profile_id != "r2-header-v1": + return SubprocessRunner.validate_external_result(value, target, scope, claim) if not isinstance(value, dict) or set(value) != set(Result.__dataclass_fields__): raise ScannerFailure("scanner_result_schema") for key in ("input_url", "final_url"): @@ -440,3 +451,103 @@ def validate_result(value, target: str, scope: RequestOrigin): if serialized_size > MAX_SCANNER_STDOUT - 64: raise ScannerFailure("scanner_output_overflow") return value + + @staticmethod + def validate_external_result(value, target, scope, claim): + """Validate fixed typed observations, never arbitrary scanner-provided prose.""" + extras = {"profile_id", "profile_sha256", "services", "checks", "manual_remainder"} + core_keys = set(Result.__dataclass_fields__) + if not isinstance(value, dict) or set(value) != core_keys | extras: + raise ScannerFailure("scanner_result_schema") + try: + binding = profile_fields(claim) + if any(value[key] != binding[key] for key in ("profile_id", "profile_sha256", "services")): + raise ValueError("binding") + validate_binding(value["profile_id"], value["profile_sha256"], value["services"]) + except (ValueError, TypeError, KeyError) as exc: + raise ScannerFailure("scanner_profile_mismatch") from exc + core = {key: value[key] for key in core_keys} + # Scope validation may allow the legacy HTTP upgrade; selected services add + # a stricter restriction, so an unaccepted HTTPS service is never inferred. + SubprocessRunner.validate_result(core, target, scope) + from urllib.parse import urlsplit + final = urlsplit(core["final_url"]) + if (final.scheme, final.port or (443 if final.scheme == "https" else 80)) not in claim.services: + raise ScannerFailure("scanner_result_scope_mismatch") + if any(core[key] for key in ("server", "powered_by", "title", "evidence", "asset_version_hints", "notes")): + raise ScannerFailure("scanner_result_sensitive") + if core["php_evidence"] not in ([], ["PHP was mentioned in public response headers; raw values were discarded."]): + raise ScannerFailure("scanner_result_sensitive") + if core["wordpress_version"]: + raise ScannerFailure("scanner_result_sensitive") + if any(not re.fullmatch(r"[0-9][A-Za-z0-9.+_~:-]{0,63}", candidate) for candidate in core["php_version_candidates"]): + raise ScannerFailure("scanner_result_sensitive") + checks = value["checks"] + ids = ("robots_policy", "public_header_triage", "security_header_presence", "tls_certificate") + selected = [f"{scheme}:{port}" for scheme, port in claim.services] + expected = {(check_id, service) for service in selected for check_id in ids} + if not isinstance(checks, list) or len(checks) != len(expected): + raise ScannerFailure("scanner_result_schema") + seen = set() + for check in checks: + if not isinstance(check, dict) or set(check) != {"id", "service", "outcome", "evidence"}: + raise ScannerFailure("scanner_result_schema") + if not isinstance(check["id"], str) or not isinstance(check["service"], str): + raise ScannerFailure("scanner_result_schema") + pair = (check["id"], check["service"]) + if pair not in expected or pair in seen: + raise ScannerFailure("scanner_result_schema") + seen.add(pair) + outcome, evidence = check["outcome"], check["evidence"] + if not isinstance(outcome, str) or outcome not in {"observed", "blocked", "unknown", "not_selected"} or not isinstance(evidence, dict): + raise ScannerFailure("scanner_result_schema") + if outcome == "not_selected": + if pair != ("tls_certificate", "http:80") or evidence != {"reason": "https_not_selected"}: + raise ScannerFailure("scanner_result_schema") + continue + if pair[1] not in selected: + raise ScannerFailure("scanner_result_scope_mismatch") + if set(evidence) == {"reason", "http_status"}: + if outcome != "unknown" or evidence["reason"] != "non_successful_response" or type(evidence["http_status"]) is not int or not 100 <= evidence["http_status"] <= 599: + raise ScannerFailure("scanner_result_schema") + continue + if set(evidence) == {"reason"}: + if outcome == "observed" or not isinstance(evidence["reason"], str) or not SAFE_ERROR_RE.fullmatch(evidence["reason"]): + raise ScannerFailure("scanner_result_schema") + continue + expected_fields = { + "robots_policy": {"http_status", "allowed"}, + "public_header_triage": {"http_status", "fetch_status"}, + "security_header_presence": {"http_status", "hsts", "csp", "frame_control", "mime_options"}, + "tls_certificate": {"verification", "protocol", "not_after"}, + }[pair[0]] + if set(evidence) != expected_fields: + raise ScannerFailure("scanner_result_schema") + if "http_status" in evidence: + status = evidence["http_status"] + if status is not None and (type(status) is not int or not 100 <= status <= 599): + raise ScannerFailure("scanner_result_schema") + if pair[0] == "robots_policy" and type(evidence["allowed"]) is not bool: + raise ScannerFailure("scanner_result_schema") + if pair[0] == "public_header_triage" and evidence["fetch_status"] not in ALLOWED_FETCH_STATUSES: + raise ScannerFailure("scanner_result_schema") + if pair[0] == "security_header_presence" and any(type(evidence[key]) is not bool for key in ("hsts", "csp", "frame_control", "mime_options")): + raise ScannerFailure("scanner_result_schema") + if pair[0] == "tls_certificate": + if outcome != "observed" or evidence["verification"] != "verified" or evidence["protocol"] not in {"TLSv1", "TLSv1.1", "TLSv1.2", "TLSv1.3"}: + raise ScannerFailure("scanner_result_schema") + expiry = evidence["not_after"] + if not isinstance(expiry, str) or (expiry and not re.fullmatch(r"[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z", expiry)): + raise ScannerFailure("scanner_result_schema") + if expiry: + try: + datetime.strptime(expiry, "%Y-%m-%dT%H:%M:%SZ") + except ValueError as exc: + raise ScannerFailure("scanner_result_schema") from exc + manual = [{"id": key, "status": "not_assessed"} for key in ( + "runtime_and_backport_confirmation", "internal_configuration_review", "compromise_investigation")] + if value["manual_remainder"] != manual: + raise ScannerFailure("scanner_result_schema") + if len(json.dumps(value, ensure_ascii=False, separators=(",", ":")).encode("utf-8")) > MAX_SCANNER_STDOUT - 64: + raise ScannerFailure("scanner_output_overflow") + return value diff --git a/integrations/worker/scanner.py b/integrations/worker/scanner.py index 84e8b58..15a30fb 100644 --- a/integrations/worker/scanner.py +++ b/integrations/worker/scanner.py @@ -92,7 +92,7 @@ def main() -> int: data = json.loads(raw.decode("utf-8")) if ( not isinstance(data, dict) - or set(data) not in ({"target"}, {"target", "task"}) + or set(data) not in ({"target"}, {"target", "task"}, {"target", "profile_id", "profile_sha256", "services"}) or not isinstance(data["target"], str) or data.get("task", "scan") not in {"scan", "control_dns", "control_https"} ): @@ -101,13 +101,34 @@ def main() -> int: scope, target = RequestOrigin.from_target(data["target"]) if target != data["target"]: return _write({"ok": False, "error": "invalid_scanner_input"}, 2) + external = "profile_id" in data + if external: + from .profiles import validate_binding + from .network import ProfileScope + services = validate_binding(data["profile_id"], data["profile_sha256"], data["services"]) + if (scope.initial_scheme, scope.initial_port) not in services: + return _write({"ok": False, "error": "invalid_scanner_input"}, 2) + external_scope, _ = ProfileScope.from_target(target, data["services"]) channel = _admission_channel() if channel is None: # Fail closed: a scanner that cannot ask for admission never sends a request. return _write({"ok": False, "error": "admission_channel_unavailable"}, 2) - guard = ChildAdmission(channel, kind="control" if task != "scan" else "http") + guard = ChildAdmission(channel, kind="external" if external else "control" if task != "scan" else "http") if task != "scan": return _control(task, scope, guard) + if external: + from .external import execute_external, check_sensitive_response + from .network import ExternalPinnedOpener, ProfileMeter + # One meter and full selected-service scope survive every executor + # factory call. No per-service reset, separate TLS channel or key. + opener = ExternalPinnedOpener(external_scope, guard, + profile_sha256=data["profile_sha256"], meter=ProfileMeter(), + body_validator=lambda body, _headers: check_sensitive_response(body)) + tls_url = "https://" + scope.hostname + "/" + result = execute_external(target, data["profile_sha256"], data["services"], + transport_factory=lambda _service: opener, + tls_probe=lambda _service: opener.inspect_tls(tls_url)) + return _write({"ok": True, "result": result}, 0) checker = Checker(timeout=10.0, delay=1.0, contact=CONTACT) checker.opener = PinnedOpener(scope, guard=guard) try: diff --git a/integrations/worker/service.py b/integrations/worker/service.py index 351ca44..d05c058 100644 --- a/integrations/worker/service.py +++ b/integrations/worker/service.py @@ -24,6 +24,7 @@ API_MAX_BODY, API_SITE_URL, JobClaim, + profile_fields, MAX_LEASE_SECONDS, ) from .runner import MAX_WALL_SECONDS, ScannerFailure, SubprocessRunner, _has_kill_capability @@ -133,7 +134,7 @@ def __init__(self, api: APIClient, spool: DurableSpool, runner, checker_revision self.clock = clock self.stop_event = threading.Event() - def _callback_body(self, job_id, lease, scope_hash, outcome, result, error, revision=None) -> bytes: + def _callback_body(self, job_id, lease, scope_hash, outcome, result, error, revision=None, *, claim=None) -> bytes: if outcome not in {"completed", "failed"}: raise SpoolError("callback_payload_invalid") if outcome == "completed": @@ -151,6 +152,8 @@ def _callback_body(self, job_id, lease, scope_hash, outcome, result, error, revi "error": error, "checker_revision": revision or self.checker_revision, } + if claim is not None: + payload.update(profile_fields(claim)) try: body = json.dumps(payload, ensure_ascii=False, separators=(",", ":"), allow_nan=False).encode("utf-8") except (TypeError, ValueError, UnicodeError) as exc: @@ -192,6 +195,9 @@ def _recover_running(self, record, now: int): datetime.fromtimestamp(record["expires_at"], timezone.utc), record["lease"], datetime.fromtimestamp(record["lease_until"], timezone.utc), + record.get("profile_id", "r2-header-v1"), + record.get("profile_sha256", ""), + tuple((item["scheme"], item["port"]) for item in record.get("services", [])), ) body = self._callback_body( claim.job_id, @@ -201,6 +207,7 @@ def _recover_running(self, record, now: int): None, "scanner_interrupted", revision=record["checker_revision"], + claim=claim, ) self.spool.save_callback(claim, body) self.spool.remove_uncertain(claim.job_id, _lease_hash(claim.lease)) @@ -257,16 +264,17 @@ def process_once(self): outcome, result, error = "failed", None, "stale_claim" else: try: - gate = AdmissionGate(self.api, claim, stop_event=self.stop_event, deadline=self.clock() + scan_timeout, clock=self.clock) - result = self.runner.run(claim.target, timeout=scan_timeout, stop_event=self.stop_event, gate=gate) - SubprocessRunner.validate_result(result, claim.target, scope) + gate = AdmissionGate(self.api, claim, kind="external" if claim.profile_id != "r2-header-v1" else "http", stop_event=self.stop_event, deadline=self.clock() + scan_timeout, clock=self.clock) + options = {"claim": claim} if claim.profile_id != "r2-header-v1" else {} + result = self.runner.run(claim.target, timeout=scan_timeout, stop_event=self.stop_event, gate=gate, **options) + SubprocessRunner.validate_result(result, claim.target, scope, **options) outcome, error = "completed", "" except ScannerFailure as exc: outcome, result, error = "failed", None, exc.slug except (TypeError, ValueError, KeyError): outcome, result, error = "failed", None, "scanner_result_invalid" - body = self._callback_body(claim.job_id, claim.lease, claim.scope_hash, outcome, result, error) + body = self._callback_body(claim.job_id, claim.lease, claim.scope_hash, outcome, result, error, claim=claim) self.spool.save_callback(claim, body) self.spool.remove_uncertain(claim.job_id, _lease_hash(claim.lease)) callbacks = self.spool.callback_records() diff --git a/integrations/worker/spool.py b/integrations/worker/spool.py index 1387590..3bc1820 100644 --- a/integrations/worker/spool.py +++ b/integrations/worker/spool.py @@ -12,6 +12,7 @@ from pathlib import Path from .protocol import API_MAX_BODY, JobClaim +from .profiles import PROFILE_ID, validate_binding MAX_SPOOL_FILE_BYTES = 128 * 1024 @@ -20,6 +21,41 @@ SAFE_FILENAME_RE = re.compile(r"(?:job|uncertain|receipt)-[0-9]{1,19}-lease-[0-9a-f]{24}\.json\Z") TEMP_FILENAME_RE = re.compile(r"\.tmp-[0-9]+-[0-9a-f]{16}\Z") SAFE_ERROR_RE = re.compile(r"[a-z][a-z0-9_]{0,63}\Z") +PROFILE_FIELDS = frozenset({"profile_id", "profile_sha256", "services"}) +CALLBACK_FIELDS = frozenset({"id", "lease", "scope_hash", "outcome", "result", "error", "checker_revision"}) + + +def _valid_profile_fields(value) -> bool: + """Only an absent legacy binding or a complete pinned external binding is valid.""" + present = PROFILE_FIELDS.intersection(value) + if not present: + return True + if present != PROFILE_FIELDS or value.get("profile_id") != PROFILE_ID: + return False + try: + validate_binding(value["profile_id"], value["profile_sha256"], value["services"]) + except (TypeError, ValueError, KeyError): + return False + return True + + +def _claim_profile_fields(claim: JobClaim) -> dict: + profile_id = getattr(claim, "profile_id", "r2-header-v1") + digest = getattr(claim, "profile_sha256", "") + services = getattr(claim, "services", ()) + if profile_id == "r2-header-v1": + if digest != "" or services != (): + raise SpoolError("callback_binding_mismatch") + return {} + try: + fields = {"profile_id": profile_id, "profile_sha256": digest, + "services": [{"scheme": scheme, "port": port} for scheme, port in services]} + except (TypeError, ValueError): + raise SpoolError("callback_binding_mismatch") from None + if not _valid_profile_fields(fields): + raise SpoolError("callback_binding_mismatch") + return fields + class SpoolError(Exception): @@ -194,6 +230,13 @@ def save_callback(self, claim: JobClaim, body: bytes): or data.get("lease") != claim.lease ): raise SpoolError("callback_binding_mismatch") + profile = _claim_profile_fields(claim) + present = PROFILE_FIELDS.intersection(data) + if profile: + if set(data) != CALLBACK_FIELDS | PROFILE_FIELDS or any(data.get(key) != value for key, value in profile.items()): + raise SpoolError("callback_binding_mismatch") + elif present: + raise SpoolError("callback_binding_mismatch") stem = f"job-{claim.job_id}-lease-{_lease_hash(claim.lease)}" record = { "kind": "callback", @@ -203,6 +246,7 @@ def save_callback(self, claim: JobClaim, body: bytes): "lease_until": int(claim.lease_until.timestamp()), "body": body.decode("utf-8"), } + record.update(profile) path = self.pending / f"{stem}.json" try: old = self._read_file(path) @@ -231,6 +275,7 @@ def save_start_uncertain(self, claim: JobClaim): "expires_at": int(claim.expires_at.timestamp()), "lease_until": int(claim.lease_until.timestamp()), } + record.update(_claim_profile_fields(claim)) path = self.uncertain / f"{stem}.json" self._atomic_write(path, self._encode(record)) return path @@ -249,6 +294,7 @@ def save_running(self, claim: JobClaim, checker_revision: str, started_at: int): "started_at": started_at, "checker_revision": checker_revision, } + record.update(_claim_profile_fields(claim)) path = self.uncertain / f"{stem}.json" self._atomic_write(path, self._encode(record)) return path @@ -273,7 +319,11 @@ def callback_records(self): @staticmethod def _valid_callback_record(record, path): required = {"kind", "id", "lease_hash", "expires_at", "lease_until", "body"} - if not isinstance(record, dict) or set(record) != required or record.get("kind") != "callback": + if not isinstance(record, dict) or record.get("kind") != "callback" or not _valid_profile_fields(record): + return False + if PROFILE_FIELDS.intersection(record): + required |= PROFILE_FIELDS + if set(record) != required: return False if isinstance(record["id"], bool) or not isinstance(record["id"], int) or record["id"] < 1: return False @@ -289,6 +339,17 @@ def _valid_callback_record(record, path): return False if not isinstance(payload, dict) or payload.get("id") != record["id"]: return False + if not _valid_profile_fields(payload): + return False + record_profile = PROFILE_FIELDS.intersection(record) + payload_profile = PROFILE_FIELDS.intersection(payload) + if record_profile != payload_profile: + return False + if payload_profile: + if set(payload) != CALLBACK_FIELDS | PROFILE_FIELDS: + return False + if any(record[key] != payload[key] for key in PROFILE_FIELDS): + return False lease = payload.get("lease") return ( isinstance(lease, str) @@ -338,6 +399,10 @@ def _valid_uncertain_record(record, path): required |= {"lease", "scope_hash", "started_at", "checker_revision"} elif kind != "start_uncertain": return False + if not _valid_profile_fields(record): + return False + if PROFILE_FIELDS.intersection(record): + required |= PROFILE_FIELDS if set(record) != required: return False if isinstance(record["id"], bool) or not isinstance(record["id"], int) or record["id"] < 1: diff --git a/test_external_worker_wire.py b/test_external_worker_wire.py new file mode 100644 index 0000000..3736fe0 --- /dev/null +++ b/test_external_worker_wire.py @@ -0,0 +1,183 @@ +"""Offline frozen-profile claim, callback and recovery regressions.""" +from copy import deepcopy +from dataclasses import replace +from datetime import datetime, timezone +import hashlib +import json +from pathlib import Path +import tempfile +import unittest +from unittest.mock import Mock, patch + +from integrations.worker.protocol import APIError, profile_fields, validate_claim +from integrations.worker.profiles import PROFILE_ID, PROFILE_SHA256 +from integrations.worker.runner import ScannerFailure, SubprocessRunner +from integrations.worker.service import WorkerService +from integrations.worker.spool import DurableSpool, SpoolError +from integrations.worker.network import RequestOrigin +from test_scan_worker import _claim, _result, _FakeAPI, NOW, REVISION, TARGET + + +def claim(): + return replace(_claim(), profile_id=PROFILE_ID, profile_sha256=PROFILE_SHA256, + services=(("https", 443),)) + + +def result(): + value = _result() + value.update(profile_fields(claim())) + value["checks"] = [ + {"id": "robots_policy", "service": "https:443", "outcome": "observed", "evidence": {"http_status": 200, "allowed": True}}, + {"id": "public_header_triage", "service": "https:443", "outcome": "observed", "evidence": {"http_status": 200, "fetch_status": "ok"}}, + {"id": "security_header_presence", "service": "https:443", "outcome": "observed", "evidence": {"http_status": 200, "hsts": True, "csp": False, "frame_control": True, "mime_options": False}}, + {"id": "tls_certificate", "service": "https:443", "outcome": "observed", "evidence": {"verification": "verified", "protocol": "TLSv1.3", "not_after": "2027-01-01T00:00:00Z"}}, + ] + value["manual_remainder"] = [{"id": key, "status": "not_assessed"} for key in ( + "runtime_and_backport_confirmation", "internal_configuration_review", "compromise_investigation")] + return value + + +class ExternalWireTests(unittest.TestCase): + def wire(self, value): + return {"job": {"id": value.job_id, "target": value.target, + "scope_hash": value.scope_hash, "terms_hash": value.terms_hash, + "expires_at": int(value.expires_at.timestamp()), "lease": value.lease, + "lease_until": int(value.lease_until.timestamp()), **profile_fields(value)}} + + def test_legacy_shape_and_external_binding(self): + legacy = self.wire(_claim()) + self.assertEqual(len(legacy["job"]), 7) + self.assertEqual(validate_claim(legacy, now=datetime.fromtimestamp(NOW, timezone.utc)), _claim()) + wire = self.wire(claim()) + self.assertEqual(len(wire["job"]), 10) + self.assertEqual(validate_claim(wire, now=datetime.fromtimestamp(NOW, timezone.utc)), claim()) + + def test_claim_refuses_unknown_partial_digest_reorder_or_primary_service(self): + for edit in ( + lambda x: x.pop("profile_sha256"), + lambda x: x.update(profile_id="unknown"), + lambda x: x.update(profile_sha256="0" * 64), + lambda x: x.update(services=[{"scheme": "http", "port": 80}]), + lambda x: x.update(services=[{"scheme": "https", "port": 443}, {"scheme": "http", "port": 80}]), + lambda x: x.update(services=[{"scheme": "https", "port": True}]), + lambda x: x.update(extra=True), + ): + value = self.wire(claim()) + edit(value["job"]) + with self.assertRaises(APIError): + validate_claim(value, now=datetime.fromtimestamp(NOW, timezone.utc)) + + def test_result_typed_observations_and_no_raw_sensitive_fields(self): + scope, _ = RequestOrigin.from_target(TARGET) + value = result() + self.assertIs(SubprocessRunner.validate_result(value, TARGET, scope, claim=claim()), value) + edits = [ + lambda x: x.update(title="Private content"), + lambda x: x.update(profile_sha256="0" * 64), + lambda x: x["checks"].append(deepcopy(x["checks"][0])), + lambda x: x["checks"][0]["evidence"].update(raw="Private content"), + lambda x: x["checks"][0]["evidence"].update(allowed=1), + lambda x: x["checks"][3]["evidence"].update(not_after="2027-02-31T00:00:00Z"), + lambda x: x["manual_remainder"].pop(), + lambda x: x["manual_remainder"][0].update(status="certified"), + lambda x: x["checks"][0].update(service="http:80"), + ] + for edit in edits: + value = result() + edit(value) + with self.assertRaises(ScannerFailure): + SubprocessRunner.validate_result(value, TARGET, scope, claim=claim()) + + def test_running_recovery_posts_same_binding_without_scanner_or_claim(self): + with tempfile.TemporaryDirectory() as root: + spool = DurableSpool(root) + spool.prepare() + spool.save_running(claim(), REVISION, NOW - 181) + api, runner = _FakeAPI(), Mock() + service = WorkerService(api, spool, runner, REVISION, clock=lambda: NOW) + self.assertEqual(service.process_once(), "callback_accepted") + payload = json.loads(api.result_bodies[0]) + self.assertEqual(payload["outcome"], "failed") + self.assertEqual(payload["error"], "scanner_interrupted") + self.assertIsNone(payload["result"]) + for key, value in profile_fields(claim()).items(): + self.assertEqual(payload[key], value) + runner.run.assert_not_called() + self.assertEqual(api.claim_calls, 0) + self.assertEqual(spool.active_uncertain(NOW), []) + + def test_callback_retry_exact_bytes_and_downgrade_refused(self): + with tempfile.TemporaryDirectory() as root: + spool = DurableSpool(root) + spool.prepare() + service = WorkerService(_FakeAPI(), spool, Mock(), REVISION, clock=lambda: NOW) + body = service._callback_body(claim().job_id, claim().lease, claim().scope_hash, + "completed", result(), "", claim=claim()) + path = spool.save_callback(claim(), body) + self.assertEqual(spool.save_callback(claim(), body), path) + self.assertEqual(spool.callback_records()[0][1]["body"].encode(), body) + record = json.loads(path.read_text()) + payload = json.loads(record["body"]) + for key in profile_fields(claim()): + payload.pop(key) + record["body"] = json.dumps(payload) + path.write_text(json.dumps(record)) + with self.assertRaises(SpoolError): + spool.callback_records() + + def test_service_completes_with_bound_runner_and_additive_callback(self): + with tempfile.TemporaryDirectory() as root: + spool = DurableSpool(root) + spool.prepare() + runner = Mock() + runner.run.return_value = result() + api = _FakeAPI(claim()) + with patch("integrations.worker.service.AdmissionGate") as gate: + service = WorkerService(api, spool, runner, REVISION, clock=lambda: NOW) + self.assertEqual(service.process_once(), "callback_accepted") + self.assertEqual(gate.call_args.kwargs["kind"], "external") + self.assertEqual(runner.run.call_args.kwargs["claim"], claim()) + self.assertLessEqual(runner.run.call_args.kwargs["timeout"], 180) + payload = json.loads(api.result_bodies[0]) + self.assertEqual(payload["outcome"], "completed") + self.assertEqual(len(payload), 10) + for key, value in profile_fields(claim()).items(): + self.assertEqual(payload[key], value) + self.assertEqual(payload["result"][key], value) + + def test_service_failure_also_carries_profile_and_retry_never_rescans(self): + with tempfile.TemporaryDirectory() as root: + spool = DurableSpool(root) + spool.prepare() + runner = Mock() + runner.run.side_effect = ScannerFailure("admission_refused") + api = _FakeAPI(claim(), result_errors=[APIError("network"), None]) + with patch("integrations.worker.service.AdmissionGate"): + service = WorkerService(api, spool, runner, REVISION, clock=lambda: NOW) + self.assertEqual(service.process_once(), "callback_pending_network") + self.assertEqual(service.process_once(), "callback_accepted") + self.assertEqual(runner.run.call_count, 1) + self.assertEqual(api.result_bodies[0], api.result_bodies[1]) + payload = json.loads(api.result_bodies[0]) + self.assertEqual(payload["outcome"], "failed") + self.assertIsNone(payload["result"]) + self.assertEqual(payload["error"], "admission_refused") + self.assertEqual(payload["profile_sha256"], PROFILE_SHA256) + + def test_start_marker_freezes_binding_before_ambiguous_start(self): + with tempfile.TemporaryDirectory() as root: + spool = DurableSpool(root) + spool.prepare() + runner = Mock() + service = WorkerService(_FakeAPI(claim(), start_error=APIError("network")), spool, + runner, REVISION, clock=lambda: NOW) + self.assertEqual(service.process_once(), "start_uncertain") + marker = spool.active_uncertain(NOW)[0] + for key, value in profile_fields(claim()).items(): + self.assertEqual(marker[key], value) + self.assertEqual(service.process_once(), "waiting_for_lease") + runner.run.assert_not_called() + + +if __name__ == "__main__": + unittest.main() From 71fc849bd4ee0c620a2065da28023326746d9e0b Mon Sep 17 00:00:00 2001 From: Timothy Wayne Gregg Date: Fri, 13 Nov 2026 23:04:02 -0500 Subject: [PATCH 03/19] fix(worker): reject contradictory automatic check observations --- integrations/worker/runner.py | 27 +++++++++++++++++++-------- test_external_worker_wire.py | 4 ++++ 2 files changed, 23 insertions(+), 8 deletions(-) diff --git a/integrations/worker/runner.py b/integrations/worker/runner.py index 6989b44..013fb69 100644 --- a/integrations/worker/runner.py +++ b/integrations/worker/runner.py @@ -508,11 +508,16 @@ def validate_external_result(value, target, scope, claim): if pair[1] not in selected: raise ScannerFailure("scanner_result_scope_mismatch") if set(evidence) == {"reason", "http_status"}: - if outcome != "unknown" or evidence["reason"] != "non_successful_response" or type(evidence["http_status"]) is not int or not 100 <= evidence["http_status"] <= 599: + if pair[0] != "security_header_presence" or outcome != "unknown" or evidence["reason"] != "non_successful_response" or type(evidence["http_status"]) is not int or not 100 <= evidence["http_status"] <= 599 or 200 <= evidence["http_status"] < 300: raise ScannerFailure("scanner_result_schema") continue if set(evidence) == {"reason"}: - if outcome == "observed" or not isinstance(evidence["reason"], str) or not SAFE_ERROR_RE.fullmatch(evidence["reason"]): + reasons = { + ("security_header_presence", "unknown"): {"service_not_assessed"}, + ("tls_certificate", "unknown"): {"service_not_assessed"}, + ("tls_certificate", "blocked"): {"tls_verification_failed"}, + } + if not isinstance(evidence["reason"], str) or evidence["reason"] not in reasons.get((pair[0], outcome), set()): raise ScannerFailure("scanner_result_schema") continue expected_fields = { @@ -527,12 +532,18 @@ def validate_external_result(value, target, scope, claim): status = evidence["http_status"] if status is not None and (type(status) is not int or not 100 <= status <= 599): raise ScannerFailure("scanner_result_schema") - if pair[0] == "robots_policy" and type(evidence["allowed"]) is not bool: - raise ScannerFailure("scanner_result_schema") - if pair[0] == "public_header_triage" and evidence["fetch_status"] not in ALLOWED_FETCH_STATUSES: - raise ScannerFailure("scanner_result_schema") - if pair[0] == "security_header_presence" and any(type(evidence[key]) is not bool for key in ("hsts", "csp", "frame_control", "mime_options")): - raise ScannerFailure("scanner_result_schema") + if pair[0] == "robots_policy": + if type(evidence["allowed"]) is not bool or outcome != ("observed" if evidence["allowed"] else "blocked"): + raise ScannerFailure("scanner_result_schema") + if pair[0] == "public_header_triage": + status_name = evidence["fetch_status"] + if not isinstance(status_name, str) or status_name not in ALLOWED_FETCH_STATUSES or outcome != ("observed" if status_name == "ok" else "unknown"): + raise ScannerFailure("scanner_result_schema") + if status_name == "ok" and (evidence["http_status"] is None or not 200 <= evidence["http_status"] < 300): + raise ScannerFailure("scanner_result_schema") + if pair[0] == "security_header_presence": + if outcome != "observed" or type(evidence["http_status"]) is not int or not 200 <= evidence["http_status"] < 300 or any(type(evidence[key]) is not bool for key in ("hsts", "csp", "frame_control", "mime_options")): + raise ScannerFailure("scanner_result_schema") if pair[0] == "tls_certificate": if outcome != "observed" or evidence["verification"] != "verified" or evidence["protocol"] not in {"TLSv1", "TLSv1.1", "TLSv1.2", "TLSv1.3"}: raise ScannerFailure("scanner_result_schema") diff --git a/test_external_worker_wire.py b/test_external_worker_wire.py index 3736fe0..41031c5 100644 --- a/test_external_worker_wire.py +++ b/test_external_worker_wire.py @@ -81,6 +81,10 @@ def test_result_typed_observations_and_no_raw_sensitive_fields(self): lambda x: x["manual_remainder"].pop(), lambda x: x["manual_remainder"][0].update(status="certified"), lambda x: x["checks"][0].update(service="http:80"), + lambda x: x["checks"][0].update(outcome="unknown"), + lambda x: x["checks"][1].update(outcome="blocked"), + lambda x: x["checks"][2].update(evidence={"reason": "private_header_value"}, outcome="unknown"), + lambda x: x["checks"][3].update(evidence={"reason": "service_not_assessed"}, outcome="observed"), ] for edit in edits: value = result() From acb6794e692c4eaa0696fbaa5880412dc2bad0e6 Mon Sep 17 00:00:00 2001 From: Timothy Wayne Gregg Date: Fri, 13 Nov 2026 23:04:47 -0500 Subject: [PATCH 04/19] feat(worker): execute bounded external observations with passive Checker --- integrations/worker/external.py | 326 ++++++++++++++++++++++++++++++++ test_external_executor.py | 203 ++++++++++++++++++++ 2 files changed, 529 insertions(+) create mode 100644 integrations/worker/external.py create mode 100644 test_external_executor.py diff --git a/integrations/worker/external.py b/integrations/worker/external.py new file mode 100644 index 0000000..52b433d --- /dev/null +++ b/integrations/worker/external.py @@ -0,0 +1,326 @@ +"""Bounded external profile execution using the maintained passive Checker. + +The caller supplies the real pinned/admitted transport and TLS probe. This +module never supplies authorization, retries, a resolver, a proxy or a new +scanner implementation. There are deliberately zero automatic retries. +""" + +from __future__ import annotations + +from dataclasses import asdict +from datetime import datetime +from http.client import HTTPException +import math +import re +import time +from urllib.parse import urljoin, urlsplit +from urllib.request import Request + +from wordpress_check import Checker, Response, assess_php2shell, origin + +from .network import RequestOrigin +from .profiles import PROFILE_ID, ProfileError, profile_manifest, validate_binding + + +CONTACT = "https://complete.tech/services/website-security/" +_RESTRICTED = {401, 403, 407, 429, 451} +_REDIRECTS = {301, 302, 303, 307, 308} +_INDICATORS = ( + ("possible_sensitive_indicator", re.compile(rb"-----BEGIN [A-Z ]*PRIVATE KEY-----")), + ("possible_sensitive_indicator", re.compile(rb"\b(?:AKIA|ASIA)[A-Z0-9]{16}\b")), + ("possible_sensitive_indicator", re.compile(rb"\baws_secret_access_key\b\s*[:=]\s*[\"']?[A-Za-z0-9/+=]{20,}", re.I)), + ("possible_compromise_indicator", re.compile(rb"\b(?:hacked|defaced)\s+by\b", re.I)), +) + + +class ProfileStopped(Exception): + """A fixed reason only: raw target data must not enter errors or logs.""" + + def __init__(self, reason): + if not isinstance(reason, str) or not re.fullmatch(r"[a-z][a-z0-9_]{0,63}", reason): + reason = "profile_stopped" + self.reason = reason + super().__init__(reason) + + +def _indicators(raw: bytes): + for reason, pattern in _INDICATORS: + if pattern.search(raw): + raise ProfileStopped(reason) + + +def check_sensitive_response(raw: bytes) -> None: + """Validate accumulated decoded bytes without retaining or echoing them.""" + if type(raw) is not bytes: + raise ProfileStopped("response_body_invalid") + _indicators(raw) + + +def _headers(message, maximum): + pairs = list(message.raw_items()) if hasattr(message, "raw_items") else list(message.items()) + total = 2 + result = {} + for key, value in pairs: + if not isinstance(key, str) or not isinstance(value, str): + raise ProfileStopped("response_headers_invalid") + encoded = (key + ": " + value + "\r\n").encode("utf-8", errors="replace") + total += len(encoded) + if total > maximum: + raise ProfileStopped("response_header_limit") + _indicators(encoded) + name = key.lower() + result[name] = result.get(name, "") + (", " if name in result else "") + value + return result + + +class _State: + def __init__(self, limits, clock): + self.limits, self.clock = limits, clock + self._last_now = None + self.deadline = self.now() + limits["active_seconds"] + self.last_started = -math.inf + self.body_bytes = 0 + self.redirects = 0 + + def now(self): + value = self.clock() + if isinstance(value, bool) or not isinstance(value, (int, float)) or not math.isfinite(value): + raise ProfileStopped("clock_invalid") + if self._last_now is not None and value < self._last_now: + raise ProfileStopped("clock_anomaly") + self._last_now = value + return value + + def remaining(self): + value = self.deadline - self.now() + if value <= 0: + raise ProfileStopped("active_time_limit") + return value + + def pace(self, delay=1.0): + wait = max(delay, self.limits["min_interval_ms"] / 1000) - (self.now() - self.last_started) + if wait > 0: + if wait >= self.remaining(): + raise ProfileStopped("active_time_limit") + time.sleep(wait) + self.remaining() + self.last_started = self.now() + + def consume(self, raw): + if type(raw) is not bytes: + raise ProfileStopped("response_body_invalid") + if len(raw) > self.limits["response_body_bytes"]: + raise ProfileStopped("response_body_limit") + self.body_bytes += len(raw) + if self.body_bytes > self.limits["aggregate_body_bytes"]: + raise ProfileStopped("aggregate_body_limit") + _indicators(raw) + self.remaining() + + +class _ManifestOpener: + def __init__(self, transport, host, services, state): + self.transport, self.host, self.services, self.state = transport, host, set(services), state + + def _validate(self, value): + try: + parts = urlsplit(value) + port = parts.port or (443 if parts.scheme == "https" else 80) + except (TypeError, ValueError): + raise ProfileStopped("request_out_of_scope") from None + if (parts.hostname != self.host or (parts.scheme, port) not in self.services + or parts.username is not None or parts.password is not None + or parts.path not in {"", "/", "/robots.txt"} or parts.query or parts.fragment): + raise ProfileStopped("request_out_of_scope") + return parts + + def open(self, request, timeout=None): + parts = self._validate(request.full_url) + if request.get_method() not in {"GET", "HEAD"} or request.data is not None: + raise ProfileStopped("request_method_forbidden") + self.state.remaining() + response = self.transport.open(request, timeout=min(timeout or 10, 10, self.state.remaining())) + try: + self.state.remaining() + headers = _headers(response.headers, self.state.limits["response_header_bytes"]) + if response.code in _RESTRICTED: + raise ProfileStopped("access_restricted") + if 500 <= response.code <= 599: + raise ProfileStopped("server_error") + if response.code in _REDIRECTS: + self.state.redirects += 1 + if self.state.redirects > self.state.limits["redirect_hops"]: + raise ProfileStopped("redirect_limit") + location = headers.get("location") + if not location: + raise ProfileStopped("redirect_invalid") + destination = self._validate(urljoin(request.full_url, location)) + if (destination.path or "/") != (parts.path or "/") or (parts.scheme == "https" and destination.scheme == "http"): + raise ProfileStopped("redirect_out_of_scope") + return response + except BaseException: + response.close() + raise + + +class _ExternalChecker(Checker): + def __init__(self, opener, state, authorized_origins): + super().__init__(timeout=10, delay=1, contact=CONTACT, authorized_origins=authorized_origins) + self.opener, self.state = opener, state + self.responses = [] + self.robots_allowed = False + + def _request(self, url, limit, delay): + # Unlike the general CLI's optional disposable request worker, this + # always uses the caller's guarded transport and shared profile clock. + self.state.pace(max(self.delay, delay or 0)) + self.last_request[origin(url)] = self.state.now() + return self._fetch(url, limit) + + def policy(self, url, authorized=None): + self.robots_allowed = False + delay = super().policy(url, authorized) + self.robots_allowed = True + return delay + + def _fetch(self, url, limit): + request = Request(url, headers={"User-Agent": self.user_agent, "Accept": "text/html,text/plain;q=0.9,*/*;q=0.1", "Accept-Encoding": "identity"}) + with self.opener.open(request, timeout=min(10, self.state.remaining())) as response: + headers = _headers(response.headers, self.state.limits["response_header_bytes"]) + if headers.get("content-encoding", "identity").lower() not in {"", "identity"}: + # A bounded decoder may normalize this header to identity. Never + # decompress arbitrary target content in a second code path. + raise ProfileStopped("content_encoding_unavailable") + raw = response.read(min(limit, self.state.limits["response_body_bytes"]) + 1) + self.state.consume(raw) + charset = response.headers.get_content_charset() if hasattr(response.headers, "get_content_charset") else None + try: + body = raw.decode(charset or "utf-8", errors="replace") + except LookupError: + body = raw.decode("utf-8", errors="replace") + self.responses.append((urlsplit(url).path or "/", response.code)) + return Response(response.code, headers, body, False) + + +def _record(check_id, service, outcome, evidence): + return {"id": check_id, "service": service, "outcome": outcome, "evidence": evidence} + + +def _head(checker, url, service): + checker.state.pace() + request = Request(url, method="HEAD", headers={"User-Agent": checker.user_agent, "Accept-Encoding": "identity"}) + with checker.opener.open(request, timeout=min(10, checker.state.remaining())) as response: + # HEAD metadata is inspected, but a body is never read, even if a server + # incorrectly sends one. Closing the response discards it. + headers = _headers(response.headers, checker.state.limits["response_header_bytes"]) + checker.state.remaining() + if not 200 <= response.code < 300: + return _record("security_header_presence", service, "unknown", {"reason": "non_successful_response", "http_status": response.code}) + return _record("security_header_presence", service, "observed", { + "http_status": response.code, "hsts": "strict-transport-security" in headers, + "csp": "content-security-policy" in headers, "frame_control": "x-frame-options" in headers, + "mime_options": "x-content-type-options" in headers, + }) + + +def _tls(probe, service_value, service_name, state): + state.pace() + result = probe(dict(service_value)) + state.remaining() + if (type(result) is not dict or set(result) != {"verified", "protocol", "not_after"} + or result["verified"] is not True + or type(result["protocol"]) is not str + or result["protocol"] not in {"TLSv1", "TLSv1.1", "TLSv1.2", "TLSv1.3"} + or type(result["not_after"]) is not str or len(result["not_after"]) > 20): + raise ProfileStopped("tls_evidence_invalid") + expiry = result["not_after"] + if expiry: + try: + parsed = datetime.strptime(expiry, "%Y-%m-%dT%H:%M:%SZ") + if parsed.strftime("%Y-%m-%dT%H:%M:%SZ") != expiry: + raise ValueError + except ValueError: + raise ProfileStopped("tls_evidence_invalid") from None + return _record("tls_certificate", service_name, "observed", { + "verification": "verified", "protocol": result["protocol"], "not_after": expiry, + }) + + +def execute_external(target, profile_sha256, services, *, transport_factory, tls_probe, clock=time.monotonic): + """Execute this exact profile with externally enforced, durable admission. + + ``transport_factory(service_dict)`` must return the caller's real pinned, + bounded and admitted opener. ``tls_probe(service_dict)`` must perform one + independently admitted verified TLS-only connection and return only + ``{verified: True, protocol, not_after}``. Missing dependencies refuse before + target traffic. Neither callback may accept user-supplied network options. + """ + selected = validate_binding(PROFILE_ID, profile_sha256, services) + scope, canonical = RequestOrigin.from_target(target) + if canonical != target or (scope.initial_scheme, scope.initial_port) not in selected: + raise ProfileError("profile_target_mismatch") + if not callable(transport_factory) or not callable(tls_probe): + raise ProfileError("profile_transport_unavailable") + manifest = profile_manifest() + state = _State(manifest["limits"], clock) + records, results = [], [] + origins = [f"{scheme}://{scope.hostname}" for scheme, _port in selected] + for scheme, port in selected: + value = {"scheme": scheme, "port": port} + service_name = f"{scheme}:{port}" + url = f"{scheme}://{scope.hostname}/" + state.remaining() + opener = _ManifestOpener(transport_factory(dict(value)), scope.hostname, selected, state) + checker = _ExternalChecker(opener, state, origins) + result = checker.check(url) + results.append(((scheme, port), result)) + robots_codes = [code for path, code in checker.responses if path == "/robots.txt"] + records.append(_record("robots_policy", service_name, "observed" if checker.robots_allowed else "blocked", { + "http_status": robots_codes[-1] if robots_codes else None, "allowed": checker.robots_allowed, + })) + success = result.fetch_status == "ok" + records.append(_record("public_header_triage", service_name, "observed" if success else "unknown", { + "http_status": result.http_status, "fetch_status": result.fetch_status, + })) + if success and checker.robots_allowed: + records.append(_head(checker, url, service_name)) + else: + records.append(_record("security_header_presence", service_name, "unknown", {"reason": "service_not_assessed"})) + if scheme != "https": + records.append(_record("tls_certificate", service_name, "not_selected", {"reason": "https_not_selected"})) + elif success and checker.robots_allowed: + try: + records.append(_tls(tls_probe, value, service_name, state)) + except (OSError, HTTPException): + records.append(_record("tls_certificate", service_name, "blocked", {"reason": "tls_verification_failed"})) + else: + records.append(_record("tls_certificate", service_name, "unknown", {"reason": "service_not_assessed"})) + primary = next(result for service, result in results if service == (scope.initial_scheme, scope.initial_port)) + output = asdict(primary) + output["input_url"] = target + for name in ("title", "server", "powered_by", "wordpress_version"): + output[name] = "" + for name in ("evidence", "asset_version_hints", "notes", "php_evidence"): + output[name] = [] + complete_services = all(result.fetch_status == "ok" for _service, result in results) + if complete_services: + candidates = sorted({candidate for _service, result in results for candidate in result.php_version_candidates + if re.fullmatch(r"[0-9][A-Za-z0-9.+_~:-]{0,63}", candidate)}) + if len(candidates) > 32: + candidates = [] + output["php_version_candidates"] = candidates + output["php_version"] = candidates[0] if len(candidates) == 1 else "" + if any(result.php_evidence for _service, result in results): + output["php_evidence"] = ["PHP was mentioned in public response headers; raw values were discarded."] + else: + # Do not present a successful port's version as the assessment of an + # incompletely observed selected service set. + output["php_version_candidates"], output["php_version"], output["wordpress_version"] = [], "", "" + if output["fetch_status"] == "ok": + output["status"] = "uncertain" + output.update(assess_php2shell(output["php_version"], version_candidates=output["php_version_candidates"] or None, + php_hint=bool(output["php_evidence"]) or output["status"] in {"wordpress", "likely_wordpress"})) + output.update({"profile_id": PROFILE_ID, "profile_sha256": profile_sha256, + "services": [dict(value) for value in services], "checks": records, + "manual_remainder": [{"id": item["check_id"], "status": "not_assessed"} for item in manifest["manual_remainder"]]}) + return output diff --git a/test_external_executor.py b/test_external_executor.py new file mode 100644 index 0000000..f2e81b8 --- /dev/null +++ b/test_external_executor.py @@ -0,0 +1,203 @@ +"""Synthetic transport tests; no target, database or production requests.""" + +from dataclasses import fields +from email.message import Message +import json +import unittest +from unittest import mock + +from wordpress_check import Result +from integrations.worker import external as E, profiles as P +from integrations.worker.network import RequestOrigin +from integrations.worker.runner import SubprocessRunner + + +TARGET = "https://example.test/" +HTTPS = [{"scheme": "https", "port": 443}] + + +class Clock: + def __init__(self): self.value = 0.0 + def __call__(self): return self.value + def sleep(self, value): self.value += value + + +class Response: + def __init__(self, status=200, body=b"", headers=None, no_read=False): + self.code, self.body, self.no_read = status, body, no_read + self.headers = Message() + for name, value in (headers or {}).items(): self.headers[name] = value + self.closed, self.reads = False, 0 + def read(self, size=-1): + self.reads += 1 + if self.no_read: raise AssertionError("metadata body was read") + return self.body if size < 0 else self.body[:size] + def close(self): self.closed = True + def __enter__(self): return self + def __exit__(self, *_args): self.close() + + +class Transport: + def __init__(self, responses): self.responses, self.requests = list(responses), [] + def open(self, request, timeout=None): + self.requests.append((request.get_method(), request.full_url, timeout)) + if not self.responses: raise AssertionError("unexpected additional request") + return self.responses.pop(0) + + +def success_responses(version="8.3.36"): + return [Response(body=b"User-agent: *\nAllow: /\n"), + Response(body=b'PRIVATE-TITLE-SENTINEL', + headers={"Content-Type": "text/html", "X-Powered-By": "PHP/" + version, "Server": "RAW-SERVER-SENTINEL"}), + Response(headers={"Strict-Transport-Security": "max-age=31536000", "Content-Security-Policy": "default-src 'self'"}, no_read=True)] + + +class ExecutorTests(unittest.TestCase): + def run_profile(self, transport, *, services=None, target=TARGET, tls=None, clock=None, digest=P.PROFILE_SHA256): + self.clock = clock or Clock() + self.tls_calls = [] + def probe(service): + self.tls_calls.append(service) + if tls is not None: return tls(service) + return {"verified": True, "protocol": "TLSv1.3", "not_after": "2099-01-01T00:00:00Z"} + with mock.patch.object(E.time, "sleep", side_effect=self.clock.sleep): + return E.execute_external(target, digest, services or HTTPS, + transport_factory=lambda _service: transport, tls_probe=probe, clock=self.clock) + + def test_real_checker_result_compatible_and_raw_content_omitted(self): + responses = success_responses() + transport = Transport(responses) + result = self.run_profile(transport) + self.assertEqual([method for method, _url, _timeout in transport.requests], ["GET", "GET", "HEAD"]) + self.assertEqual(result["profile_id"], P.PROFILE_ID) + self.assertEqual(result["php_version"], "8.3.36") + self.assertEqual(result["php2shell_exploitability"], "not_tested") + self.assertEqual(len(result["checks"]), 4) + self.assertTrue(all(item["status"] == "not_assessed" for item in result["manual_remainder"])) + text = json.dumps(result) + for sentinel in ("PRIVATE-TITLE-SENTINEL", "RAW-SERVER-SENTINEL", "WordPress 6.8.3"): + self.assertNotIn(sentinel, text) + self.assertEqual(result["notes"], []) + core = {field.name: result[field.name] for field in fields(Result)} + self.assertEqual(SubprocessRunner.validate_result(core, TARGET, RequestOrigin.from_target(TARGET)[0]), core) + self.assertTrue(all(response.closed for response in responses)) + self.assertEqual(responses[-1].reads, 0) + + def test_missing_binding_refuses_before_transport(self): + transport = Transport([]) + with self.assertRaises(P.ProfileError): self.run_profile(transport, digest="0" * 64) + self.assertEqual(transport.requests, []) + + def test_target_service_cannot_be_added_implicitly(self): + transport = Transport([]) + with self.assertRaises(P.ProfileError): + self.run_profile(transport, services=[{"scheme": "http", "port": 80}]) + self.assertEqual(transport.requests, []) + + def test_robots_denial_stops_all_remaining_connections(self): + transport = Transport([Response(body=b"User-agent: *\nDisallow: /\n")]) + result = self.run_profile(transport) + self.assertEqual(len(transport.requests), 1) + self.assertEqual(self.tls_calls, []) + self.assertEqual(result["status"], "blocked") + self.assertFalse(result["checks"][0]["evidence"]["allowed"]) + + def test_sensitive_body_stops_without_emitting_target_bytes(self): + sentinel = b"-----BEGIN PRIVATE KEY-----\nPRIVATE-KEY-SENTINEL" + transport = Transport([Response(body=b"User-agent: *\nAllow: /\n"), Response(body=sentinel)]) + with self.assertRaises(E.ProfileStopped) as caught: self.run_profile(transport) + self.assertEqual(str(caught.exception), "possible_sensitive_indicator") + self.assertNotIn("SENTINEL", str(caught.exception)) + self.assertEqual(len(transport.requests), 2) + self.assertEqual(self.tls_calls, []) + + def test_decoded_accumulated_body_hook_detects_split_marker(self): + E.check_sensitive_response(b"-----BEGIN PRI") + with self.assertRaises(E.ProfileStopped): E.check_sensitive_response(b"-----BEGIN PRIVATE KEY-----") + + def test_possible_compromise_indicator_is_not_a_diagnosis(self): + with self.assertRaises(E.ProfileStopped) as caught: + E.check_sensitive_response(b"This page says Hacked by SYNTHETIC fixture") + self.assertEqual(str(caught.exception), "possible_compromise_indicator") + + def test_header_limit_checked_before_body_and_response_closed(self): + response = Response(headers={"X-Large": "A" * 32768}, no_read=True) + transport = Transport([response]) + with self.assertRaises(E.ProfileStopped) as caught: self.run_profile(transport) + self.assertEqual(str(caught.exception), "response_header_limit") + self.assertTrue(response.closed) + self.assertEqual(response.reads, 0) + + def test_body_limit_and_unsupported_compression_refuse(self): + for response, reason in ((Response(body=b"A" * 262145), "response_body_limit"), + (Response(body=b"GZIP-SENTINEL", headers={"Content-Encoding": "gzip"}, no_read=True), "content_encoding_unavailable")): + with self.subTest(reason=reason), self.assertRaises(E.ProfileStopped) as caught: + self.run_profile(Transport([response])) + self.assertEqual(str(caught.exception), reason) + self.assertTrue(response.closed) + + def test_access_denial_or_server_error_never_retries(self): + for status in (401, 403, 407, 429, 451, 500, 503): + transport = Transport([Response(status=status, no_read=True)]) + with self.subTest(status=status), self.assertRaises(E.ProfileStopped): self.run_profile(transport) + self.assertEqual(len(transport.requests), 1) + + def test_secret_or_cross_scope_redirect_never_fetched(self): + for location in ("/wp-config.php", "https://other.test/robots.txt", "/robots.txt?token=PRIVATE", "http://example.test/robots.txt"): + transport = Transport([Response(status=302, headers={"Location": location}, no_read=True)]) + with self.subTest(location=location), self.assertRaises(E.ProfileStopped): self.run_profile(transport) + self.assertEqual(len(transport.requests), 1) + + def test_http_only_does_not_probe_https(self): + transport = Transport(success_responses()) + result = self.run_profile(transport, target="http://example.test/", services=[{"scheme": "http", "port": 80}]) + self.assertEqual(self.tls_calls, []) + self.assertEqual(result["checks"][-1]["outcome"], "not_selected") + self.assertTrue(all(url.startswith("http://") for _method, url, _timeout in transport.requests)) + + def test_conflicting_services_preserve_version_uncertainty(self): + transport = Transport(success_responses("8.3.35") + success_responses("8.3.36")) + result = self.run_profile(transport, services=[{"scheme": "http", "port": 80}, {"scheme": "https", "port": 443}]) + self.assertEqual(result["php_version_candidates"], ["8.3.35", "8.3.36"]) + self.assertEqual(result["php2shell_status"], "unknown") + self.assertEqual(len(result["checks"]), 8) + + def test_incomplete_selected_service_does_not_project_other_port_version(self): + transport = Transport(success_responses() + [Response(status=404), Response(status=404)]) + result = self.run_profile(transport, services=[{"scheme": "http", "port": 80}, {"scheme": "https", "port": 443}]) + self.assertEqual(result["php_version_candidates"], []) + self.assertEqual(result["php2shell_status"], "unknown") + self.assertEqual(result["notes"], []) + self.assertEqual(result["checks"][5]["outcome"], "unknown") + self.assertEqual(self.tls_calls, []) + core = {field.name: result[field.name] for field in fields(Result)} + SubprocessRunner.validate_result(core, TARGET, RequestOrigin.from_target(TARGET)[0]) + + def test_tls_evidence_cannot_include_raw_certificate_or_invalid_date(self): + bad = [dict(verified=True, protocol="TLSv1.3", not_after="2099-02-30T00:00:00Z"), + dict(verified=True, protocol="TLSv1.3", not_after="", raw_certificate="PRIVATE"), + dict(verified=False, protocol="TLSv1.3", not_after="")] + for value in bad: + with self.subTest(value=value), self.assertRaises(E.ProfileStopped): + self.run_profile(Transport(success_responses()), tls=lambda _service: value) + + def test_deadline_after_transport_closes_before_body_read(self): + clock, response = Clock(), Response(no_read=True) + class Slow(Transport): + def open(self, request, timeout=None): + clock.value += 601 + return super().open(request, timeout) + with self.assertRaises(E.ProfileStopped) as caught: self.run_profile(Slow([response]), clock=clock) + self.assertEqual(str(caught.exception), "active_time_limit") + self.assertTrue(response.closed) + + def test_aggregate_and_backward_clock_limits(self): + clock = Clock() + state = E._State(dict(P.profile_manifest()["limits"], aggregate_body_bytes=4), clock) + state.consume(b"1234") + with self.assertRaises(E.ProfileStopped): state.consume(b"5") + clock.value = -1 + with self.assertRaises(E.ProfileStopped): state.remaining() + + +if __name__ == "__main__": unittest.main() From 6a90bc32e338fc11e9e202623060817417af5f19 Mon Sep 17 00:00:00 2001 From: Timothy Wayne Gregg Date: Fri, 13 Nov 2026 23:06:46 -0500 Subject: [PATCH 05/19] build: preserve exact LF bytes of profile manifests --- .gitattributes | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitattributes b/.gitattributes index 30ad131..3633598 100644 --- a/.gitattributes +++ b/.gitattributes @@ -2,6 +2,7 @@ *.py text eol=lf *.md text eol=lf *.yml text eol=lf +integrations/profiles/*.json text eol=lf *.txt text eol=lf # Unified patch context includes a leading space before source indentation. deployment/*.patch -whitespace From 7362719f090361a5ecc3f42cc7c64873b3b390c5 Mon Sep 17 00:00:00 2001 From: Timothy Wayne Gregg Date: Fri, 13 Nov 2026 23:04:51 -0500 Subject: [PATCH 06/19] fix(worker): preserve safe profile stop categories without raw errors --- integrations/worker/scanner.py | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/integrations/worker/scanner.py b/integrations/worker/scanner.py index 15a30fb..4de56d6 100644 --- a/integrations/worker/scanner.py +++ b/integrations/worker/scanner.py @@ -117,17 +117,26 @@ def main() -> int: if task != "scan": return _control(task, scope, guard) if external: - from .external import execute_external, check_sensitive_response - from .network import ExternalPinnedOpener, ProfileMeter + from .external import execute_external, check_sensitive_response, ProfileStopped + from .network import ExternalPinnedOpener, ProfileMeter, BoundaryViolation # One meter and full selected-service scope survive every executor # factory call. No per-service reset, separate TLS channel or key. opener = ExternalPinnedOpener(external_scope, guard, profile_sha256=data["profile_sha256"], meter=ProfileMeter(), body_validator=lambda body, _headers: check_sensitive_response(body)) tls_url = "https://" + scope.hostname + "/" - result = execute_external(target, data["profile_sha256"], data["services"], - transport_factory=lambda _service: opener, - tls_probe=lambda _service: opener.inspect_tls(tls_url)) + try: + result = execute_external(target, data["profile_sha256"], data["services"], + transport_factory=lambda _service: opener, + tls_probe=lambda _service: opener.inspect_tls(tls_url)) + except AdmissionRefused: + return _write({"ok": False, "error": "admission_refused"}, 2) + except (ProfileStopped, BoundaryViolation) as exc: + # These two categories are safe operational signals. Every other + # refusal has one generic value, never target-provided text. + reason = getattr(exc, "reason", "") + error = reason if reason in {"possible_sensitive_indicator", "possible_compromise_indicator"} else "profile_stopped" + return _write({"ok": False, "error": error}, 2) return _write({"ok": True, "result": result}, 0) checker = Checker(timeout=10.0, delay=1.0, contact=CONTACT) checker.opener = PinnedOpener(scope, guard=guard) From d7df5d2c23fa3c301dc5cb5af29c2dc3cc5ae520 Mon Sep 17 00:00:00 2001 From: Timothy Wayne Gregg Date: Fri, 13 Nov 2026 23:06:23 -0500 Subject: [PATCH 07/19] fix(worker): retain legacy admission and isolated package compatibility --- integrations/worker/protocol.py | 12 ++++++++++-- test_external_worker_wire.py | 32 +++++++++++++++++++++++++++++++- 2 files changed, 41 insertions(+), 3 deletions(-) diff --git a/integrations/worker/protocol.py b/integrations/worker/protocol.py index b2c9e0b..57bac0c 100644 --- a/integrations/worker/protocol.py +++ b/integrations/worker/protocol.py @@ -15,7 +15,6 @@ from urllib.request import Request from .network import PinnedOpener, RequestOrigin -from .profiles import validate_binding API_SITE_URL = "https://complete.tech" @@ -67,6 +66,7 @@ def profile_fields(claim: JobClaim) -> dict: if claim.profile_sha256 or claim.services: raise ValueError("profile_binding_mismatch") return {} + from .profiles import validate_binding services = [{"scheme": scheme, "port": port} for scheme, port in claim.services] validate_binding(claim.profile_id, claim.profile_sha256, services) return {"profile_id": claim.profile_id, "profile_sha256": claim.profile_sha256, "services": services} @@ -152,6 +152,7 @@ def validate_claim(value, *, now: datetime | None = None) -> JobClaim | None: raise APIError("invalid_lease_bounds") profile_id, digest, services = "r2-header-v1", "", () if "profile_id" in data: + from .profiles import validate_binding try: profile_id, digest = data["profile_id"], data["profile_sha256"] services = validate_binding(profile_id, digest, data["services"]) @@ -249,7 +250,14 @@ def admit(self, claim: JobClaim, kind: str) -> dict: A short timeout keeps a slow store from stalling the supervisor; a timeout is a refusal. """ - if kind not in {"http", "tls", "probe", "control", "control_probe"}: + allowed = {"http", "probe", "control", "control_probe"} + if isinstance(claim, JobClaim) and claim.profile_id != "r2-header-v1": + try: + profile_fields(claim) + except (TypeError, ValueError) as exc: + raise APIError("invalid_profile_binding") from exc + allowed = {"http", "tls", "probe"} + if kind not in allowed: raise APIError("invalid_admission_kind") return self.post( "/ct-scan/v1/admit", diff --git a/test_external_worker_wire.py b/test_external_worker_wire.py index 41031c5..fdf4ff8 100644 --- a/test_external_worker_wire.py +++ b/test_external_worker_wire.py @@ -9,7 +9,7 @@ import unittest from unittest.mock import Mock, patch -from integrations.worker.protocol import APIError, profile_fields, validate_claim +from integrations.worker.protocol import APIClient, APIError, profile_fields, validate_claim from integrations.worker.profiles import PROFILE_ID, PROFILE_SHA256 from integrations.worker.runner import ScannerFailure, SubprocessRunner from integrations.worker.service import WorkerService @@ -92,6 +92,36 @@ def test_result_typed_observations_and_no_raw_sensitive_fields(self): with self.assertRaises(ScannerFailure): SubprocessRunner.validate_result(value, TARGET, scope, claim=claim()) + def test_real_executor_fixture_results_validate_for_all_selected_services(self): + from test_external_executor import ExecutorTests, Transport, Response, success_responses + fixture = ExecutorTests() + cases = [ + ((("https", 443),), success_responses()), + ((("http", 80),), success_responses()), + ((("http", 80), ("https", 443)), success_responses("8.3.35") + success_responses()), + ((("https", 443),), [Response(body=b"User-agent: *\nDisallow: /\n")]), + ((("http", 80), ("https", 443)), success_responses() + [Response(status=404), Response(status=404)]), + ] + for selected, responses in cases: + target = "http://example.test/" if len(selected) == 1 and selected[0][0] == "http" else TARGET + current = replace(claim(), target=target, scope_hash=hashlib.sha256(target.encode()).hexdigest(), services=selected) + value = fixture.run_profile(Transport(responses), target=target, services=profile_fields(current)["services"]) + with self.subTest(services=selected, status=value["status"]): + SubprocessRunner.validate_result(value, target, RequestOrigin.from_target(target)[0], claim=current) + + def test_tls_admission_requires_pinned_external_claim(self): + client = APIClient(b"x" * 32, transport=Mock()) + with patch.object(client, "post", return_value={"ok": True}) as post: + with self.assertRaises(APIError): + client.admit(_claim(), "tls") + post.assert_not_called() + client.admit(claim(), "tls") + self.assertEqual(post.call_args.args[1]["kind"], "tls") + with self.assertRaises(APIError): + client.admit(claim(), "control") + with self.assertRaises(APIError): + client.admit(replace(claim(), profile_sha256="0" * 64), "tls") + def test_running_recovery_posts_same_binding_without_scanner_or_claim(self): with tempfile.TemporaryDirectory() as root: spool = DurableSpool(root) From 3b0bbbd1c837cefed4a2c7b41187d8a5150a9de2 Mon Sep 17 00:00:00 2001 From: Timothy Wayne Gregg Date: Fri, 13 Nov 2026 23:07:05 -0500 Subject: [PATCH 08/19] test(worker): verify one external meter and bounded scanner dispatch --- test_external_worker_wire.py | 40 ++++++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/test_external_worker_wire.py b/test_external_worker_wire.py index fdf4ff8..75902b9 100644 --- a/test_external_worker_wire.py +++ b/test_external_worker_wire.py @@ -3,6 +3,7 @@ from dataclasses import replace from datetime import datetime, timezone import hashlib +import io import json from pathlib import Path import tempfile @@ -198,6 +199,45 @@ def test_service_failure_also_carries_profile_and_retry_never_rescans(self): self.assertEqual(payload["error"], "admission_refused") self.assertEqual(payload["profile_sha256"], PROFILE_SHA256) + def test_scanner_dispatch_shares_one_meter_transport_and_admission_channel(self): + from integrations.worker import scanner, network, external + selected = replace(claim(), services=(("http", 80), ("https", 443))) + message = {"target": TARGET, **profile_fields(selected)} + scope, meter, opener, channel, guard = Mock(), Mock(), Mock(), Mock(), Mock() + guard.kind = "external" + opener.inspect_tls.return_value = {"verified": True, "protocol": "TLSv1.3", "not_after": ""} + def execute(target, digest, services, *, transport_factory, tls_probe): + self.assertEqual((target, digest, services), (TARGET, PROFILE_SHA256, message["services"])) + self.assertIs(transport_factory(services[0]), opener) + self.assertIs(transport_factory(services[1]), opener) + self.assertEqual(tls_probe(services[1])["verified"], True) + return result() + class Boundary(Exception): + pass + with patch.object(scanner.os, "geteuid", return_value=65534), \ + patch.object(scanner.signal, "alarm"), \ + patch.object(scanner.sys, "stdin", Mock(buffer=io.BytesIO(json.dumps(message).encode()))), \ + patch.object(scanner, "_admission_channel", return_value=channel), \ + patch.object(scanner, "ChildAdmission", return_value=guard) as child, \ + patch.object(scanner, "_write", side_effect=lambda value, code: (value, code)), \ + patch.object(network, "ProfileScope", create=True) as scope_class, \ + patch.object(network, "ProfileMeter", return_value=meter, create=True) as meter_class, \ + patch.object(network, "ExternalPinnedOpener", return_value=opener, create=True) as opener_class, \ + patch.object(network, "BoundaryViolation", Boundary, create=True), \ + patch.object(external, "execute_external", side_effect=execute): + scope_class.from_target.return_value = (scope, TARGET) + value, code = scanner.main() + self.assertEqual(code, 0) + self.assertTrue(value["ok"]) + child.assert_called_once_with(channel, kind="external") + meter_class.assert_called_once_with() + self.assertEqual(opener_class.call_args.kwargs["profile_sha256"], PROFILE_SHA256) + self.assertIs(opener_class.call_args.kwargs["meter"], meter) + validator = opener_class.call_args.kwargs["body_validator"] + with self.assertRaises(external.ProfileStopped): + validator(b"-----BEGIN PRIVATE KEY-----", {}) + opener.inspect_tls.assert_called_once_with("https://example.test/") + def test_start_marker_freezes_binding_before_ambiguous_start(self): with tempfile.TemporaryDirectory() as root: spool = DurableSpool(root) From c15657d18a6ea61d81afc4bbf7f475ac48fc6b18 Mon Sep 17 00:00:00 2001 From: Timothy Wayne Gregg Date: Fri, 13 Nov 2026 23:12:20 -0500 Subject: [PATCH 09/19] Add manifest-bound external transport and shared admission boundaries --- integrations/worker/admission.py | 86 ++++- integrations/worker/network.py | 556 +++++++++++++++++++++++++++++ integrations/worker/network_dns.py | 60 ++++ test_external_boundary.py | 441 +++++++++++++++++++++++ 4 files changed, 1131 insertions(+), 12 deletions(-) create mode 100644 integrations/worker/network_dns.py create mode 100644 test_external_boundary.py diff --git a/integrations/worker/admission.py b/integrations/worker/admission.py index c6a92c6..d72cc64 100644 --- a/integrations/worker/admission.py +++ b/integrations/worker/admission.py @@ -98,6 +98,29 @@ def parse_grant(value) -> Grant: ) +def parse_external_grant(value, *, kind, profile_sha256) -> Grant: + """Validate the additional immutable-profile reservation, never an r2 grant.""" + expected = {"ok", "admitted", "sequence", "valid_ms", "timeout_ms", "http_remaining", + "kind", "profile_sha256", "tls_remaining", "body_bytes_remaining", "active_ms_remaining"} + if (type(value) is not dict or set(value) != expected or value.get("ok") is not True + or value.get("admitted") is not True or kind not in {"http", "tls"} + or value.get("kind") != kind or value.get("profile_sha256") != profile_sha256): + raise AdmissionDenied("admission_response_invalid") + keys = ("sequence", "valid_ms", "timeout_ms", "http_remaining", "tls_remaining", + "body_bytes_remaining", "active_ms_remaining") + if not all(_is_int(value[key]) for key in keys): + raise AdmissionDenied("admission_response_invalid") + if (not 1 <= value["sequence"] <= 120 or not 1 <= value["valid_ms"] <= 1000 + or not 1 <= value["timeout_ms"] <= 10000 or not 0 <= value["http_remaining"] <= 100 + or not 0 <= value["tls_remaining"] <= 20 or not 0 <= value["body_bytes_remaining"] <= 5242880 + or not 0 <= value["active_ms_remaining"] <= 600000 + or (kind == "http" and value["http_remaining"] == 100) + or (kind == "tls" and value["tls_remaining"] == 20)): + raise AdmissionDenied("admission_response_invalid") + return Grant(value["sequence"], value["valid_ms"] / 1000.0, value["timeout_ms"] / 1000.0, + value["http_remaining"] if kind == "http" else value["tls_remaining"]) + + def parse_refusal(error: APIError) -> AdmissionDenied: """Turn a non-2xx answer into a refusal; unknown or malformed answers stay refusals.""" payload = getattr(error, "payload", None) @@ -117,8 +140,14 @@ class AdmissionGate: """Supervisor-side client for the WordPress admission route.""" def __init__(self, api, claim, *, kind="http", stop_event=None, deadline=None, clock=time.time, monotonic=time.monotonic): - if kind not in {"http", "control"}: + if kind not in {"http", "control", "external"}: raise ValueError("invalid_admission_kind") + if kind == "external": + from .profiles import PROFILE_ID, validate_binding + if getattr(claim, "profile_id", None) != PROFILE_ID: + raise ValueError("invalid_admission_profile") + services = [{"scheme": scheme, "port": port} for scheme, port in claim.services] + validate_binding(claim.profile_id, claim.profile_sha256, services) self.api = api self.claim = claim self.kind = kind @@ -127,6 +156,8 @@ def __init__(self, api, claim, *, kind="http", stop_event=None, deadline=None, c self.deadline = deadline self._clock = clock self._monotonic = monotonic + self._external_sequence = 0 + self._external_remaining = None def _wait(self, seconds: float): if self.stop_event is not None: @@ -172,15 +203,34 @@ def run(): raise AdmissionDenied("guard_unavailable") from error return box["value"] - def admit(self) -> Grant: + def admit(self, *, kind=None) -> Grant: """Reserve exactly one request, or raise AdmissionDenied. Retries are bounded.""" - deadline = self._monotonic() + MAX_ADMIT_WAIT_SECONDS + external = self.kind == "external" + if external: + kind = "http" if kind is None else kind + if kind not in {"http", "tls"}: + raise AdmissionDenied("admission_kind_invalid") + elif kind is not None and kind != self.kind: + raise AdmissionDenied("admission_kind_invalid") + else: + kind = self.kind + deadline = self._monotonic() + (15.0 if external else MAX_ADMIT_WAIT_SECONDS) stale = 0 while True: self._check_local() started = self._monotonic() try: - grant = replace(parse_grant(self._call(self.kind)), issued_at=started) + response = self._call(kind) + if external: + grant = parse_external_grant(response, kind=kind, profile_sha256=self.claim.profile_sha256) + remaining = tuple(response[key] for key in ("http_remaining", "tls_remaining", "body_bytes_remaining", "active_ms_remaining")) + if (grant.sequence <= self._external_sequence or (self._external_remaining is not None + and any(new > old for new, old in zip(remaining, self._external_remaining)))): + raise AdmissionDenied("admission_reservation_replayed") + self._external_sequence, self._external_remaining = grant.sequence, remaining + else: + grant = parse_grant(response) + grant = replace(grant, issued_at=started) if self._monotonic() - started > MAX_GRANT_AGE_SECONDS: # WordPress answered, but too slowly for the one-second lifetime to be useful. stale += 1 @@ -205,7 +255,7 @@ def probe(self): A busy queue lock is retried briefly within one call budget; if it stays busy the probe reports ``queue_busy``, which the pump treats as inconclusive. """ - probe_kind = "probe" if self.kind == "http" else "control_probe" + probe_kind = "control_probe" if self.kind == "control" else "probe" busy_until = self._monotonic() + CALL_BUDGET_SECONDS while True: self._check_local() @@ -233,7 +283,9 @@ class AdmissionPump: """ def __init__(self, sock, gate, *, kind="http", monotonic=time.monotonic): - if kind not in {"http", "control"}: + if kind not in {"http", "control", "external"}: + raise ValueError("invalid_admission_kind") + if kind == "external" and getattr(gate, "kind", None) != "external": raise ValueError("invalid_admission_kind") self.sock = sock self.gate = gate @@ -282,13 +334,15 @@ def _handle(self, line: bytes): self._inflight_until = None self._inflight_n = None return None - if set(message) == {"op", "kind", "n"} and message["op"] == "admit" and message["kind"] == self.kind: + message_kind = message.get("kind") + allowed_kind = (type(message_kind) is str and message_kind in {"http", "tls"}) if self.kind == "external" else message_kind == self.kind + if set(message) == {"op", "kind", "n"} and message["op"] == "admit" and allowed_kind: if self.inflight or number != self._last_n + 1: # One connection at a time and strictly sequential requests. return self._fail("admission_protocol_violation") self._last_n = number try: - grant = self.gate.admit() + grant = self.gate.admit(kind=message["kind"]) if self.kind == "external" else self.gate.admit() except AdmissionDenied as denied: self._send({"ok": False, "n": number, "reason": denied.slug}) return self._fail(denied.slug) @@ -392,7 +446,7 @@ class ChildAdmission: """Scanner-side client of the supervisor socket. Holds no key and no state of record.""" def __init__(self, sock, *, kind="http", monotonic=time.monotonic): - if kind not in {"http", "control"}: + if kind not in {"http", "control", "external"}: raise ValueError("invalid_admission_kind") self.sock = sock self.kind = kind @@ -414,7 +468,7 @@ def _poison(self, reason: str): raise AdmissionRefused(reason) def _read_line(self) -> bytes: - deadline = self._monotonic() + CHILD_REPLY_TIMEOUT_SECONDS + deadline = self._monotonic() + (20.0 if self.kind == "external" else CHILD_REPLY_TIMEOUT_SECONDS) data = bytearray() while True: remaining = deadline - self._monotonic() @@ -433,7 +487,15 @@ def _read_line(self) -> bytes: if len(data) > MAX_MESSAGE_BYTES: self._poison("admission_reply_invalid") - def admit(self): + def admit(self, *, kind=None): + if self.kind == "external": + kind = "http" if kind is None else kind + if kind not in {"http", "tls"}: + self._poison("admission_kind_invalid") + elif kind is not None and kind != self.kind: + self._poison("admission_kind_invalid") + else: + kind = self.kind if self._dead: raise AdmissionRefused("admission_channel_failed") if self._outstanding: @@ -441,7 +503,7 @@ def admit(self): self._n += 1 try: self.sock.settimeout(5.0) - self.sock.sendall(json.dumps({"op": "admit", "kind": self.kind, "n": self._n}, separators=(",", ":")).encode("ascii") + b"\n") + self.sock.sendall(json.dumps({"op": "admit", "kind": kind, "n": self._n}, separators=(",", ":")).encode("ascii") + b"\n") except OSError: self._poison("admission_channel_failed") try: diff --git a/integrations/worker/network.py b/integrations/worker/network.py index 4461ce6..33d3b58 100644 --- a/integrations/worker/network.py +++ b/integrations/worker/network.py @@ -7,12 +7,161 @@ import re import socket import ssl +from email.message import Message +import threading +import time +import zlib from dataclasses import dataclass, field from urllib.parse import urlsplit, urlunsplit + MAX_URL_LENGTH = 4096 +_BOUNDARY_REASONS = frozenset({ + "profile_connection_outstanding", "profile_rate_exceeded", "profile_duration_exhausted", + "profile_http_budget_exhausted", "profile_tls_budget_exhausted", "profile_kind_invalid", + "profile_byte_budget_exhausted", "profile_scope_changed", "profile_headers_too_large", + "profile_request_timeout", "profile_peer_mismatch", "profile_tunnel_forbidden", + "profile_sensitive_indicator", "profile_access_challenge", "profile_compromise_indicator", + "profile_access_restricted", "profile_service_unavailable", "profile_encoding_unsupported", + "profile_encoding_invalid", "profile_body_too_large", "profile_response_failed", + "profile_response_closed", "profile_content_rejected", "profile_timeout_invalid", + "profile_method_forbidden", "profile_connection_failed", "profile_tls_out_of_scope", + "profile_tls_metadata_invalid", "profile_tls_failed", "profile_dns_invalid", + "profile_dns_failed", "profile_dns_timeout", "profile_redirect_budget_exhausted", +}) + + +class BoundaryViolation(Exception): + """A log-safe profile refusal which the public checker must not swallow.""" + + def __init__(self, reason): + self.reason = reason if type(reason) is str and reason in _BOUNDARY_REASONS else "profile_boundary_refused" + super().__init__(self.reason) + + +@dataclass(frozen=True) +class ProfileScope: + """The explicit service subset signed for external-web-v1, never inferred.""" + + hostname: str + services: tuple[tuple[str, int], ...] + + @classmethod + def from_target(cls, target, services): + origin, normalized = RequestOrigin.from_target(target) + if target != normalized or not isinstance(services, (list, tuple)) or not services: + raise ValueError("invalid_profile_scope") + checked = [] + for item in services: + if not isinstance(item, dict) or set(item) != {"scheme", "port"}: + raise ValueError("invalid_profile_scope") + scheme, port = item["scheme"], item["port"] + if type(scheme) is not str or type(port) is not int or (scheme, port) not in {("http", 80), ("https", 443)}: + raise ValueError("invalid_profile_scope") + checked.append((scheme, port)) + if checked != sorted(set(checked)) or (origin.initial_scheme, origin.initial_port) not in checked: + raise ValueError("invalid_profile_scope") + return cls(origin.hostname, tuple(checked)), normalized + + def validate_request(self, value, *, mark_upgrade=True): + if not isinstance(value, str) or not value or len(value) > MAX_URL_LENGTH: + raise ValueError("request_out_of_scope") + if any(char.isspace() or ord(char) < 32 or ord(char) == 127 for char in value): + raise ValueError("request_out_of_scope") + try: + parts = urlsplit(value) + host = _canonical_host(parts.hostname or "") + service = (parts.scheme, _effective_port(parts)) + except ValueError as exc: + raise ValueError("request_out_of_scope") from exc + if (host != self.hostname or service not in self.services + or parts.username is not None or parts.password is not None + or parts.query or parts.fragment or parts.path not in {"", "/", "/robots.txt"}): + raise ValueError("request_out_of_scope") + return service[0], service[1], host, parts.path or "/" + + +class ProfileMeter: + """Local second boundary; durable reservations remain the server's authority. + + Counts never reset or refund after an attempted admission. A refused or + interrupted attempt cannot be retried with a fresh local budget. + """ + + def __init__(self, *, monotonic=time.monotonic): + self._clock = monotonic + self._lock = threading.RLock() + self.started = monotonic() + self.last = None + self.http_used = self.tls_used = self.body_used = self.wire_used = 0 + self.redirects = 0 + self.inflight = False + self.failure = None + self.binding = None + + def bind(self, value): + with self._lock: + if self.binding is not None and self.binding != value: + self.refuse("profile_scope_changed") + self.binding = value + + def refuse(self, reason): + self.failure = self.failure or BoundaryViolation(reason).reason + raise BoundaryViolation(self.failure) + + def check(self): + if self.failure: + raise BoundaryViolation(self.failure) + elapsed = self._clock() - self.started + if elapsed < 0 or elapsed >= 600: + self.refuse("profile_duration_exhausted") + + def begin(self, kind): + with self._lock: + self.check() + if self.inflight: + self.refuse("profile_connection_outstanding") + now = self._clock() + if self.last is not None and now - self.last < 1: + self.refuse("profile_rate_exceeded") + if kind == "http": + if self.http_used >= 100: + self.refuse("profile_http_budget_exhausted") + self.http_used += 1 + elif kind == "tls": + if self.tls_used >= 20: + self.refuse("profile_tls_budget_exhausted") + self.tls_used += 1 + else: + self.refuse("profile_kind_invalid") + self.inflight = True + self.last = now + + def admitted(self): + with self._lock: + self.last = self._clock() + + def redirect(self): + with self._lock: + self.check() + self.redirects += 1 + if self.redirects > 3: + self.refuse("profile_redirect_budget_exhausted") + + def consume(self, count, *, wire=False): + with self._lock: + self.check() + name = "wire_used" if wire else "body_used" + if type(count) is not int or count < 0 or getattr(self, name) + count > 5 * 1024 * 1024: + self.refuse("profile_byte_budget_exhausted") + setattr(self, name, getattr(self, name) + count) + + def done(self): + with self._lock: + self.inflight = False + def _canonical_host(value: str) -> str: if not value or "%" in value or value.endswith("."): @@ -355,3 +504,410 @@ def open(self, request, timeout=None): if guard is not None: guard.done() raise + + +class _HeaderReader: + """Count status/header bytes while http.client is parsing, before allocation.""" + + def __init__(self, stream, maximum): + self.stream, self.maximum, self.used = stream, maximum, 0 + + def readline(self, size=-1): + remaining = self.maximum - self.used + 1 + value = self.stream.readline(remaining if size < 0 else min(size, remaining)) + self.used += len(value) + if self.used > self.maximum: + raise BoundaryViolation("profile_headers_too_large") + return value + + def __getattr__(self, name): + return getattr(self.stream, name) + + +class _BoundedHTTPResponse(http.client.HTTPResponse): + def begin(self): + original = self.fp + self.fp = _HeaderReader(original, 32768) + try: + super().begin() + finally: + # Keep the same buffered stream and its already received bytes. + self.fp = original + + +class _RequestDeadline: + """A total deadline holding a duplicate raw FD, even after getresponse(). + + http.client can relinquish connection.sock while the response stream still + owns it. Shutting down the duplicate interrupts header/body/TLS trickles + without taking a buffered reader lock in the timer thread. + """ + + def __init__(self, seconds, *, monotonic=time.monotonic): + self._clock = monotonic + self.until = monotonic() + seconds + self.expired = False + self._closed = False + self._socket = None + self._lock = threading.Lock() + self._timer = threading.Timer(seconds, self._expire) + self._timer.daemon = True + self._timer.start() + + def _expire(self): + with self._lock: + if self._closed: + return + self.expired = True + sock = self._socket + if sock is not None: + try: + sock.shutdown(socket.SHUT_RDWR) + except OSError: + pass + + def attach(self, sock): + duplicate = sock.dup() + with self._lock: + if self._closed or self.expired: + duplicate.close() + raise BoundaryViolation("profile_request_timeout") + self._socket = duplicate + + def remaining(self): + remaining = self.until - self._clock() + if self.expired or remaining <= 0: + raise BoundaryViolation("profile_request_timeout") + return min(10.0, remaining) + + def close(self): + with self._lock: + if self._closed: + return + self._closed = True + sock, self._socket = self._socket, None + self._timer.cancel() + if sock is not None: + sock.close() + + +def _profile_socket(connection, deadline): + """Publish the raw socket to the deadline before any connect/handshake.""" + connection._before_connect() + sock = socket.socket(connection._pinned_family, socket.SOCK_STREAM) + connection.sock = sock + try: + sock.settimeout(deadline.remaining()) + deadline.attach(sock) + address = connection._pinned_address + target = (address, connection.port, 0, 0) if connection._pinned_family == socket.AF_INET6 else (address, connection.port) + sock.connect(target) + if ipaddress.ip_address(sock.getpeername()[0]) != ipaddress.ip_address(address): + raise BoundaryViolation("profile_peer_mismatch") + deadline.remaining() + return sock + except BaseException: + sock.close() + connection.sock = None + raise + + +class _ProfileHTTPConnection(PinnedHTTPConnection): + response_class = _BoundedHTTPResponse + + def connect(self): + self.sock = _profile_socket(self, self._deadline) + + +class _ProfileHTTPSConnection(PinnedHTTPSConnection): + response_class = _BoundedHTTPResponse + + def connect(self): + if self._tunnel_host: + raise BoundaryViolation("profile_tunnel_forbidden") + raw = _profile_socket(self, self._deadline) + try: + self.sock = self._context.wrap_socket(raw, server_hostname=self.host) + self._deadline.remaining() + except BaseException: + raw.close() + self.sock = None + raise + + +def _default_content_validator(body, headers): + """Stop on explicit indicators without classifying or retaining their values.""" + header_text = "\n".join(str(name) + ":" + str(value) for name, value in headers.items()).encode("utf-8", "replace") + lower = body.lower() + b"\n" + header_text.lower() + if (b"-----begin " in lower and b"private key-----" in lower) or b"aws_secret_access_key" in lower: + raise BoundaryViolation("profile_sensitive_indicator") + if b"cf-chl-" in lower or b"__cf_chl" in lower: + raise BoundaryViolation("profile_access_challenge") + if re.search(rb"\bhacked\s+by\b", lower): + raise BoundaryViolation("profile_compromise_indicator") + + +class _ExternalResponse: + def __init__(self, response, connection, deadline, meter, done, *, method, validator): + self._response, self._connection, self._deadline = response, connection, deadline + self._meter, self._done = meter, done + self._method, self._validator = method, validator + self._closed = False + self._buffer = None + self._position = 0 + self.code = response.status + self.headers = Message() + for name, value in response.headers.items(): + self.headers[name] = value + self.original_headers = tuple(response.headers.items()) + self.decoded = False + + def _load(self): + if self._buffer is not None: + return + try: + self._deadline.remaining() + if self.code in {401, 403, 407, 429}: + raise BoundaryViolation("profile_access_restricted") + if self.code >= 500: + raise BoundaryViolation("profile_service_unavailable") + encodings = self.headers.get_all("Content-Encoding", []) + if len(encodings) > 1: + raise BoundaryViolation("profile_encoding_unsupported") + encoding = (encodings[0] if encodings else "identity").strip().lower() + if encoding not in {"", "identity", "gzip"}: + raise BoundaryViolation("profile_encoding_unsupported") + if self._method == "HEAD": + # No body read, including when a server incorrectly supplies one. + self._validator(b"", self.headers) + self._buffer = b"" + self.decoded = True + return + decoder = zlib.decompressobj(16 + zlib.MAX_WBITS) if encoding == "gzip" else None + result = bytearray() + wire = 0 + while True: + self._deadline.remaining() + chunk = self._response.read(min(8192, 262144 - wire + 1)) + self._deadline.remaining() + if not chunk: + break + wire += len(chunk) + self._meter.consume(len(chunk), wire=True) + if wire > 262144: + raise BoundaryViolation("profile_body_too_large") + decoded = decoder.decompress(chunk, 262144 - len(result) + 1) if decoder else chunk + self._meter.consume(len(decoded)) + if len(result) + len(decoded) > 262144: + raise BoundaryViolation("profile_body_too_large") + result.extend(decoded) + if decoder and (decoder.unconsumed_tail or decoder.unused_data): + raise BoundaryViolation("profile_encoding_invalid") + if decoder and not decoder.eof: + raise BoundaryViolation("profile_encoding_invalid") + self._validator(bytes(result), self.headers) + self._buffer = bytes(result) + if encoding == "gzip": + # Checker receives decoded bytes; it must not attempt a second + # decode or reject a decoded robots file as compressed text. + del self.headers["Content-Encoding"] + del self.headers["Content-Length"] + self.headers["Content-Encoding"] = "identity" + self.decoded = True + except BaseException as exc: + self._buffer = None + reason = exc.reason if isinstance(exc, BoundaryViolation) else "profile_response_failed" + self._meter.failure = self._meter.failure or reason + try: + self.close() + finally: + self._meter.refuse(reason) + + def read(self, size=-1): + if self._closed: + raise BoundaryViolation("profile_response_closed") + if type(size) is not int or size < -1: + raise ValueError("invalid_read_size") + self._load() + stop = len(self._buffer) if size < 0 else min(len(self._buffer), self._position + size) + value = self._buffer[self._position:stop] + self._position = stop + return value + + def close(self): + if self._closed: + return + self._closed = True + try: + self._response.close() + finally: + try: + self._connection.close() + finally: + self._deadline.close() + self._done() + self._buffer = None + + def __enter__(self): + return self + + def __exit__(self, *_exc): + self.close() + + +class ExternalPinnedOpener: + """Manifest-bound transport; separate from the frozen r2-header-v1 adapter.""" + + def __init__(self, scope, guard, *, profile_sha256, meter, body_validator=None, ssl_context=None, + monotonic=time.monotonic, wait=time.sleep): + from .profiles import PROFILE_ID, validate_binding + if not isinstance(scope, ProfileScope) or not isinstance(meter, ProfileMeter): + raise ValueError("invalid_profile_transport") + if _canonical_host(scope.hostname) != scope.hostname or getattr(guard, "kind", None) != "external": + raise ValueError("invalid_profile_transport") + services = [{"scheme": scheme, "port": port} for scheme, port in scope.services] + validate_binding(PROFILE_ID, profile_sha256, services) + self.scope, self.guard, self.meter = scope, guard, meter + self.ssl_context = ssl_context or ssl.create_default_context() + if not self.ssl_context.check_hostname or self.ssl_context.verify_mode != ssl.CERT_REQUIRED: + raise ValueError("profile_tls_verification_required") + def validate(body, headers): + _default_content_validator(body, headers) + if body_validator is not None and body_validator(body, headers) is False: + raise BoundaryViolation("profile_content_rejected") + self._validator = validate + self._clock, self._wait = monotonic, wait + self.meter.bind((scope.hostname, scope.services, profile_sha256)) + + def _begin(self, kind, timeout): + self.meter.check() + if self.meter.inflight: + self.meter.refuse("profile_connection_outstanding") + # Bounded pacing before requesting the authoritative reservation. Stop + # admission is still checked afterwards, so waiting cannot grant traffic. + while self.meter.last is not None: + self.meter.check() + remaining = 1 - (self._clock() - self.meter.last) + if remaining <= 0: + break + self._wait(min(0.05, remaining)) + self.meter.begin(kind) + if timeout is None: + timeout = 10.0 + if type(timeout) not in {int, float} or not 0 < timeout <= 10: + self.meter.done() + self.meter.refuse("profile_timeout_invalid") + deadline = None + try: + self.guard.admit(kind=kind) + # The server may conservatively hold the previous reservation for + # eleven seconds. Waiting for permission does not start a target + # request. The supervisor still kills admitted work at ten seconds + # from the authoritative grant's issue time. + deadline = _RequestDeadline(timeout, monotonic=self._clock) + deadline.remaining() + self.guard.ensure_fresh() + self.meter.admitted() + return deadline + except BaseException: + if deadline is not None: + deadline.close() + self.guard.done() + self.meter.done() + raise + + def _addresses(self, host, port, deadline): + from .network_dns import DNSFailure, resolve_answers + try: + answers = resolve_answers(host, port, deadline.remaining()) + return _public_addresses(host, port, lambda *_args, **_kw: answers) + except DNSFailure as exc: + raise BoundaryViolation(str(exc)) from exc + + def _connection(self, scheme, port, host, deadline): + answers = self._addresses(host, port, deadline) + family, address = answers[0] + connection_type = _ProfileHTTPSConnection if scheme == "https" else _ProfileHTTPConnection + options = dict(pinned_family=family, pinned_address=address, timeout=deadline.remaining(), + before_connect=self.guard.ensure_fresh) + if scheme == "https": + options["context"] = self.ssl_context + connection = connection_type(host, port, **options) + connection._deadline = deadline + return connection + + def _done(self): + try: + self.guard.done() + finally: + self.meter.done() + + def open(self, request, timeout=None): + url = request.full_url if hasattr(request, "full_url") else str(request) + scheme, port, host, target = self.scope.validate_request(url) + method = request.get_method() if hasattr(request, "get_method") else "GET" + if method not in {"GET", "HEAD"} or getattr(request, "data", None) is not None: + raise BoundaryViolation("profile_method_forbidden") + if method == "HEAD" and target != "/": + raise BoundaryViolation("profile_method_forbidden") + deadline = self._begin("http", timeout) + connection = None + try: + connection = self._connection(scheme, port, host, deadline) + supplied = dict(request.header_items()) if hasattr(request, "header_items") else {} + # Never forward credentials, cookies, proxies, caller Host or body + # negotiation from the application into a target request. + allowed = {name: value for name, value in supplied.items() if name.lower() in {"user-agent", "accept"}} + allowed.update({"Connection": "close", "Accept-Encoding": "identity"}) + connection.request(method, target, headers=allowed) + response = connection.getresponse() + deadline.remaining() + if response.status in {301, 302, 303, 307, 308}: + self.meter.redirect() + return _ExternalResponse(response, connection, deadline, self.meter, self._done, + method=method, validator=self._validator) + except BaseException as exc: + reason = "profile_request_timeout" if deadline.expired else ( + exc.reason if isinstance(exc, BoundaryViolation) else "profile_connection_failed") + self.meter.failure = self.meter.failure or BoundaryViolation(reason).reason + try: + if connection is not None: + connection.close() + finally: + try: + deadline.close() + finally: + try: + self._done() + finally: + self.meter.refuse(reason) + + def inspect_tls(self, url, timeout=None): + scheme, port, host, target = self.scope.validate_request(url) + if scheme != "https" or target != "/": + raise BoundaryViolation("profile_tls_out_of_scope") + deadline = self._begin("tls", timeout) + connection = None + try: + connection = self._connection(scheme, port, host, deadline) + connection.connect() + certificate = connection.sock.getpeercert() + protocol = connection.sock.version() + if type(protocol) is not str or not protocol: + raise BoundaryViolation("profile_tls_metadata_invalid") + expiry = certificate.get("notAfter") + not_after = "" + if type(expiry) is str: + from datetime import datetime, timezone + not_after = datetime.fromtimestamp(ssl.cert_time_to_seconds(expiry), timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") + deadline.remaining() + return {"verified": True, "protocol": protocol, "not_after": not_after} + except BaseException as exc: + reason = "profile_request_timeout" if deadline.expired else ( + exc.reason if isinstance(exc, BoundaryViolation) else "profile_tls_failed") + self.meter.refuse(reason) + finally: + if connection is not None: + connection.close() + deadline.close() + self._done() diff --git a/integrations/worker/network_dns.py b/integrations/worker/network_dns.py new file mode 100644 index 0000000..5aebc20 --- /dev/null +++ b/integrations/worker/network_dns.py @@ -0,0 +1,60 @@ +"""Bounded system DNS lookup for the isolated external profile scanner. + +The helper receives no admission descriptor, authentication key or configurable +resolver. Its process remains in the scanner's process group for outer cleanup. +""" + +from __future__ import annotations + +import json +import os +import subprocess +import sys + + +_LOOKUP = """import json,socket,sys +try: + a=socket.getaddrinfo(sys.argv[1],int(sys.argv[2]),type=socket.SOCK_STREAM) + if not 016384: raise ValueError() + sys.stdout.buffer.write(b) +except BaseException: + sys.exit(2) +""" + + +class DNSFailure(Exception): + pass + + +def resolve_answers(host, port, timeout): + """Resolve once in a killable process; never leave a timed-out resolver alive.""" + if (not isinstance(host, str) or not host.isascii() or not host + or type(port) is not int or port not in {80, 443} + or type(timeout) not in {int, float} or not 0 < timeout <= 10): + raise DNSFailure("profile_dns_invalid") + process = None + try: + process = subprocess.Popen( + [sys.executable, "-I", "-c", _LOOKUP, host, str(port)], + stdin=subprocess.DEVNULL, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, + close_fds=True, env={"PATH": os.defpath, "LC_ALL": "C"}, + ) + try: + output, _ = process.communicate(timeout=timeout) + except BaseException: + process.kill() + process.communicate() + raise + if process.returncode or not 0 < len(output) <= 16384: + raise DNSFailure("profile_dns_failed") + value = json.loads(output.decode("ascii")) + if type(value) is not list or not 0 < len(value) <= 256: + raise DNSFailure("profile_dns_failed") + return value + except subprocess.TimeoutExpired as exc: + raise DNSFailure("profile_dns_timeout") from exc + except (OSError, ValueError, UnicodeError) as exc: + raise DNSFailure("profile_dns_failed") from exc + diff --git a/test_external_boundary.py b/test_external_boundary.py new file mode 100644 index 0000000..e7b1882 --- /dev/null +++ b/test_external_boundary.py @@ -0,0 +1,441 @@ +"""Synthetic-only external-profile transport/admission regressions.""" + +import gzip +import io +import json +import os +from email.message import Message +import socket +import socketserver +import subprocess +import threading +import time +import unittest +from datetime import datetime, timezone +from types import SimpleNamespace +from unittest import mock +from urllib.request import Request + +from integrations.worker import admission, network, network_dns +from integrations.worker.profiles import PROFILE_SHA256 + + +SERVICES = [{"scheme": "http", "port": 80}, {"scheme": "https", "port": 443}] +TARGET = "http://fixture.example/" +PUBLIC_ANSWERS = [[socket.AF_INET, socket.SOCK_STREAM, 6, "", ["93.184.216.34", 80]]] + + +class Guard: + kind = "external" + + def __init__(self): + self.kinds = [] + self.finished = 0 + + def admit(self, *, kind): + self.kinds.append(kind) + + def ensure_fresh(self): + pass + + def done(self): + self.finished += 1 + + +class Response: + def __init__(self, body=b"safe", status=200, headers=None): + self.status = status + self.headers = Message() + for name, value in (headers or {}).items(): + self.headers[name] = value + self.data = io.BytesIO(body) + self.closed = False + self.reads = 0 + + def read(self, amount=-1): + self.reads += 1 + return self.data.read(amount) + + def close(self): + self.closed = True + + +class Connection: + def __init__(self, response): + self.response = response + self.requests = [] + self.closed = False + + def request(self, *args, **kwargs): + self.requests.append((args, kwargs)) + + def getresponse(self): + return self.response + + def close(self): + self.closed = True + + +def opener(*, guard=None, meter=None, **kwargs): + scope, _ = network.ProfileScope.from_target(TARGET, SERVICES) + return network.ExternalPinnedOpener(scope, guard or Guard(), profile_sha256=PROFILE_SHA256, + meter=meter or network.ProfileMeter(), **kwargs) + + +class ProfileScopeTests(unittest.TestCase): + def test_explicit_services_and_safe_paths_only(self): + scope, canonical = network.ProfileScope.from_target(TARGET, SERVICES) + self.assertEqual(canonical, TARGET) + self.assertEqual(scope.validate_request("https://fixture.example/robots.txt"), + ("https", 443, "fixture.example", "/robots.txt")) + for target in ("https://www.fixture.example/", "https://fixture.example:444/", + "https://fixture.example/wp-config.php", "https://fixture.example/%2f", + "https://fixture.example/?x=1", "https://fixture.example/#token", + "https://u:p@fixture.example/"): + with self.subTest(target=target), self.assertRaises(ValueError): + scope.validate_request(target) + + def test_invalid_services_and_binding_are_refused(self): + for value in ([], list(reversed(SERVICES)), SERVICES + SERVICES, [{"scheme": [], "port": 80}], + [{"scheme": "http", "port": True}], [{"scheme": "http", "port": 81}]): + with self.assertRaises(ValueError): + network.ProfileScope.from_target(TARGET, value) + scope, _ = network.ProfileScope.from_target(TARGET, SERVICES) + with self.assertRaises(ValueError): + network.ExternalPinnedOpener(scope, Guard(), profile_sha256="0" * 64, + meter=network.ProfileMeter()) + + def test_shared_meter_cannot_change_scope(self): + meter = network.ProfileMeter() + opener(meter=meter) + other, _ = network.ProfileScope.from_target("http://other.example/", SERVICES) + with self.assertRaises(network.BoundaryViolation): + network.ExternalPinnedOpener(other, Guard(), profile_sha256=PROFILE_SHA256, meter=meter) + + +class BodyAndHeaderTests(unittest.TestCase): + def use(self, response, *, request=None, **kwargs): + transport = opener(**kwargs) + connection = Connection(response) + patcher = mock.patch.object(transport, "_connection", return_value=connection) + patcher.start() + self.addCleanup(patcher.stop) + return transport, connection, transport.open(request or Request(TARGET)) + + def test_header_parser_enforces_aggregate_limit_before_allocation(self): + raw = io.BytesIO(b"HTTP/1.1 200 OK\r\nX-Big: " + b"a" * 40000 + b"\r\n\r\n") + sock = SimpleNamespace(makefile=lambda _mode: raw) + response = network._BoundedHTTPResponse(sock) + with self.assertRaises(network.BoundaryViolation): + response.begin() + self.assertLessEqual(raw.tell(), 32769) + + def test_gzip_is_decoded_bounded_and_headers_normalized(self): + response = Response(gzip.compress(b"User-agent: *\nAllow: /\n"), + headers={"Content-Encoding": "gzip", "Content-Length": "42"}) + transport, connection, stream = self.use(response) + with stream: + self.assertEqual(stream.read(4) + stream.read(), b"User-agent: *\nAllow: /\n") + self.assertEqual(stream.headers["Content-Encoding"], "identity") + self.assertIsNone(stream.headers["Content-Length"]) + self.assertTrue(response.closed and connection.closed) + self.assertFalse(transport.meter.inflight) + + def test_gzip_bomb_and_extra_member_never_return_prefix(self): + for body in (gzip.compress(b"x" * 262145), gzip.compress(b"safe") + gzip.compress(b"hidden")): + transport, connection, stream = self.use(Response(body, headers={"Content-Encoding": "gzip"})) + with self.assertRaises(network.BoundaryViolation): + stream.read(1) + self.assertTrue(connection.closed) + self.assertIsNone(stream._buffer) + self.assertIsNotNone(transport.meter.failure) + + def test_body_limit_and_cumulative_budget_are_enforced(self): + for body, meter in ((b"a" * 262145, network.ProfileMeter()), (b"abc", network.ProfileMeter())): + if len(body) == 3: + meter.body_used = 5242880 - 2 + transport, connection, stream = self.use(Response(body), meter=meter) + with self.assertRaises(network.BoundaryViolation): + stream.read() + self.assertTrue(connection.closed) + + def test_head_never_reads_unsolicited_body(self): + response = Response(b"-----BEGIN PRIVATE KEY-----secret") + transport, connection, stream = self.use(response, request=Request(TARGET, method="HEAD")) + with stream: + self.assertEqual(stream.read(), b"") + self.assertEqual(response.reads, 0) + + def test_default_sensitive_guard_cannot_be_replaced(self): + transport, connection, stream = self.use(Response(b"-----BEGIN PRIVATE KEY-----PRIVATE"), + body_validator=lambda *_args: None) + with self.assertRaisesRegex(network.BoundaryViolation, "profile_sensitive_indicator"): + stream.read() + self.assertIsNone(stream._buffer) + + def test_header_indicator_and_access_restrictions_stop_without_body(self): + for response in (Response(b"body", headers={"X-Notice": "-----BEGIN PRIVATE KEY-----"}), + Response(b"body", status=403), Response(b"body", status=429), Response(b"body", status=503)): + transport, connection, stream = self.use(response) + with self.assertRaises(network.BoundaryViolation): + stream.read() + if response.status != 200: + self.assertEqual(response.reads, 0) + + def test_no_credentials_or_proxy_headers_are_forwarded(self): + request = Request(TARGET, headers={"Authorization": "sentinel", "Cookie": "sentinel", + "Host": "other.example", "Proxy-Authorization": "sentinel"}) + _transport, connection, stream = self.use(Response(), request=request) + stream.close() + sent = connection.requests[0][1]["headers"] + self.assertEqual(set(sent), {"Connection", "Accept-Encoding"}) + + def test_unsafe_dns_prevents_connection(self): + transport = opener() + for addresses in ([[socket.AF_INET, socket.SOCK_STREAM, 6, "", ["127.0.0.1", 80]]], + PUBLIC_ANSWERS + [[socket.AF_INET, socket.SOCK_STREAM, 6, "", ["10.0.0.1", 80]]]): + with mock.patch.object(network_dns, "resolve_answers", return_value=addresses), \ + mock.patch.object(network, "_ProfileHTTPConnection") as connection: + with self.assertRaises(network.BoundaryViolation): + transport.open(Request(TARGET)) + connection.assert_not_called() + # A refused attempt poisons its job; a second scenario uses a fresh job. + transport = opener() + + +class AdmissionTests(unittest.TestCase): + def grant(self, sequence, kind): + return {"ok": True, "admitted": True, "sequence": sequence, "valid_ms": 1000, + "timeout_ms": 10000, "http_remaining": 99, "kind": kind, + "profile_sha256": PROFILE_SHA256, "tls_remaining": 20 if kind == "http" else 19, + "body_bytes_remaining": 4980736, "active_ms_remaining": 600000 - sequence * 10000} + + def claim(self): + return SimpleNamespace(profile_id="external-web-v1", profile_sha256=PROFILE_SHA256, + services=(("http", 80), ("https", 443)), + expires_at=datetime.fromtimestamp(time.time() + 300, timezone.utc)) + + def test_external_strict_schema_and_profile_binding(self): + valid = self.grant(1, "http") + self.assertEqual(admission.parse_external_grant(valid, kind="http", profile_sha256=PROFILE_SHA256).sequence, 1) + for changed in ({**valid, "profile_sha256": "0" * 64}, {**valid, "kind": "tls"}, + {**valid, "http_remaining": True}, {**valid, "timeout_ms": 10001}, + {**valid, "body_bytes_remaining": -1}, {**valid, "extra": 1}): + with self.assertRaises(admission.AdmissionDenied): + admission.parse_external_grant(changed, kind="http", profile_sha256=PROFILE_SHA256) + + def test_one_gate_carries_both_kinds_and_rejects_replay(self): + responses = [self.grant(1, "http"), self.grant(2, "tls"), self.grant(2, "tls")] + calls = [] + api = SimpleNamespace(admit=lambda claim, kind: calls.append(kind) or responses.pop(0)) + gate = admission.AdmissionGate(api, self.claim(), kind="external") + gate.admit(kind="http") + gate.admit(kind="tls") + with self.assertRaisesRegex(admission.AdmissionDenied, "replayed"): + gate.admit(kind="tls") + self.assertEqual(calls, ["http", "tls", "tls"]) + + def test_pump_uses_combined_sequence_and_one_inflight_slot(self): + parent, child = socket.socketpair() + self.addCleanup(parent.close) + self.addCleanup(child.close) + kinds = [] + gate = SimpleNamespace(kind="external", admit=lambda *, kind: kinds.append(kind) or + admission.Grant(len(kinds), 1, 10, 1, time.monotonic())) + pump = admission.AdmissionPump(parent, gate, kind="external") + self.assertIsNone(pump._handle(b'{"op":"admit","kind":"http","n":1}')) + child.recv(1024) + self.assertIsNone(pump._handle(b'{"op":"done","n":1}')) + self.assertIsNone(pump._handle(b'{"op":"admit","kind":"tls","n":2}')) + child.recv(1024) + self.assertIn("protocol", pump._handle(b'{"op":"admit","kind":"http","n":3}')) + self.assertEqual(kinds, ["http", "tls"]) + + def test_malformed_kind_is_a_refusal_not_an_exception(self): + parent, child = socket.socketpair() + self.addCleanup(parent.close) + self.addCleanup(child.close) + pump = admission.AdmissionPump(parent, SimpleNamespace(kind="external"), kind="external") + self.assertIn("protocol", pump._handle(b'{"op":"admit","kind":[],"n":1}')) + + +class DNSAndTLSBoundaryTests(unittest.TestCase): + def test_dns_timeout_kills_and_reaps_the_helper(self): + processes = [] + real_popen = subprocess.Popen + def start(*args, **kwargs): + process = real_popen(*args, **kwargs) + processes.append(process) + return process + started = time.monotonic() + with mock.patch.object(network_dns, "_LOOKUP", "import time;time.sleep(30)"), \ + mock.patch.object(network_dns.subprocess, "Popen", side_effect=start): + with self.assertRaisesRegex(network_dns.DNSFailure, "timeout"): + network_dns.resolve_answers("localhost", 80, 0.1) + self.assertLess(time.monotonic() - started, 1) + self.assertIsNotNone(processes[0].returncode) + if os.name == "posix": + with self.assertRaises(ChildProcessError): + os.waitpid(processes[0].pid, os.WNOHANG) + + def test_dns_helper_gets_no_parent_secrets_or_inherited_channel(self): + read_fd, write_fd = os.pipe() + self.addCleanup(os.close, read_fd) + self.addCleanup(os.close, write_fd) + os.set_inheritable(read_fd, True) + script = ("import json,os;closed=False\n" + "try:os.fstat(" + str(read_fd) + ")\n" + "except OSError:closed=True\n" + "assert closed and 'CT_PRIVATE_SENTINEL' not in os.environ\n" + "print(json.dumps([[2,1,6,'',['127.0.0.1',80]]]))") + with mock.patch.dict(os.environ, {"CT_PRIVATE_SENTINEL": "never-forward"}), \ + mock.patch.object(network_dns, "_LOOKUP", script): + answers = network_dns.resolve_answers("localhost", 80, 1) + self.assertEqual(answers[0][4], ["127.0.0.1", 80]) + + def test_tls_only_never_sends_http_or_returns_raw_certificate(self): + guard = Guard() + transport = opener(guard=guard) + tls_socket = SimpleNamespace( + getpeercert=lambda: {"notAfter": "Nov 14 00:00:00 2030 GMT", "secret": "not-returned"}, + version=lambda: "TLSv1.3") + connection = SimpleNamespace(sock=tls_socket, connect=mock.Mock(), close=mock.Mock(), request=mock.Mock()) + with mock.patch.object(transport, "_connection", return_value=connection): + result = transport.inspect_tls("https://fixture.example/") + self.assertEqual(result, {"verified": True, "protocol": "TLSv1.3", "not_after": "2030-11-14T00:00:00Z"}) + self.assertEqual(guard.kinds, ["tls"]) + connection.request.assert_not_called() + connection.close.assert_called_once() + self.assertFalse(transport.meter.inflight) + + def test_insecure_tls_context_refused_before_any_admission(self): + import ssl + context = ssl._create_unverified_context() + guard = Guard() + with self.assertRaisesRegex(ValueError, "verification"): + opener(guard=guard, ssl_context=context) + self.assertEqual(guard.kinds, []) + + def test_count_duration_and_redirect_ceilings_cannot_reset(self): + class Clock: + now = 0.0 + def __call__(self): return self.now + for kind, maximum in (("http", 100), ("tls", 20)): + clock = Clock() + meter = network.ProfileMeter(monotonic=clock) + for _ in range(maximum): + meter.begin(kind) + meter.done() + clock.now += 1 + with self.assertRaises(network.BoundaryViolation): + meter.begin(kind) + clock = Clock() + meter = network.ProfileMeter(monotonic=clock) + clock.now = 600 + with self.assertRaises(network.BoundaryViolation): + meter.begin("http") + meter = network.ProfileMeter() + for _ in range(3): meter.redirect() + with self.assertRaisesRegex(network.BoundaryViolation, "redirect"): + meter.redirect() + + +class TrickleTests(unittest.TestCase): + def fixture(self, *, headers=False): + class Handler(socketserver.BaseRequestHandler): + def handle(self): + self.request.recv(4096) + try: + if headers: + self.request.sendall(b"HTTP/1.1 200 OK\r\nX-Slow: ") + else: + self.request.sendall(b"HTTP/1.1 200 OK\r\nContent-Length: 100\r\nConnection: close\r\n\r\n") + for _ in range(100): + self.request.sendall(b"x") + time.sleep(0.02) + except OSError: + pass + server = socketserver.ThreadingTCPServer(("127.0.0.1", 0), Handler) + server.daemon_threads = True + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + self.addCleanup(server.server_close) + self.addCleanup(server.shutdown) + def connect(connection, deadline): + # Owned loopback fixture only. Address-safety is separately tested; + # production resolver/classifier code is never changed by this test. + connection._before_connect() + raw = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + connection.sock = raw + raw.settimeout(deadline.remaining()) + deadline.attach(raw) + raw.connect(server.server_address) + return raw + return connect + + def test_body_deadline_survives_connection_sock_handoff(self): + transport = opener() + started = time.monotonic() + with mock.patch.object(network_dns, "resolve_answers", return_value=PUBLIC_ANSWERS), \ + mock.patch.object(network, "_profile_socket", self.fixture()): + stream = transport.open(Request(TARGET), timeout=0.15) + self.assertIsNone(stream._connection.sock) + with self.assertRaises(network.BoundaryViolation): + stream.read() + self.assertLess(time.monotonic() - started, 0.8) + self.assertFalse(transport.meter.inflight) + + def test_slow_header_has_total_not_inactivity_deadline(self): + transport = opener() + started = time.monotonic() + with mock.patch.object(network_dns, "resolve_answers", return_value=PUBLIC_ANSWERS), \ + mock.patch.object(network, "_profile_socket", self.fixture(headers=True)): + with self.assertRaises(network.BoundaryViolation): + stream = transport.open(Request(TARGET), timeout=0.15) + stream.read() + self.assertLess(time.monotonic() - started, 0.8) + self.assertFalse(transport.meter.inflight) + + def test_real_head_does_not_read_or_decompress_unexpected_body(self): + completed = threading.Event() + sent = [] + class Handler(socketserver.BaseRequestHandler): + def handle(self): + self.request.recv(4096) + try: + self.request.sendall(b"HTTP/1.1 200 OK\r\nContent-Encoding: gzip\r\nContent-Length: 1048576\r\nConnection: close\r\n\r\n") + time.sleep(0.05) + for _ in range(64): + self.request.sendall(b"x" * 16384) + sent.append(16384) + time.sleep(0.01) + except OSError: + pass + finally: + completed.set() + server = socketserver.ThreadingTCPServer(("127.0.0.1", 0), Handler) + server.daemon_threads = True + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + self.addCleanup(server.server_close) + self.addCleanup(server.shutdown) + def connect(connection, deadline): + connection._before_connect() + raw = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + connection.sock = raw + deadline.attach(raw) + raw.connect(server.server_address) + return raw + transport = opener() + with mock.patch.object(network_dns, "resolve_answers", return_value=PUBLIC_ANSWERS), \ + mock.patch.object(network, "_profile_socket", connect), \ + mock.patch.object(network._BoundedHTTPResponse, "read", side_effect=AssertionError("HEAD body read")): + with transport.open(Request(TARGET, method="HEAD"), timeout=0.5) as stream: + self.assertEqual(stream.read(), b"") + self.assertTrue(completed.wait(1)) + self.assertLess(sum(sent), 1048576) + self.assertEqual(transport.meter.body_used, 0) + + +if __name__ == "__main__": + unittest.main() From 8fa82a5cf18543cdf3bfeeda6baa1451416c5520 Mon Sep 17 00:00:00 2001 From: Timothy Wayne Gregg Date: Sat, 14 Nov 2026 02:04:34 -0500 Subject: [PATCH 10/19] fix(worker): bound transfer framing and preserve safe profile stops --- integrations/worker/network.py | 82 ++++++++++++++++++++++++++++------ test_external_boundary.py | 58 +++++++++++++++++++++++- 2 files changed, 125 insertions(+), 15 deletions(-) diff --git a/integrations/worker/network.py b/integrations/worker/network.py index 33d3b58..751f2b9 100644 --- a/integrations/worker/network.py +++ b/integrations/worker/network.py @@ -30,6 +30,8 @@ "profile_method_forbidden", "profile_connection_failed", "profile_tls_out_of_scope", "profile_tls_metadata_invalid", "profile_tls_failed", "profile_dns_invalid", "profile_dns_failed", "profile_dns_timeout", "profile_redirect_budget_exhausted", + "profile_response_truncated", "profile_framing_invalid", + "possible_sensitive_indicator", "possible_compromise_indicator", }) @@ -507,32 +509,55 @@ def open(self, request, timeout=None): class _HeaderReader: - """Count status/header bytes while http.client is parsing, before allocation.""" + """Bound bytes below HTTP transfer decoding, including chunk metadata.""" def __init__(self, stream, maximum): self.stream, self.maximum, self.used = stream, maximum, 0 + self.body = False + self.body_used = 0 + self.on_body = None + + def _body_count(self, count): + self.body_used += count + if self.on_body is not None: + self.on_body(count) + if self.body_used > 262144: + raise BoundaryViolation("profile_body_too_large") def readline(self, size=-1): remaining = self.maximum - self.used + 1 + if self.body: + remaining = min(remaining, 262144 - self.body_used + 1) value = self.stream.readline(remaining if size < 0 else min(size, remaining)) self.used += len(value) + if self.body: + self._body_count(len(value)) + if not value: + raise BoundaryViolation("profile_response_truncated") if self.used > self.maximum: raise BoundaryViolation("profile_headers_too_large") return value + def read(self, size=-1): + if not self.body: + raise BoundaryViolation("profile_framing_invalid") + remaining = 262144 - self.body_used + 1 + value = self.stream.read(remaining if size < 0 else min(size, remaining)) + self._body_count(len(value)) + return value + def __getattr__(self, name): return getattr(self.stream, name) class _BoundedHTTPResponse(http.client.HTTPResponse): def begin(self): - original = self.fp - self.fp = _HeaderReader(original, 32768) - try: - super().begin() - finally: - # Keep the same buffered stream and its already received bytes. - self.fp = original + self._wire_reader = _HeaderReader(self.fp, 32768) + self.fp = self._wire_reader + super().begin() + # Keep the bounded wrapper: chunk extensions, CRLF delimiters and + # trailers are parsed later by HTTPResponse.read(), not by begin(). + self._wire_reader.body = True class _RequestDeadline: @@ -640,11 +665,11 @@ def _default_content_validator(body, headers): header_text = "\n".join(str(name) + ":" + str(value) for name, value in headers.items()).encode("utf-8", "replace") lower = body.lower() + b"\n" + header_text.lower() if (b"-----begin " in lower and b"private key-----" in lower) or b"aws_secret_access_key" in lower: - raise BoundaryViolation("profile_sensitive_indicator") + raise BoundaryViolation("possible_sensitive_indicator") if b"cf-chl-" in lower or b"__cf_chl" in lower: raise BoundaryViolation("profile_access_challenge") if re.search(rb"\bhacked\s+by\b", lower): - raise BoundaryViolation("profile_compromise_indicator") + raise BoundaryViolation("possible_compromise_indicator") class _ExternalResponse: @@ -660,6 +685,19 @@ def __init__(self, response, connection, deadline, meter, done, *, method, valid for name, value in response.headers.items(): self.headers[name] = value self.original_headers = tuple(response.headers.items()) + encodings = self.headers.get_all("Content-Encoding", []) + self._encoding = (encodings[0] if len(encodings) == 1 else "identity").strip().lower() + self._encoding_count = len(encodings) + if self._encoding == "gzip" and self._encoding_count == 1 and method != "HEAD": + # Decoding is guaranteed before read() can return any bytes. Make + # that interface explicit for callers which inspect headers first. + del self.headers["Content-Encoding"] + del self.headers["Content-Length"] + self.headers["Content-Encoding"] = "identity" + reader = getattr(response, "_wire_reader", None) + self._wire_accounted = isinstance(reader, _HeaderReader) + if self._wire_accounted: + reader.on_body = lambda count: meter.consume(count, wire=True) self.decoded = False def _load(self): @@ -671,18 +709,26 @@ def _load(self): raise BoundaryViolation("profile_access_restricted") if self.code >= 500: raise BoundaryViolation("profile_service_unavailable") - encodings = self.headers.get_all("Content-Encoding", []) - if len(encodings) > 1: + if self._encoding_count > 1: raise BoundaryViolation("profile_encoding_unsupported") - encoding = (encodings[0] if encodings else "identity").strip().lower() + encoding = self._encoding if encoding not in {"", "identity", "gzip"}: raise BoundaryViolation("profile_encoding_unsupported") + lengths = self.headers.get_all("Content-Length", []) if encoding != "gzip" else [ + value for name, value in self.original_headers if name.lower() == "content-length"] + transfers = self.headers.get_all("Transfer-Encoding", []) + if (len(lengths) > 1 or len(transfers) > 1 or (lengths and transfers) + or (transfers and transfers[0].strip().lower() != "chunked") + or (lengths and not re.fullmatch(r"[0-9]{1,10}", lengths[0].strip()))): + raise BoundaryViolation("profile_framing_invalid") if self._method == "HEAD": # No body read, including when a server incorrectly supplies one. self._validator(b"", self.headers) self._buffer = b"" self.decoded = True return + if lengths and int(lengths[0]) > 262144: + raise BoundaryViolation("profile_body_too_large") decoder = zlib.decompressobj(16 + zlib.MAX_WBITS) if encoding == "gzip" else None result = bytearray() wire = 0 @@ -691,9 +737,12 @@ def _load(self): chunk = self._response.read(min(8192, 262144 - wire + 1)) self._deadline.remaining() if not chunk: + if getattr(self._response, "length", None) not in {None, 0}: + raise BoundaryViolation("profile_response_truncated") break wire += len(chunk) - self._meter.consume(len(chunk), wire=True) + if not self._wire_accounted: + self._meter.consume(len(chunk), wire=True) if wire > 262144: raise BoundaryViolation("profile_body_too_large") decoded = decoder.decompress(chunk, 262144 - len(result) + 1) if decoder else chunk @@ -717,6 +766,11 @@ def _load(self): except BaseException as exc: self._buffer = None reason = exc.reason if isinstance(exc, BoundaryViolation) else "profile_response_failed" + category = getattr(exc, "reason", None) + if type(category) is str and category in {"possible_sensitive_indicator", "possible_compromise_indicator"}: + # The executor's validator has the same two fixed operational + # categories; never copy arbitrary exception text or reasons. + reason = category self._meter.failure = self._meter.failure or reason try: self.close() diff --git a/test_external_boundary.py b/test_external_boundary.py index e7b1882..bbd7dc6 100644 --- a/test_external_boundary.py +++ b/test_external_boundary.py @@ -114,6 +114,12 @@ def test_shared_meter_cannot_change_scope(self): class BodyAndHeaderTests(unittest.TestCase): + def raw_response(self, payload): + raw = io.BytesIO(payload) + response = network._BoundedHTTPResponse(SimpleNamespace(makefile=lambda _mode: raw)) + response.begin() + return response + def use(self, response, *, request=None, **kwargs): transport = opener(**kwargs) connection = Connection(response) @@ -141,6 +147,43 @@ def test_gzip_is_decoded_bounded_and_headers_normalized(self): self.assertTrue(response.closed and connection.closed) self.assertFalse(transport.meter.inflight) + def test_gzip_headers_are_normalized_before_the_first_read(self): + response = Response(gzip.compress(b"safe"), headers={"Content-Encoding": "gzip"}) + _transport, _connection, stream = self.use(response) + self.assertEqual(stream.headers["Content-Encoding"], "identity") + self.assertEqual(response.reads, 0) + with stream: + self.assertEqual(stream.read(), b"safe") + + def test_chunk_metadata_and_trailers_are_bounded_and_counted(self): + payload = b"HTTP/1.1 200 OK\r\nTransfer-Encoding: chunked\r\n\r\n4\r\nsafe\r\n0\r\nX-End: yes\r\n\r\n" + transport, _connection, stream = self.use(self.raw_response(payload)) + with stream: + self.assertEqual(stream.read(), b"safe") + self.assertEqual(transport.meter.wire_used, len(payload.split(b"\r\n\r\n", 1)[1])) + for suffix in (b"1;" + b"x" * 40000 + b"\r\na\r\n0\r\n\r\n", + b"1\r\na\r\n0\r\nX-End: " + b"x" * 40000 + b"\r\n\r\n"): + transport, connection, stream = self.use(self.raw_response( + b"HTTP/1.1 200 OK\r\nTransfer-Encoding: chunked\r\n\r\n" + suffix)) + with self.assertRaises(network.BoundaryViolation): + stream.read(1) + self.assertTrue(connection.closed) + self.assertIsNone(stream._buffer) + self.assertIsNotNone(transport.meter.failure) + + def test_ambiguous_or_truncated_framing_never_returns_a_prefix(self): + for payload in ( + b"Content-Length: 10\r\n\r\nsafe", + b"Content-Length: 4\r\nContent-Length: 4\r\n\r\nsafe", + b"Transfer-Encoding: chunked\r\nContent-Length: 4\r\n\r\n4\r\nsafe\r\n0\r\n\r\n", + b"Transfer-Encoding: chunked\r\n\r\n4\r\nsafe\r\n0\r\n"): + transport, connection, stream = self.use(self.raw_response(b"HTTP/1.1 200 OK\r\n" + payload)) + with self.assertRaises(network.BoundaryViolation): + stream.read(1) + self.assertTrue(connection.closed) + self.assertIsNone(stream._buffer) + self.assertIsNotNone(transport.meter.failure) + def test_gzip_bomb_and_extra_member_never_return_prefix(self): for body in (gzip.compress(b"x" * 262145), gzip.compress(b"safe") + gzip.compress(b"hidden")): transport, connection, stream = self.use(Response(body, headers={"Content-Encoding": "gzip"})) @@ -169,10 +212,23 @@ def test_head_never_reads_unsolicited_body(self): def test_default_sensitive_guard_cannot_be_replaced(self): transport, connection, stream = self.use(Response(b"-----BEGIN PRIVATE KEY-----PRIVATE"), body_validator=lambda *_args: None) - with self.assertRaisesRegex(network.BoundaryViolation, "profile_sensitive_indicator"): + with self.assertRaisesRegex(network.BoundaryViolation, "possible_sensitive_indicator"): stream.read() self.assertIsNone(stream._buffer) + def test_validator_stop_categories_are_fixed_and_never_echo_unknown_reasons(self): + class Stop(Exception): + def __init__(self, reason): self.reason = reason + for reason in ("possible_sensitive_indicator", "possible_compromise_indicator", "private-target-value", []): + def validate(_body, _headers): raise Stop(reason) + transport, connection, stream = self.use(Response(b"safe"), body_validator=validate) + with self.assertRaises(network.BoundaryViolation) as refused: + stream.read(1) + self.assertEqual(refused.exception.reason, + reason if type(reason) is str and reason.startswith("possible_") else "profile_response_failed") + self.assertTrue(connection.closed) + self.assertIsNone(stream._buffer) + def test_header_indicator_and_access_restrictions_stop_without_body(self): for response in (Response(b"body", headers={"X-Notice": "-----BEGIN PRIVATE KEY-----"}), Response(b"body", status=403), Response(b"body", status=429), Response(b"body", status=503)): From 4cfea8488fc8a190b7ddbdc166167be2394949fe Mon Sep 17 00:00:00 2001 From: Timothy Wayne Gregg Date: Sat, 14 Nov 2026 02:06:31 -0500 Subject: [PATCH 11/19] docs(worker): explain unpublished external profile and acceptance limits --- docs/EXTERNAL_PROFILE.md | 46 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) create mode 100644 docs/EXTERNAL_PROFILE.md diff --git a/docs/EXTERNAL_PROFILE.md b/docs/EXTERNAL_PROFILE.md new file mode 100644 index 0000000..1b9b3c3 --- /dev/null +++ b/docs/EXTERNAL_PROFILE.md @@ -0,0 +1,46 @@ +# Unpublished automatic external profile + +`external-web-v1` is an integration-only candidate. The public single-file +checker keeps its existing interface. These sources do not provision a worker, +adopt an agreement, authorize a customer target, or enable the WordPress queue. + +The canonical manifest is `integrations/profiles/external-web-v1.json`, pinned +by SHA-256 `1f2a19c959570157ee9df788141ae1e21b8f67aba8c6ce99b768efbd42eb7a37`. +Changing those bytes requires a separately reviewed profile and new assent; +neither configuration nor a retry can widen a saved authorization. A job binds +the manifest identity and explicit selected HTTP/80 and HTTPS/443 services. + +The manifest permits robots policy, public homepage header/version triage, +security-header presence and a verified TLS certificate observation. It permits +only the selected hostname, root and robots paths. Advertised versions remain +evidence rather than verified runtime or exploitability. Internal configuration, +runtime/backport confirmation and compromise investigation remain unassessed. +No title, HTML, arbitrary header value, asset URL or raw certificate is retained +in the typed result. Possible sensitive-data or compromise indicators stop the +whole job with a fixed category; they do not initiate investigation. + +The worker uses one meter and one admitted transport across every check, +redirect and selected service. DNS answers are bounded, private/reserved answers +are refused, and accepted addresses are pinned while preserving Host and TLS +SNI. A disposable resolver receives no authentication key or admission channel. +HTTP framing, header bytes, wire and decompressed body bytes and total request +time are bounded. HEAD observations do not read an unsolicited response body. + +Each HTTP or TLS connection needs a fresh supervisor grant bound to the saved +profile. The manifest ceilings are 100 HTTP transactions, 20 additional TLS +checks, 600 active seconds, 5 MiB aggregate body bytes, 256 KiB per response, +32 KiB headers, ten seconds per request, one connection and at most one check +per second. The implementation can stop earlier: it reserves the full response +and time allowance before connecting, does not refund reservations or retry +automatically, and retains the existing 175-second child/180-second supervisor +limits. A restarted or uncertain job never acquires a fresh budget or rescans. + +Source tests use injected transports, mocks and ephemeral local fixtures. They +do not establish production egress enforcement or customer authorization. +Before deployment, the matching WordPress binding must pass real disposable +database and controlled-network acceptance tests, including cancellation, +expiry, exact-target authority, mailbox/report recipients, budgets, retained +evidence and rollback. The current-base candidate still needs exact-head CI +and two fresh independent reviews. Site release and activation are separate +coordinated operations; keep the candidate disabled until those prerequisites +are verified. From 920c6e37320d3737e432f44f136e3af6520462de Mon Sep 17 00:00:00 2001 From: Timothy Wayne Gregg Date: Sat, 14 Nov 2026 02:09:28 -0500 Subject: [PATCH 12/19] test(worker): exercise combined external admission and scanner pipeline --- test_external_pipeline.py | 173 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 173 insertions(+) create mode 100644 test_external_pipeline.py diff --git a/test_external_pipeline.py b/test_external_pipeline.py new file mode 100644 index 0000000..f8cd0df --- /dev/null +++ b/test_external_pipeline.py @@ -0,0 +1,173 @@ +"""Real external scanner/admission chain with in-memory target connections only. + +This is a source integration regression, not a privileged worker, WordPress, +database, TLS-handshake or external-network acceptance test. +""" + +from datetime import datetime, timezone +import hashlib +import io +import json +import select +import socket +import threading +import time +from types import SimpleNamespace +import unittest +from unittest.mock import Mock, patch + +from integrations.worker import admission, network, network_dns, scanner +from integrations.worker.profiles import PROFILE_ID, PROFILE_SHA256 +from integrations.worker.protocol import JobClaim, profile_fields +from integrations.worker.runner import SubprocessRunner + + +TARGET = "https://example.test/" +ROBOTS = b"User-agent: *\nAllow: /\n" +HOME = b'' + + +class ExternalPipelineTests(unittest.TestCase): + def run_scanner(self, *, home=HOME): + expiry = datetime.fromtimestamp(time.time() + 300, timezone.utc) + claim = JobClaim(1, TARGET, hashlib.sha256(TARGET.encode()).hexdigest(), + "a" * 64, expiry, "b" * 32, expiry, + PROFILE_ID, PROFILE_SHA256, (("http", 80), ("https", 443))) + parent, child = socket.socketpair() + self.addCleanup(parent.close) + self.addCleanup(child.close) + stopped = threading.Event() + kinds, connections, openers, service_errors = [], [], [], [] + + class API: + def admit(self, _claim, kind): + if kind == "probe": + return {"ok": True, "live": True} + kinds.append(kind) + http, tls = kinds.count("http"), kinds.count("tls") + return {"ok": True, "admitted": True, "sequence": len(kinds), + "valid_ms": 1000, "timeout_ms": 10000, + "http_remaining": 100 - http, "kind": kind, + "profile_sha256": PROFILE_SHA256, "tls_remaining": 20 - tls, + "body_bytes_remaining": 5242880 - http * 262144, + "active_ms_remaining": 600000 - len(kinds) * 10000} + + gate = admission.AdmissionGate(API(), claim, kind="external") + pump = admission.AdmissionPump(parent, gate, kind="external") + + def serve(): + try: + while not stopped.is_set(): + if select.select([parent], [], [], 0.02)[0] and pump.service(): + return + if pump.tick(): + return + except BaseException as exc: + service_errors.append(type(exc).__name__) + + service = threading.Thread(target=serve, daemon=True) + service.start() + + class Connection: + def __init__(self, scheme): + self.scheme, self.method, self.path = scheme, None, None + self.closed = False + self.response = None + self.sock = SimpleNamespace( + getpeercert=lambda: {"notAfter": "Nov 14 00:00:00 2030 GMT"}, + version=lambda: "TLSv1.3") + connections.append(self) + + def request(self, method, path, headers): + self.method, self.path = method, path + + def getresponse(self): + body = ROBOTS if self.path == "/robots.txt" else home + payload = (b"HTTP/1.1 200 OK\r\nContent-Type: text/html\r\n" + b"X-Powered-By: PHP/8.3.36\r\nContent-Length: " + + str(len(body)).encode() + b"\r\n\r\n" + body) + stream = io.BytesIO(payload) + self.response = network._BoundedHTTPResponse( + SimpleNamespace(makefile=lambda _mode: stream), method=self.method) + self.response.begin() + return self.response + + def connect(self): + pass + + def close(self): + self.closed = True + + original_init = network.ExternalPinnedOpener.__init__ + + def capture_opener(instance, *args, **kwargs): + original_init(instance, *args, **kwargs) + openers.append(instance) + + message = {"target": TARGET, **profile_fields(claim)} + try: + # Preserve the actual scanner, ChildAdmission socket protocol, + # supervisor gate/pump, pinned opener, HTTP parser, meter, executor + # and result validator. Replace only target connections and the + # scanner's process-entry IO/identity/alarm setup. + with patch.object(scanner.os, "geteuid", return_value=65534), \ + patch.object(scanner.signal, "alarm"), \ + patch.object(scanner.signal, "signal"), \ + patch.object(scanner.sys, "stdin", Mock(buffer=io.BytesIO(json.dumps(message).encode()))), \ + patch.object(scanner, "_admission_channel", return_value=child), \ + patch.object(scanner, "_write", side_effect=lambda value, code: (value, code)), \ + patch.object(network.ExternalPinnedOpener, "__init__", capture_opener), \ + patch.object(network.ExternalPinnedOpener, "_connection", + side_effect=lambda scheme, *_args: Connection(scheme)), \ + patch.object(network_dns, "resolve_answers", side_effect=AssertionError("unexpected target DNS")), \ + patch.object(network, "_public_addresses", side_effect=AssertionError("unexpected target DNS")), \ + patch.object(network, "_profile_socket", side_effect=AssertionError("unexpected target connection")): + value, code = scanner.main() + until = time.monotonic() + 1 + while pump.inflight and time.monotonic() < until: + stopped.wait(0.005) + self.assertFalse(pump.inflight) + self.assertIsNone(pump.fatal) + self.assertEqual(service_errors, []) + self.assertEqual(len(openers), 1) + self.assertTrue(all(connection.closed for connection in connections)) + self.assertTrue(all(connection.response is None or connection.response.isclosed() + for connection in connections)) + self.assertFalse(openers[0].meter.inflight) + self.assertEqual(pump.grants, len(kinds)) + self.assertEqual(pump._last_n, len(kinds)) + return value, code, claim, kinds, connections, openers[0].meter + finally: + stopped.set() + service.join(1) + self.assertFalse(service.is_alive(), "admission fixture thread must stop") + pump.wait_probe(1) + + def test_real_chain_shares_budget_and_sequences_across_both_services_and_tls(self): + value, code, claim, kinds, connections, meter = self.run_scanner() + self.assertEqual(code, 0) + self.assertIs(value["ok"], True) + SubprocessRunner.validate_result(value["result"], TARGET, + network.RequestOrigin.from_target(TARGET)[0], claim=claim) + self.assertEqual(kinds, ["http"] * 6 + ["tls"]) + self.assertEqual((meter.http_used, meter.tls_used), (6, 1)) + self.assertEqual([(item.scheme, item.method, item.path) for item in connections], [ + ("http", "GET", "/robots.txt"), ("http", "GET", "/"), ("http", "HEAD", "/"), + ("https", "GET", "/robots.txt"), ("https", "GET", "/"), ("https", "HEAD", "/"), + ("https", None, None)]) + self.assertEqual(meter.body_used, 2 * (len(ROBOTS) + len(HOME))) + self.assertEqual(meter.wire_used, meter.body_used) + self.assertIsNone(meter.failure) + + def test_sensitive_body_stops_real_chain_without_later_service_or_tls_admission(self): + value, code, _claim, kinds, connections, meter = self.run_scanner( + home=b"-----BEGIN PRIVATE KEY-----synthetic-do-not-return") + self.assertEqual((value, code), ({"ok": False, "error": "possible_sensitive_indicator"}, 2)) + self.assertEqual(kinds, ["http", "http"]) + self.assertEqual(len(connections), 2) + self.assertEqual((meter.http_used, meter.tls_used), (2, 0)) + self.assertEqual(meter.failure, "possible_sensitive_indicator") + + +if __name__ == "__main__": + unittest.main() From c06a684a51cdda4f5b3d14f25e221a4925b413a2 Mon Sep 17 00:00:00 2001 From: Timothy Wayne Gregg Date: Sat, 14 Nov 2026 02:10:07 -0500 Subject: [PATCH 13/19] feat(wordpress): bind unpublished external profile evidence and reservations Refs #27. Preserve legacy defaults; add pure cross-source frozen-evidence and typed-result checks. Real database and automatic lifecycle verification remain pending. --- docs/EXTERNAL_WORDPRESS.md | 65 +++ integrations/wordpress/ct-scan.php | 507 +++++++++++++++++- .../automatic-evidence-cross.php | 59 ++ .../standalone-tests/external-profile.php | 84 +++ .../standalone-tests/manual-lifecycle.php | 4 +- .../wordpress/tests/external-reservations.php | 40 ++ 6 files changed, 738 insertions(+), 21 deletions(-) create mode 100644 docs/EXTERNAL_WORDPRESS.md create mode 100644 integrations/wordpress/standalone-tests/automatic-evidence-cross.php create mode 100644 integrations/wordpress/standalone-tests/external-profile.php create mode 100644 integrations/wordpress/tests/external-reservations.php diff --git a/docs/EXTERNAL_WORDPRESS.md b/docs/EXTERNAL_WORDPRESS.md new file mode 100644 index 0000000..11ea030 --- /dev/null +++ b/docs/EXTERNAL_WORDPRESS.md @@ -0,0 +1,65 @@ +# Unpublished external WordPress binding (#27) + +This future source extends the maintained queue plugin with schema 9: an exact +immutable profile digest, explicit selected services, and durable cumulative +HTTP/TLS/body/active-time reservations. The existing `r2-header-v1` profile and +wire messages retain their defaults. External claims and result callbacks add +only `profile_id`, `profile_sha256`, and `services`; start/admit messages retain +their existing shape. Missing or changed bindings refuse work. Configuration +still defaults to disabled and no production release or customer consent is +established by this source. + +Before creating or continuing an external job, the core independently reads the +frozen automatic envelope through the separate G2 adapter and checks its raw +metadata, canonical release/profile snapshots, exact Target Summary, signer +and acceptance digests, encoded post representation and fixed UTC expiry. +The automatic contract's identity header ends before its separately checked +canonical Summary. Legacy contract validation retains its existing boundaries. + +External results retain only the fixed typed observation vocabulary, bounded +version tokens and the maintained Checker's public interpretation vocabulary. +Unknown check fields, raw page/title/header text, arbitrary interpretation +strings, unselected services and changed manifest identities are refused. +Conservative reservations charge 256 KiB and 10 seconds per HTTP admission, +and 10 seconds per TLS-only admission, without refunds. New connections wait +until the prior reservation's bounded timeout has elapsed. These conservative +rules can stop before the disclosed maximum counts. + +Pure source checks (no WordPress bootstrap, database, DNS, HTTP or mail): + +```sh +php integrations/wordpress/standalone-tests/external-profile.php +php integrations/wordpress/standalone-tests/report-access.php +php integrations/wordpress/standalone-tests/manual-lifecycle.php +``` + +For a combined G1/G2 frozen-evidence test, pass an isolated G2 source root and +an explicitly **SYNTHETIC** canonical generator release. The fixture calls the +actual route decoder, persistence producer/readback/copy helpers, and core +`ct_scan_validate_saved_request()` using synthetic WordPress API stores. It +covers Unicode, ampersands, apostrophes, exact summary/signature/expiry binding, +immutable-field tampering, duplicate metadata and altered post representation. +It does not invoke acceptance, create a queue job, adopt a release, or mail. +The optional trusted WordPress `kses.php` path adds the real pure entity +normalizer without bootstrapping WordPress. + +```sh +CT_AUTO_CROSS_G2_ROOT=/isolated/G2-source \ +CT_AUTO_CROSS_RELEASE=/private/SYNTHETIC/release.json \ +CT_AUTO_CROSS_KSES=/trusted/wp-includes/kses.php \ +php integrations/wordpress/standalone-tests/automatic-evidence-cross.php +``` + +`integrations/wordpress/tests/external-reservations.php` is a separate disposable +WordPress/database fixture source. It tests real schema columns, committed +reservations, stale-row CAS refusal, inflight exclusion and restored malformed +state using an owned fixture row. It does not represent a real external +acceptance or target scan. It has not been executed locally in this recovery. +Run only through the reviewed fixture support and an allocated protected +mutation slot; never run it against production. + +Actual database concurrency, the full accepted automatic route/outbox/queue +lifecycle, mailbox and authority gates, worker/result/report delivery, rollback, +exact-head CI and two fresh final reviews remain required. The upstream +lifecycle fixes also remain a dependency. This future branch does not satisfy +those gates, enable scans, or close the related issues. diff --git a/integrations/wordpress/ct-scan.php b/integrations/wordpress/ct-scan.php index 2603af7..1ae4d01 100644 --- a/integrations/wordpress/ct-scan.php +++ b/integrations/wordpress/ct-scan.php @@ -17,8 +17,8 @@ } if ( ! defined( 'CT_SCAN_SCHEMA_VERSION' ) ) { - // 7 combined retention/hold/tombstone columns with `control_admitted`; 8 adds one-time report capabilities and report closure metadata. - define( 'CT_SCAN_SCHEMA_VERSION', '8' ); + // 9 adds immutable external-profile service bindings and conservative cumulative reservations. + define( 'CT_SCAN_SCHEMA_VERSION', '9' ); } /** @@ -30,7 +30,178 @@ * 20 requests. Unknown profiles are refused. */ function ct_scan_current_profile() { - return 'r2-header-v1'; + $profile = ct_scan_config()['execution_profile']; + return ct_scan_profile_limits( $profile ) ? $profile : ''; +} + +/** The reviewed external profile is immutable. A changed or missing artifact is not a new permission. */ +function ct_scan_external_profile_manifest() { + static $loaded = null; + if ( null !== $loaded ) { + return $loaded; + } + $loaded = false; + $paths = array( __DIR__ . '/ct-scan-profiles/external-web-v1.json', dirname( __DIR__ ) . '/profiles/external-web-v1.json' ); + if ( defined( 'CT_SCAN_PROFILE_MANIFEST_DIR' ) && is_string( CT_SCAN_PROFILE_MANIFEST_DIR ) && '/' === substr( CT_SCAN_PROFILE_MANIFEST_DIR, 0, 1 ) && false === strpos( CT_SCAN_PROFILE_MANIFEST_DIR, "\0" ) ) { + array_unshift( $paths, rtrim( CT_SCAN_PROFILE_MANIFEST_DIR, '/' ) . '/external-web-v1.json' ); + } + foreach ( $paths as $path ) { + if ( is_link( $path ) || ! is_file( $path ) || ! is_readable( $path ) || filesize( $path ) > 65536 ) { + continue; + } + $bytes = file_get_contents( $path ); + if ( ! is_string( $bytes ) || strlen( $bytes ) > 65536 || ! hash_equals( ct_scan_external_profile_digest(), hash( 'sha256', $bytes ) ) ) { + continue; + } + $value = json_decode( $bytes, true ); + if ( JSON_ERROR_NONE !== json_last_error() || ! is_array( $value ) || 1 !== ( $value['schema_version'] ?? null ) + || 'external-web-v1' !== ( $value['profile_id'] ?? null ) + || wp_json_encode( ct_scan_canonicalize( $value ), JSON_UNESCAPED_SLASHES | JSON_UNESCAPED_UNICODE ) . "\n" !== $bytes ) { + continue; + } + $loaded = $value; + break; + } + return $loaded; +} + +/** Set to the exact reviewed manifest bytes when the implementation is integrated. */ +function ct_scan_external_profile_digest() { + return '1f2a19c959570157ee9df788141ae1e21b8f67aba8c6ce99b768efbd42eb7a37'; +} + +/** Canonical selected services are explicit; target normalization cannot add another service. */ +function ct_scan_external_services( $services, $target = null ) { + if ( ! is_array( $services ) || ! ct_scan_is_list( $services ) || count( $services ) < 1 || count( $services ) > 2 ) { + return false; + } + $canonical = array(); + foreach ( array( array( 'port' => 80, 'scheme' => 'http' ), array( 'port' => 443, 'scheme' => 'https' ) ) as $allowed ) { + foreach ( $services as $service ) { + if ( ! is_array( $service ) || ! ct_scan_expected_keys( $service, array( 'scheme', 'port' ) ) + || ! is_string( $service['scheme'] ) || ! is_int( $service['port'] ) ) { + return false; + } + if ( $service['scheme'] === $allowed['scheme'] && $service['port'] === $allowed['port'] ) { + if ( in_array( $allowed, $canonical, true ) ) { return false; } + $canonical[] = $allowed; + } + } + } + if ( count( $canonical ) !== count( $services ) ) { return false; } + foreach ( $services as $i => $service ) { if ( $service['scheme'] !== $canonical[$i]['scheme'] || $service['port'] !== $canonical[$i]['port'] ) { return false; } } + if ( null !== $target ) { + $normalized = ct_scan_normalize_target( $target ); + if ( is_wp_error( $normalized ) || $normalized !== $target ) { return false; } + $parts = is_string( $target ) ? wp_parse_url( $target ) : false; + if ( ! is_array( $parts ) || empty( $parts['scheme'] ) || empty( $parts['host'] ) ) { return false; } + $initial = array( 'port' => $parts['port'] ?? ( 'https' === $parts['scheme'] ? 443 : 80 ), 'scheme' => $parts['scheme'] ); + if ( ! in_array( $initial, $canonical, true ) ) { return false; } + } + return $canonical; +} + +/** Read the four complete profile bindings; r2 never acquires external permissions from these fields. */ +function ct_scan_external_saved_binding( $request_id, $target ) { + $values = array(); + foreach ( array( '_ct_terms_profile_id', '_ct_terms_profile_sha256', '_ct_terms_scope_services', '_ct_terms_profile_snapshot', '_ct_terms_authorization_expires_at' ) as $key ) { + $rows = get_post_meta( $request_id, $key, false ); + if ( ! is_array( $rows ) || 1 !== count( $rows ) || ! is_string( $rows[0] ) ) { return false; } + $values[ $key ] = $rows[0]; + } + $manifest = ct_scan_external_profile_manifest(); + if ( ! $manifest || 'external-web-v1' !== $values['_ct_terms_profile_id'] + || ! hash_equals( ct_scan_external_profile_digest(), $values['_ct_terms_profile_sha256'] ) + || ! hash_equals( ct_scan_external_profile_digest(), hash( 'sha256', $values['_ct_terms_profile_snapshot'] ) ) ) { return false; } + $services = json_decode( $values['_ct_terms_scope_services'], true ); + $services = JSON_ERROR_NONE === json_last_error() ? ct_scan_external_services( $services, $target ) : false; + if ( false === $services || wp_json_encode( ct_scan_canonicalize( $services ), JSON_UNESCAPED_SLASHES ) !== $values['_ct_terms_scope_services'] ) { return false; } + if ( ! ct_scan_utc_parse( $values['_ct_terms_authorization_expires_at'] ) ) { return false; } + return array( 'profile_id' => 'external-web-v1', 'profile_sha256' => ct_scan_external_profile_digest(), 'services' => $services, 'expires_at_utc' => $values['_ct_terms_authorization_expires_at'] ); +} + +/** A site adapter supplies a frozen envelope; it cannot substitute a boolean for core validation. */ +function ct_scan_external_evidence( $request_id, $fields, $binding ) { + if ( ! function_exists( 'ct_scan_read_automatic_evidence_v1' ) ) { return false; } + try { $envelope = ct_scan_read_automatic_evidence_v1( $request_id, $fields ); } catch ( Throwable $error ) { return false; } + $map = array( + 'release_sha256' => '_ct_auto_release_sha256', 'release_snapshot' => '_ct_auto_release_snapshot', + 'terms_version' => '_ct_terms_version', 'terms_html' => '_ct_terms_snapshot', 'terms_sha256' => '_ct_terms_sha256', + 'acceptance_statement' => '_ct_terms_statement', 'profile_id' => '_ct_terms_profile_id', 'profile_sha256' => '_ct_terms_profile_sha256', + 'profile_snapshot' => '_ct_terms_profile_snapshot', 'scope_services_json' => '_ct_terms_scope_services', + 'target' => '_ct_scan_target', 'legal_name' => '_ct_terms_client_legal_name', 'signer_name' => '_ct_terms_signer_name', + 'signer_role' => '_ct_terms_signer_role', 'typed_signature' => '_ct_terms_typed_signature', 'email' => '_ct_scan_email', 'referrer' => '_ct_ref', + 'summary_text' => '_ct_summary_text', 'summary_sha256' => '_ct_summary_sha256', 'expires_at_utc' => '_ct_terms_authorization_expires_at', + 'accepted_at_utc' => '_ct_terms_accepted_at', 'idempotency_key' => '_ct_auto_idempotency_key', 'evidence_sha256' => '_ct_auto_evidence_sha256', + 'review_id' => '_ct_auto_review_id', 'review_issued_at_utc' => '_ct_auto_review_issued_at', 'review_expires_at_utc' => '_ct_auto_review_expires_at', + ); + if ( ! is_array( $envelope ) || ! ct_scan_expected_keys( $envelope, array_merge( array_keys( $map ), array( 'scope_services' ) ) ) ) { return false; } + foreach ( $map as $field => $key ) { + $rows = get_post_meta( $request_id, $key, false ); + if ( ! is_string( $envelope[$field] ) || strlen( $envelope[$field] ) > 1048576 || ! is_array( $rows ) || 1 !== count( $rows ) || ! is_string( $rows[0] ) || ! hash_equals( $rows[0], $envelope[$field] ) ) { return false; } + } + foreach ( array( 'release_sha256', 'terms_sha256', 'profile_sha256', 'summary_sha256', 'evidence_sha256' ) as $field ) { + if ( ! preg_match( '/\A[a-f0-9]{64}\z/', $envelope[$field] ) ) { return false; } + } + foreach ( array( 'idempotency_key', 'review_id' ) as $field ) { + if ( ! preg_match( '/\A[A-Za-z0-9_~-]{16,128}\z/', $envelope[$field] ) ) { return false; } + } + $services = ct_scan_external_services( $envelope['scope_services'], $envelope['target'] ); + if ( false === $services || $services !== $binding['services'] || 'external-web-v1' !== $envelope['profile_id'] + || ! hash_equals( $binding['profile_sha256'], $envelope['profile_sha256'] ) || ! hash_equals( $binding['expires_at_utc'], $envelope['expires_at_utc'] ) + || wp_json_encode( ct_scan_canonicalize( $services ), JSON_UNESCAPED_SLASHES ) !== $envelope['scope_services_json'] + || ! hash_equals( $envelope['release_sha256'], hash( 'sha256', $envelope['release_snapshot'] ) ) + || ! hash_equals( $envelope['terms_sha256'], hash( 'sha256', $envelope['terms_html'] ) ) + || ! hash_equals( $envelope['summary_sha256'], hash( 'sha256', $envelope['summary_text'] ) ) ) { return false; } + $release = json_decode( $envelope['release_snapshot'], true ); + if ( JSON_ERROR_NONE !== json_last_error() || ! is_array( $release ) || 1 !== ( $release['schema_version'] ?? null ) + || ! isset( $release['release'], $release['profile'], $release['terms'], $release['copy'] ) || ! is_array( $release['release'] ) || ! is_array( $release['profile'] ) || ! is_array( $release['terms'] ) || ! is_array( $release['copy'] ) + || wp_json_encode( ct_scan_canonicalize( $release ), JSON_UNESCAPED_SLASHES ) . "\n" !== $envelope['release_snapshot'] + || ( $release['release']['terms_version'] ?? null ) !== $envelope['terms_version'] + || ( $release['profile']['profile_id'] ?? null ) !== $envelope['profile_id'] || ( $release['profile']['sha256'] ?? null ) !== $envelope['profile_sha256'] + || ( $release['profile']['manifest_json'] ?? null ) !== $envelope['profile_snapshot'] + || ( $release['terms']['html'] ?? null ) !== $envelope['terms_html'] || ( $release['terms']['sha256'] ?? null ) !== $envelope['terms_sha256'] + || ( $release['copy']['acceptance_statement'] ?? null ) !== $envelope['acceptance_statement'] + || 2592000 !== ( $release['release']['authorization_max_seconds'] ?? null ) ) { return false; } + if ( ! preg_match( '/\A(\d{4}-\d{2}-\d{2})-r[1-9][0-9]{0,2}\z/', $envelope['terms_version'], $version_match ) || ! ct_scan_utc_parse( $version_match[1] . 'T00:00:00Z' ) ) { return false; } + $contract = get_post_meta( $request_id, '_ct_terms_contract', false ); + $plain = $release['terms']['plain_text'] ?? null; + if ( ! is_string( $plain ) || '' === $plain || strlen( $plain ) > 524288 || ! is_array( $contract ) || 1 !== count( $contract ) || ! is_string( $contract[0] ) ) { return false; } + $parts = explode( "\nTERMS AS ACCEPTED\n", $contract[0], 2 ); + if ( 2 !== count( $parts ) || ! hash_equals( $plain, $parts[1] ) ) { return false; } + $format = get_post_meta( $request_id, '_ct_auto_contract_format', false ); + $post = get_post( $request_id ); + $technical_raw = get_post_meta( $request_id, '_ct_auto_technical_snapshot', false ); + $technical_sha = get_post_meta( $request_id, '_ct_auto_technical_sha256', false ); + $terms_url = get_post_meta( $request_id, '_ct_terms_url', false ); + if ( ! is_array( $format ) || array( 'escaped-pre-v1' ) !== $format || ! $post || ! is_array( $technical_raw ) || 1 !== count( $technical_raw ) || ! is_string( $technical_raw[0] ) + || strlen( $technical_raw[0] ) > 8192 || ! is_array( $technical_sha ) || 1 !== count( $technical_sha ) || ! is_string( $technical_sha[0] ) + || ! hash_equals( hash( 'sha256', $technical_raw[0] ), $technical_sha[0] ) || ! is_array( $terms_url ) || 1 !== count( $terms_url ) || ! is_string( $terms_url[0] ) ) { return false; } + $technical = json_decode( $technical_raw[0], true ); + if ( JSON_ERROR_NONE !== json_last_error() || ! is_array( $technical ) || ! ct_scan_expected_keys( $technical, array( 'client_ip', 'remote_addr', 'ip_source', 'user_agent', 'idempotency_key', 'evidence_sha256', 'accepted_at_utc' ) ) + || wp_json_encode( ct_scan_canonicalize( $technical ), JSON_UNESCAPED_SLASHES ) !== $technical_raw[0] ) { return false; } + foreach ( array( 'client_ip' => '_ct_terms_ip', 'remote_addr' => '_ct_terms_remote_addr', 'ip_source' => '_ct_terms_ip_source', 'user_agent' => '_ct_terms_user_agent' ) as $field => $key ) { + $values = get_post_meta( $request_id, $key, false ); + if ( ! is_string( $technical[$field] ) || strlen( $technical[$field] ) > 1024 || ! is_array( $values ) || 1 !== count( $values ) || $values[0] !== $technical[$field] ) { return false; } + } + foreach ( array( 'idempotency_key', 'evidence_sha256', 'accepted_at_utc' ) as $field ) { if ( $technical[$field] !== $envelope[$field] ) { return false; } } + $context = array( 'schema_version' => 1, 'envelope' => $envelope, 'technical' => $technical, 'terms_url' => $terms_url[0] ); + $body = 'CT-AUTO-EVIDENCE-V1 ' . base64_encode( wp_json_encode( ct_scan_canonicalize( $context ), JSON_UNESCAPED_SLASHES ) ) . "\n" . ct_scan_automatic_contract_body_v1( $contract[0] ); + if ( $post->post_content !== $body ) { return false; } + $accepted = ct_scan_utc_parse( $envelope['accepted_at_utc'] ); + $issued = ct_scan_utc_parse( $envelope['review_issued_at_utc'] ); + $review_end = ct_scan_utc_parse( $envelope['review_expires_at_utc'] ); + $expires = ct_scan_utc_parse( $envelope['expires_at_utc'] ); + $cap = ct_scan_utc_parse( $release['release']['authorization_cap_utc'] ?? null ); + if ( ! $accepted || ! $issued || ! $review_end || ! $expires || ! $cap || $issued > $accepted || $accepted > $review_end + || $review_end->getTimestamp() - $issued->getTimestamp() > 900 || $expires <= $accepted || $expires > $accepted->modify( '+30 days' ) || $expires > $cap ) { return false; } + $without_digest = $envelope; + unset( $without_digest['evidence_sha256'] ); + $canonical = wp_json_encode( ct_scan_canonicalize( $without_digest ), JSON_UNESCAPED_SLASHES ); + $whole = get_post_meta( $request_id, '_ct_auto_envelope', false ); + if ( ! is_string( $canonical ) || ! hash_equals( $envelope['evidence_sha256'], hash( 'sha256', $canonical ) ) || ! is_array( $whole ) || 1 !== count( $whole ) + || $whole[0] !== wp_json_encode( ct_scan_canonicalize( $envelope ), JSON_UNESCAPED_SLASHES ) ) { return false; } + return $envelope; } /** Disclosed limits of the exact-target control check. */ @@ -49,6 +220,11 @@ function ct_scan_control_limits() { } function ct_scan_profile_limits( $profile_id ) { + if ( 'external-web-v1' === $profile_id ) { + $manifest = ct_scan_external_profile_manifest(); + if ( ! $manifest || ! isset( $manifest['limits'] ) || ! is_array( $manifest['limits'] ) ) { return false; } + return array_merge( $manifest['limits'], array( 'http_requests' => 100, 'min_interval_ms' => 1000, 'valid_ms' => 1000, 'timeout_ms' => 10000 ) ); + } $profiles = array( 'r2-header-v1' => array( 'http_requests' => 20, @@ -72,6 +248,7 @@ function ct_scan_config() { $config = array( 'enabled' => false, + 'execution_profile' => 'r2-header-v1', 'supported_terms_version' => CT_SCAN_DEFAULT_TERMS_VERSION, 'expected_checker_revision'=> '', 'expected_rules_as_of' => '2026-10-21', @@ -121,6 +298,8 @@ function ct_scan_config() { && strlen( $config['worker_hmac_secret'] ) >= 32 && true === $config['egress_enforced'] && ct_scan_policy_declared( $config ) + && false !== ct_scan_profile_limits( $config['execution_profile'] ) + && ( 'external-web-v1' !== $config['execution_profile'] || ( ! ct_scan_is_legacy_version( $config['supported_terms_version'] ) && 'strict' === $config['terms_policy'][ $config['supported_terms_version'] ] ) ) && is_string( $config['expected_checker_revision'] ) && 1 === preg_match( '/\A[a-f0-9]{40}\z/i', $config['expected_checker_revision'] ) && is_string( $config['supported_terms_version'] ) @@ -170,13 +349,30 @@ function ct_scan_policy_declared( $config ) { * The canonical signer-aware Target Summary for strict terms versions. It is deterministic from the * accepted request fields and the code-defined profile limits. Returns '' for an unknown profile. */ -function ct_scan_target_summary( $target, $legal_name, $signer_name, $signer_role, $email, $profile = null ) { +function ct_scan_target_summary( $target, $legal_name, $signer_name, $signer_role, $email, $profile = null, $services = null, $expires_at_utc = null ) { $profile = null === $profile ? ct_scan_current_profile() : $profile; $limits = ct_scan_profile_limits( $profile ); if ( ! $limits || ! is_string( $target ) || ! is_string( $legal_name ) || ! is_string( $signer_name ) || ! is_string( $signer_role ) || ! is_string( $email ) || '' === $legal_name || '' === $signer_name || '' === $signer_role || '' === $email ) { return ''; } + if ( 'external-web-v1' === $profile ) { + $services = ct_scan_external_services( $services, $target ); + if ( false === $services || ! ct_scan_utc_parse( $expires_at_utc ) ) { return ''; } + $names = array_map( static function ( $service ) { return $service['scheme'] . ':' . $service['port']; }, $services ); + return 'Website (scan scope): ' . $target . "\n" + . 'Client legal name: ' . $legal_name . "\n" + . 'Signer: ' . $signer_name . "\n" + . 'Signer role: ' . $signer_role . "\n" + . 'Client email: ' . $email . "\n" + . 'Selected services: ' . implode( ', ', $names ) . "\n" + . 'Profile: external-web-v1' . "\n" + . 'Profile SHA-256: ' . ct_scan_external_profile_digest() . "\n" + . "Limits: at most 100 HTTP transactions, 20 additional TLS-only checks, 600 cumulative active seconds, 5242880 body bytes; one connection at a time and at most one check per second\n" + . "Response limits: 262144 decompressed body bytes, 32768 header bytes, 10 seconds per request; no more than two transient retries and three redirect hops per check\n" + . 'Authorization expires (UTC): ' . $expires_at_utc . "\n" + . "Early closure: completion, cancellation, or an earlier accepted deadline ends new target requests\n"; + } return 'Website (scan scope): ' . $target . "\n" . 'Client legal name: ' . $legal_name . "\n" . 'Signer: ' . $signer_name . "\n" @@ -280,7 +476,14 @@ function ct_scan_maybe_install_schema() { closure_reason varchar(64) NULL, revoked_at datetime NULL, profile_id varchar(32) NOT NULL DEFAULT 'r2-header-v1', + profile_sha256 char(64) NULL, + services_json varchar(256) NULL, http_admitted smallint(5) unsigned NOT NULL DEFAULT 0, + tls_admitted smallint(5) unsigned NOT NULL DEFAULT 0, + reserved_body_bytes int(10) unsigned NOT NULL DEFAULT 0, + reserved_active_ms int(10) unsigned NOT NULL DEFAULT 0, + inflight_until_ms bigint(20) unsigned NULL, + reservation_version tinyint(1) unsigned NOT NULL DEFAULT 0, last_admit_ms bigint(20) unsigned NULL, summary_sha256 char(64) NULL, bound_required tinyint(1) unsigned NOT NULL DEFAULT 0, @@ -345,7 +548,7 @@ function ct_scan_maybe_install_schema() { ) {$charset};"; dbDelta( $sql ); $required = array( - $jobs => array( 'job_id', 'request_id', 'target_origin', 'scope_hash', 'terms_version', 'terms_hash', 'terms_accepted_at', 'authorization_expires_at', 'status', 'lease_hash', 'result_json', 'receipt_hash', 'report_notification_status', 'report_notification_at', 'report_delivered_at', 'closure_source', 'closed_by', 'closure_reason', 'report_access_token_hash', 'report_access_expires_at', 'report_access_used_at', 'stop_token_hash', 'profile_id', 'http_admitted', 'last_admit_ms', 'summary_sha256', 'bound_required', 'accepted_copy_status', 'accepted_copy_at', 'control_required', 'control_state', 'control_token_hash', 'control_page_hash', 'control_requested_method', 'control_attempts', 'control_last_attempt_ms', 'control_leased_ms', 'control_admitted', 'control_lease_hash', 'control_lease_until', 'control_method', 'control_verified_at', 'control_valid_until', 'control_evidence', 'closed_at', 'retain_until', 'hold_reason', 'hold_until', 'hold_set_by', 'hold_set_at', 'hold_released_by', 'hold_released_at', 'identity', 'retention_checked_at' ), + $jobs => array( 'job_id', 'request_id', 'target_origin', 'scope_hash', 'terms_version', 'terms_hash', 'terms_accepted_at', 'authorization_expires_at', 'status', 'lease_hash', 'result_json', 'receipt_hash', 'report_notification_status', 'report_notification_at', 'report_delivered_at', 'closure_source', 'closed_by', 'closure_reason', 'report_access_token_hash', 'report_access_expires_at', 'report_access_used_at', 'stop_token_hash', 'profile_id', 'http_admitted', 'last_admit_ms', 'summary_sha256', 'bound_required', 'accepted_copy_status', 'accepted_copy_at', 'control_required', 'control_state', 'control_token_hash', 'control_page_hash', 'control_requested_method', 'control_attempts', 'control_last_attempt_ms', 'control_leased_ms', 'control_admitted', 'control_lease_hash', 'control_lease_until', 'control_method', 'control_verified_at', 'control_valid_until', 'control_evidence', 'closed_at', 'retain_until', 'hold_reason', 'hold_until', 'hold_set_by', 'hold_set_at', 'hold_released_by', 'hold_released_at', 'identity', 'retention_checked_at', 'profile_sha256', 'services_json', 'tls_admitted', 'reserved_body_bytes', 'reserved_active_ms', 'inflight_until_ms', 'reservation_version' ), $nonces => array( 'nonce_hash', 'expires_at' ), $tombstones => array( 'tombstone_id', 'job_id', 'request_id', 'scope_mac', 'created_at', 'deleted_at', 'reason', 'identity' ), ); @@ -444,6 +647,31 @@ function ct_scan_utc_parse( $value ) { } } +/** Pure producer helpers use the same versioned records the independent reader checks below. */ +function ct_scan_authority_digest_v1( $legal_name, $signer_name, $signer_role, $signature ) { + foreach ( array( $legal_name, $signer_name, $signer_role, $signature ) as $value ) { + $length = is_string( $value ) ? ( function_exists( 'mb_strlen' ) ? mb_strlen( $value, 'UTF-8' ) : strlen( $value ) ) : 0; + if ( ! is_string( $value ) || '' === $value || trim( $value ) !== $value || $length > 120 || preg_match( '/[\x00-\x1f\x7f<>]/', $value ) || sanitize_text_field( $value ) !== $value ) { return false; } + } + if ( ! hash_equals( $signer_name, $signature ) ) { return false; } + return hash( 'sha256', 'Client legal name: ' . $legal_name . "\nSigner full name: " . $signer_name . "\nSigner role (stated): " . $signer_role . "\nTyped signature: " . $signature . "\n" ); +} + +function ct_scan_acceptance_digest_v1( $terms_version, $terms_sha256, $summary_sha256, $authority_sha256, $statement, $accepted_at_utc ) { + if ( ! is_string( $terms_version ) || ! preg_match( '/\A\d{4}-\d{2}-\d{2}-r[1-9][0-9]{0,2}\z/', $terms_version ) + || ! is_string( $statement ) || '' === $statement || strlen( $statement ) > 8192 || ! ct_scan_utc_parse( $accepted_at_utc ) ) { return false; } + foreach ( array( $terms_sha256, $summary_sha256, $authority_sha256 ) as $hash ) { + if ( ! is_string( $hash ) || ! preg_match( '/\A[a-f0-9]{64}\z/', $hash ) ) { return false; } + } + return hash( 'sha256', 'Terms version: ' . $terms_version . "\nTerms SHA-256: " . $terms_sha256 . "\nTarget Summary SHA-256: " . $summary_sha256 . "\nSigner evidence SHA-256: " . $authority_sha256 . "\nAcceptance statement SHA-256: " . hash( 'sha256', $statement ) . "\nAccepted at (UTC): " . $accepted_at_utc . "\n" ); +} + +/** Preserve raw signed bytes without asking WordPress to treat untrusted identity text as markup. */ +function ct_scan_automatic_contract_body_v1( $contract ) { + if ( ! is_string( $contract ) || '' === $contract || strlen( $contract ) > 1048576 || 1 !== preg_match( '//u', $contract ) ) { return false; } + return 'CT-AUTOMATIC-CONTRACT-BASE64: ' . base64_encode( $contract ) . "\n
" . htmlspecialchars( $contract, ENT_QUOTES | ENT_SUBSTITUTE, 'UTF-8' ) . '
'; +} + function ct_scan_meta_string( $post_id, $key ) { $value = get_post_meta( $post_id, $key, true ); return is_string( $value ) ? $value : ''; @@ -511,7 +739,7 @@ function ct_scan_check_bound_evidence( $request_id, $terms_hash, $identity, $req // profile, limits, authorization period): nothing before, between or after, so no decoy or appended // instruction can ride along, and the limits the signer saw are the limits the job runs under. if ( ! empty( $identity['strict_authority'] ) ) { - $expected = ct_scan_target_summary( $identity['target'], $identity['legal_name'], $identity['signer_name'], $identity['signer_role'], $identity['email'], $profile ); + $expected = ct_scan_target_summary( $identity['target'], $identity['legal_name'], $identity['signer_name'], $identity['signer_role'], $identity['email'], $profile, $identity['services'] ?? null, $identity['expires_at_utc'] ?? null ); } else { $expected = ct_scan_legacy_target_summary( $identity['target'], $identity['name'], $identity['email'], $profile ); } @@ -602,6 +830,7 @@ function ct_scan_validate_saved_request( $request_id, $fields, $require_current_ // Authorization is judged on what the database holds now, never on an object-cache copy. clean_post_cache( (int) $request_id ); $config = ct_scan_config(); + $profile = is_array( $frozen ) ? (string) $frozen['profile'] : ct_scan_current_profile(); $post = get_post( $request_id ); if ( ! $post || 'ct_promo_request' !== $post->post_type || 'private' !== $post->post_status ) { return new WP_Error( 'ct_scan_request_missing', 'The saved private request could not be verified.' ); @@ -663,14 +892,41 @@ function ct_scan_validate_saved_request( $request_id, $fields, $require_current_ return new WP_Error( 'ct_scan_contract_mismatch', 'The saved acceptance copy does not match the authorized request fields.' ); } } - if ( false === strpos( $post->post_content, $contract ) ) { + $stored_contract = 'external-web-v1' === $profile ? ct_scan_automatic_contract_body_v1( $contract ) : $contract; + if ( false === $stored_contract || false === strpos( $post->post_content, $stored_contract ) ) { return new WP_Error( 'ct_scan_contract_readback_mismatch', 'The saved acceptance copy could not be read back from the request.' ); } // A job's evidence requirement and profile are frozen when it is created, so a later // configuration change can neither relax nor tighten an authorization already given. $bound_required = is_array( $frozen ) ? (bool) $frozen['bound_required'] : 'strict' === ct_scan_terms_policy( $terms_version ); $profile = is_array( $frozen ) ? (string) $frozen['profile'] : ct_scan_current_profile(); - $signer = ct_scan_read_signer_evidence( $request_id, $terms_version, $terms_hash, ct_scan_meta_string( $request_id, '_ct_summary_sha256' ), $accepted_raw, $name, $contract, $bound_required ); + $profile_binding = null; + $automatic = null; + $identity_contract = $contract; + if ( 'external-web-v1' === $profile ) { + $profile_binding = ct_scan_external_saved_binding( $request_id, $target ); + if ( ! $bound_required || ct_scan_is_legacy_version( $terms_version ) || ! $profile_binding ) { + return new WP_Error( 'ct_scan_profile_binding_invalid', 'The saved automatic profile authorization could not be verified.' ); + } + $automatic = ct_scan_external_evidence( $request_id, $fields, $profile_binding ); + if ( ! $automatic || $automatic['target'] !== $target || $automatic['email'] !== $email || $automatic['signer_name'] !== $name || $automatic['referrer'] !== $ref ) { + return new WP_Error( 'ct_scan_automatic_evidence_invalid', 'The saved automatic acceptance envelope could not be verified.' ); + } + if ( is_array( $frozen ) && ( ! isset( $frozen['profile_sha256'], $frozen['services_json'] ) || ! is_string( $frozen['profile_sha256'] ) + || ! hash_equals( $profile_binding['profile_sha256'], $frozen['profile_sha256'] ) || $frozen['services_json'] !== $automatic['scope_services_json'] ) ) { + return new WP_Error( 'ct_scan_profile_binding_changed', 'The accepted services or profile no longer match the frozen job.' ); + } + // The automatic representation separately binds its exact canonical Summary. + // Its repeated identity labels are not a second signer header. Keep legacy + // all-header checks unchanged and check the automatic header independently. + $sections = explode( "\n\n", $contract, 2 ); + $terms = explode( "\nTERMS AS ACCEPTED\n", $contract, 2 ); + if ( 2 !== count( $sections ) || 2 !== count( $terms ) || $sections[1] !== $automatic['summary_text'] . "\nTERMS AS ACCEPTED\n" . $terms[1] ) { + return new WP_Error( 'ct_scan_contract_mismatch', 'The automatic header and canonical Summary boundary could not be verified.' ); + } + $identity_contract = $sections[0] . "\nTERMS AS ACCEPTED\n" . $terms[1]; + } + $signer = ct_scan_read_signer_evidence( $request_id, $terms_version, $terms_hash, ct_scan_meta_string( $request_id, '_ct_summary_sha256' ), $accepted_raw, $name, $identity_contract, $bound_required ); if ( is_wp_error( $signer ) ) { return $signer; } @@ -681,7 +937,7 @@ function ct_scan_validate_saved_request( $request_id, $fields, $require_current_ 'Typed signature' => $signer['signer_name'], ) ); foreach ( $expected_lines as $label => $expected ) { - if ( 1 !== preg_match_all( '/^' . preg_quote( $label, '/' ) . ':[ \\t]*(.*)$/m', $contract, $matches ) || trim( $matches[1][0] ) !== $expected ) { + if ( 1 !== preg_match_all( '/^' . preg_quote( $label, '/' ) . ':[ \\t]*(.*)$/m', $identity_contract, $matches ) || trim( $matches[1][0] ) !== $expected ) { return new WP_Error( 'ct_scan_contract_mismatch', 'The saved acceptance copy does not match the authorized request fields.' ); } } @@ -694,12 +950,24 @@ function ct_scan_validate_saved_request( $request_id, $fields, $require_current_ 'legal_name' => $signer['legal_name'], 'signer_name' => $signer['signer_name'], 'signer_role' => $signer['signer_role'], + 'services' => $profile_binding['services'] ?? null, + 'expires_at_utc' => $profile_binding['expires_at_utc'] ?? null, ); $bound = ct_scan_check_bound_evidence( $request_id, $terms_hash, $summary_identity, $bound_required, $profile ); if ( is_wp_error( $bound ) ) { return $bound; } - if ( $require_current_terms ) { + if ( 'external-web-v1' === $profile ) { + $expires = ct_scan_utc_parse( $profile_binding['expires_at_utc'] ); + if ( ! $expires || $expires > $accepted_at->modify( '+30 days' ) || $expires <= $accepted_at + || ( '' !== $frozen_expiry && $expires->format( 'Y-m-d H:i:s' ) !== $frozen_expiry ) ) { + return new WP_Error( 'ct_scan_expiry_invalid', 'The frozen automatic authorization expiry could not be verified.' ); + } + if ( $require_current_terms ) { + $window_end = ct_scan_utc_parse( $config['authorization_end_at'] ); + if ( ! $window_end || $expires > $window_end ) { return new WP_Error( 'ct_scan_expiry_invalid', 'The displayed authorization exceeds the configured publication window.' ); } + } + } elseif ( $require_current_terms ) { $window_end = ct_scan_utc_parse( $config['authorization_end_at'] ); if ( ! $window_end ) { return new WP_Error( 'ct_scan_config_invalid', 'The authorization end date is not configured.' ); @@ -740,6 +1008,9 @@ function ct_scan_validate_saved_request( $request_id, $fields, $require_current_ 'authorization_expires' => $expires->setTimezone( new DateTimeZone( 'UTC' ) )->format( 'Y-m-d H:i:s' ), 'summary_sha256' => $bound['summary_sha256'], 'bound_required' => $bound_required, + 'profile_id' => $profile, + 'profile_sha256' => $profile_binding['profile_sha256'] ?? null, + 'services_json' => $automatic['scope_services_json'] ?? null, ); } @@ -896,7 +1167,10 @@ function ct_scan_on_request_ready( $request_id, $fields ) { 'confirmation_token_hash' => hash( 'sha256', $token ), 'confirmation_expires_at' => $confirm_expires, 'stop_token_hash' => hash( 'sha256', $stop_token ), - 'profile_id' => ct_scan_current_profile(), + 'profile_id' => $accepted['profile_id'], + 'profile_sha256' => $accepted['profile_sha256'], + 'services_json' => $accepted['services_json'], + 'reservation_version' => 'external-web-v1' === $accepted['profile_id'] ? 1 : 0, 'summary_sha256' => $accepted['summary_sha256'], 'bound_required' => $accepted['bound_required'] ? 1 : 0, // Opaque identity mirrored on the private request: lets a deletion tombstone recognise @@ -907,7 +1181,7 @@ function ct_scan_on_request_ready( $request_id, $fields ) { // A profile broader than the r2 header screening can never run without target control. 'control_required' => ( 'strict' === ct_scan_terms_policy( $accepted['terms_version'] ) || 'r2-header-v1' !== ct_scan_current_profile() ) ? 1 : 0, ), - array( '%d', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%d', '%s', '%d' ) + null ); if ( 1 !== $inserted ) { $existing = $wpdb->get_row( $wpdb->prepare( "SELECT * FROM {$jobs} WHERE request_id = %d LIMIT 1", $request_id ) ); @@ -997,7 +1271,7 @@ function ct_scan_authorization_is_live( $row ) { 'name' => $row->contact_name, 'referrer' => $row->referrer, ); - $accepted = ct_scan_validate_saved_request( (int) $row->request_id, $fields, false, $row->authorization_expires_at, array( 'bound_required' => ! empty( $row->bound_required ), 'profile' => (string) $row->profile_id ) ); + $accepted = ct_scan_validate_saved_request( (int) $row->request_id, $fields, false, $row->authorization_expires_at, array( 'bound_required' => ! empty( $row->bound_required ), 'profile' => (string) $row->profile_id, 'profile_sha256' => $row->profile_sha256 ?? null, 'services_json' => $row->services_json ?? null ) ); if ( is_wp_error( $accepted ) ) { return false; } @@ -1046,7 +1320,7 @@ function ct_scan_maintenance_locked() { $now, $now ) ); - $queued = $wpdb->get_results( "SELECT job_id,request_id,target_origin,scope_hash,terms_version,terms_hash,terms_accepted_at,authorization_expires_at,contact_email,contact_name,referrer,revoked_at,report_delivered_at,summary_sha256,bound_required,profile_id,status,control_required,control_verified_at,control_valid_until FROM {$table} WHERE status = 'queued'" ); + $queued = $wpdb->get_results( "SELECT job_id,request_id,target_origin,scope_hash,terms_version,terms_hash,terms_accepted_at,authorization_expires_at,contact_email,contact_name,referrer,revoked_at,report_delivered_at,summary_sha256,bound_required,profile_id,profile_sha256,services_json,status,control_required,control_verified_at,control_valid_until FROM {$table} WHERE status = 'queued'" ); foreach ( (array) $queued as $row ) { if ( ! ct_scan_authorization_is_live( $row ) ) { $wpdb->update( $table, array( 'status' => 'cancelled', 'error_code' => 'authorization_changed', 'finished_at' => $now, 'lease_hash' => null, 'lease_until' => null ), array( 'job_id' => (int) $row->job_id ), array( '%s', '%s', '%s', '%s', '%s' ), array( '%d' ) ); @@ -2717,6 +2991,8 @@ function ct_scan_claim_locked( WP_REST_Request $request ) { ct_scan_mark_request_status( (int) $row->request_id, 'cancelled', 'authorization_changed' ); return ct_scan_rest_response( 200, array( 'job' => null ) ); } + $profile_fields = ct_scan_job_profile_fields( $row ); + if ( false === $profile_fields ) { return ct_scan_rest_error( 410, 'profile_binding_invalid' ); } $lease = bin2hex( random_bytes( 32 ) ); $expiry = strtotime( $row->authorization_expires_at . ' UTC' ); $lease_until = min( time() + 300, $expiry ); @@ -2734,7 +3010,7 @@ function ct_scan_claim_locked( WP_REST_Request $request ) { return ct_scan_rest_error( 503, 'claim_race' ); } return ct_scan_rest_response( 200, array( - 'job' => array( + 'job' => array_merge( $profile_fields, array( 'id' => (int) $row->job_id, 'target' => $row->target_origin, 'scope_hash' => $row->scope_hash, @@ -2742,7 +3018,7 @@ function ct_scan_claim_locked( WP_REST_Request $request ) { 'expires_at' => (int) $expiry, 'lease' => $lease, 'lease_until' => (int) $lease_until, - ), + ) ), ) ); } finally { ct_scan_lock_release(); @@ -2758,6 +3034,31 @@ function ct_scan_callback_payload( $data ) { return true; } +/** Frozen additive wire fields. Legacy jobs never acquire the automatic profile. */ +function ct_scan_job_profile_fields( $row ) { + if ( 'r2-header-v1' === ( $row->profile_id ?? null ) ) { return array(); } + if ( 'external-web-v1' !== ( $row->profile_id ?? null ) || ! isset( $row->profile_sha256, $row->services_json, $row->target_origin ) + || ! is_string( $row->profile_sha256 ) || ! is_string( $row->services_json ) + || ! hash_equals( ct_scan_external_profile_digest(), $row->profile_sha256 ) || ! ct_scan_external_profile_manifest() ) { return false; } + $services = json_decode( $row->services_json, true ); + $services = JSON_ERROR_NONE === json_last_error() ? ct_scan_external_services( $services, $row->target_origin ) : false; + if ( false === $services || wp_json_encode( ct_scan_canonicalize( $services ), JSON_UNESCAPED_SLASHES ) !== $row->services_json ) { return false; } + return array( 'profile_id' => 'external-web-v1', 'profile_sha256' => $row->profile_sha256, 'services' => $services ); +} + +function ct_scan_result_profile_matches( $data, $row ) { + $profile = ct_scan_job_profile_fields( $row ); + if ( false === $profile ) { return false; } + $keys = array( 'id', 'lease', 'scope_hash', 'outcome', 'result', 'error', 'checker_revision' ); + if ( ! ct_scan_expected_keys( $data, array_merge( $keys, array_keys( $profile ) ) ) ) { return false; } + foreach ( $profile as $key => $value ) { + if ( 'services' === $key ) { + if ( ct_scan_external_services( $data[$key], $row->target_origin ) !== $value ) { return false; } + } elseif ( ! is_string( $data[$key] ) || ! hash_equals( $value, $data[$key] ) ) { return false; } + } + return true; +} + function ct_scan_start( WP_REST_Request $request ) { $body = ct_scan_rest_authenticate( $request, '/ct-scan/v1/start' ); if ( is_wp_error( $body ) ) { @@ -2863,6 +3164,10 @@ function ct_scan_admit_locked( $data, $now_ms ) { if ( ! $limits ) { return $refuse( 410, 'profile_unknown' ); } + if ( 'external-web-v1' === $row->profile_id ) { + return ct_scan_external_admit_locked( $row, $data, $now_ms ); + } + if ( ! in_array( $data['kind'], array( 'http', 'probe' ), true ) ) { return $refuse( 400, 'admission_kind_invalid' ); } if ( 'probe' === $data['kind'] ) { return array( 200, array( 'ok' => true, 'live' => true ) ); } @@ -2901,6 +3206,61 @@ function ct_scan_admit_locked( $data, $now_ms ) { ) ); } +/** Read numeric database fields without silently turning malformed state into a fresh budget. */ +function ct_scan_external_counter( $value, $maximum ) { + if ( is_bool( $value ) || ( ! is_int( $value ) && ! is_string( $value ) ) || ! preg_match( '/\A(?:0|[1-9][0-9]{0,15})\z/', (string) $value ) ) { return false; } + $number = (int) $value; + return $number >= 0 && $number <= $maximum ? $number : false; +} + +/** Caller already holds the queue lock and checked live authorization, recipient and lease. */ +function ct_scan_external_admit_locked( $row, $data, $now_ms ) { + global $wpdb; + $refuse = static function ( $status, $reason, $extra = array() ) { return array( $status, array_merge( array( 'ok' => false, 'reason' => $reason ), $extra ) ); }; + if ( ! is_int( $now_ms ) || $now_ms < 1577836800000 || ! in_array( $data['kind'], array( 'http', 'tls', 'probe' ), true ) ) { return $refuse( 400, 'admission_kind_invalid' ); } + $fields = array( 'http_admitted' => 100, 'tls_admitted' => 20, 'reserved_body_bytes' => 5242880, 'reserved_active_ms' => 600000, 'reservation_version' => 1 ); + $state = array(); + foreach ( $fields as $field => $maximum ) { + if ( ! property_exists( $row, $field ) || false === ( $state[$field] = ct_scan_external_counter( $row->$field, $maximum ) ) ) { return $refuse( 410, 'budget_state_invalid' ); } + } + if ( 1 !== $state['reservation_version'] || ! isset( $row->profile_sha256, $row->services_json ) || ! is_string( $row->profile_sha256 ) || ! is_string( $row->services_json ) + || ! hash_equals( ct_scan_external_profile_digest(), $row->profile_sha256 ) || ! ct_scan_external_profile_manifest() + || $state['reserved_body_bytes'] !== $state['http_admitted'] * 262144 || $state['reserved_active_ms'] !== ( $state['http_admitted'] + $state['tls_admitted'] ) * 10000 + || ! property_exists( $row, 'last_admit_ms' ) || ! property_exists( $row, 'inflight_until_ms' ) || ! isset( $row->started_at ) || ! ct_scan_valid_datetime( $row->started_at ) ) { return $refuse( 410, 'budget_state_invalid' ); } + $services = json_decode( $row->services_json, true ); + $services = JSON_ERROR_NONE === json_last_error() ? ct_scan_external_services( $services, $row->target_origin ) : false; + if ( false === $services || wp_json_encode( ct_scan_canonicalize( $services ), JSON_UNESCAPED_SLASHES ) !== $row->services_json ) { return $refuse( 410, 'profile_binding_invalid' ); } + $count = $state['http_admitted'] + $state['tls_admitted']; + $last = null === $row->last_admit_ms ? null : ct_scan_external_counter( $row->last_admit_ms, PHP_INT_MAX ); + $until = null === $row->inflight_until_ms ? null : ct_scan_external_counter( $row->inflight_until_ms, PHP_INT_MAX ); + if ( ( 0 === $count && ( null !== $last || null !== $until ) ) || ( $count > 0 && ( false === $last || false === $until || null === $last || null === $until || $until !== $last + 11000 ) ) ) { return $refuse( 410, 'budget_state_invalid' ); } + $started_ms = strtotime( $row->started_at . ' UTC' ) * 1000; + if ( $now_ms < $started_ms || ( null !== $last && $now_ms < $last ) ) { return $refuse( 410, 'clock_anomaly' ); } + if ( $now_ms - $started_ms >= 600000 ) { return $refuse( 409, 'active_budget_exhausted' ); } + if ( 'probe' === $data['kind'] ) { return array( 200, array( 'ok' => true, 'live' => true ) ); } + if ( null !== $until && $now_ms < $until ) { return $refuse( 429, 'rate_wait', array( 'retry_after_ms' => $until - $now_ms ) ); } + $is_http = 'http' === $data['kind']; + if ( ! $is_http && ! in_array( array( 'port' => 443, 'scheme' => 'https' ), $services, true ) ) { return $refuse( 410, 'service_not_selected' ); } + $body = $is_http ? 262144 : 0; + if ( ( $is_http && $state['http_admitted'] >= 100 ) || ( ! $is_http && $state['tls_admitted'] >= 20 ) || $state['reserved_body_bytes'] + $body > 5242880 || $state['reserved_active_ms'] + 10000 > 600000 ) { return $refuse( 409, 'budget_exhausted' ); } + $updated = $wpdb->update( ct_scan_jobs_table(), array( + 'http_admitted' => $state['http_admitted'] + ( $is_http ? 1 : 0 ), 'tls_admitted' => $state['tls_admitted'] + ( $is_http ? 0 : 1 ), + 'reserved_body_bytes' => $state['reserved_body_bytes'] + $body, 'reserved_active_ms' => $state['reserved_active_ms'] + 10000, + 'last_admit_ms' => $now_ms, 'inflight_until_ms' => $now_ms + 11000, + ), array( + 'job_id' => (int) $row->job_id, 'status' => 'running', 'lease_hash' => $row->lease_hash, 'profile_sha256' => $row->profile_sha256, + 'http_admitted' => $row->http_admitted, 'tls_admitted' => $row->tls_admitted, 'reserved_body_bytes' => $row->reserved_body_bytes, + 'reserved_active_ms' => $row->reserved_active_ms, 'last_admit_ms' => $row->last_admit_ms, 'inflight_until_ms' => $row->inflight_until_ms, + 'reservation_version' => 1, 'services_json' => $row->services_json, + ) ); + if ( 1 !== $updated ) { return $refuse( 503, 'admission_write_failed' ); } + return array( 200, array( 'ok' => true, 'admitted' => true, 'sequence' => $count + 1, 'valid_ms' => 1000, 'timeout_ms' => 10000, + 'http_remaining' => 100 - $state['http_admitted'] - ( $is_http ? 1 : 0 ), 'kind' => $data['kind'], 'profile_sha256' => $row->profile_sha256, + 'tls_remaining' => 20 - $state['tls_admitted'] - ( $is_http ? 0 : 1 ), 'body_bytes_remaining' => 5242880 - $state['reserved_body_bytes'] - $body, + 'active_ms_remaining' => 600000 - $state['reserved_active_ms'] - 10000, + ) ); +} + function ct_scan_admit( WP_REST_Request $request ) { $body = ct_scan_rest_authenticate( $request, '/ct-scan/v1/admit' ); if ( is_wp_error( $body ) ) { @@ -2908,7 +3268,7 @@ function ct_scan_admit( WP_REST_Request $request ) { } $data = ct_scan_decode_request_body( $body ); if ( is_wp_error( $data ) || ! ct_scan_expected_keys( $data, array( 'id', 'lease', 'scope_hash', 'kind' ) ) || ! ct_scan_callback_payload( $data ) - || ! is_string( $data['kind'] ) || ! in_array( $data['kind'], array( 'http', 'probe', 'control', 'control_probe' ), true ) ) { + || ! is_string( $data['kind'] ) || ! in_array( $data['kind'], array( 'http', 'tls', 'probe', 'control', 'control_probe' ), true ) ) { return ct_scan_rest_error( 400, 'request_invalid' ); } if ( ! ct_scan_lock_acquire() ) { @@ -2951,8 +3311,116 @@ function ct_scan_result_url_matches_scope( $url, $target ) { && ( 'https' !== $base['scheme'] || 'https' === $scheme ); } +/** Accept only the reviewed fixed observation vocabulary, never raw target material. */ +function ct_scan_external_check_valid( $check ) { + if ( ! is_array( $check ) || ! ct_scan_expected_keys( $check, array( 'id', 'service', 'outcome', 'evidence' ) ) + || ! is_string( $check['id'] ) || ! is_string( $check['service'] ) || ! is_string( $check['outcome'] ) || ! is_array( $check['evidence'] ) ) { return false; } + $id = $check['id']; $outcome = $check['outcome']; $e = $check['evidence']; + if ( 'not_selected' === $outcome ) { return 'tls_certificate' === $id && 'http:80' === $check['service'] && array( 'reason' => 'https_not_selected' ) === $e; } + if ( ct_scan_expected_keys( $e, array( 'reason', 'http_status' ) ) ) { + return 'security_header_presence' === $id && 'unknown' === $outcome && 'non_successful_response' === $e['reason'] + && is_int( $e['http_status'] ) && $e['http_status'] >= 100 && $e['http_status'] <= 599 && ( $e['http_status'] < 200 || $e['http_status'] >= 300 ); + } + if ( ct_scan_expected_keys( $e, array( 'reason' ) ) ) { + return ( 'unknown' === $outcome && in_array( $id, array( 'security_header_presence', 'tls_certificate' ), true ) && 'service_not_assessed' === $e['reason'] ) + || ( 'tls_certificate' === $id && 'blocked' === $outcome && 'tls_verification_failed' === $e['reason'] ); + } + $fields = array( + 'robots_policy' => array( 'http_status', 'allowed' ), + 'public_header_triage' => array( 'http_status', 'fetch_status' ), + 'security_header_presence' => array( 'http_status', 'hsts', 'csp', 'frame_control', 'mime_options' ), + 'tls_certificate' => array( 'verification', 'protocol', 'not_after' ), + ); + if ( ! isset( $fields[$id] ) || ! ct_scan_expected_keys( $e, $fields[$id] ) ) { return false; } + if ( array_key_exists( 'http_status', $e ) && null !== $e['http_status'] && ( ! is_int( $e['http_status'] ) || $e['http_status'] < 100 || $e['http_status'] > 599 ) ) { return false; } + if ( 'robots_policy' === $id ) { return is_bool( $e['allowed'] ) && ( $e['allowed'] ? 'observed' : 'blocked' ) === $outcome; } + if ( 'public_header_triage' === $id ) { + return is_string( $e['fetch_status'] ) && in_array( $e['fetch_status'], array( 'ok', 'blocked', 'timeout', 'dns_error' ), true ) + && ( 'ok' === $e['fetch_status'] ? 'observed' : 'unknown' ) === $outcome + && ( 'ok' !== $e['fetch_status'] || ( is_int( $e['http_status'] ) && $e['http_status'] >= 200 && $e['http_status'] < 300 ) ); + } + if ( 'security_header_presence' === $id ) { + if ( 'observed' !== $outcome || ! is_int( $e['http_status'] ) || $e['http_status'] < 200 || $e['http_status'] >= 300 ) { return false; } + foreach ( array( 'hsts', 'csp', 'frame_control', 'mime_options' ) as $key ) { if ( ! is_bool( $e[$key] ) ) { return false; } } + return true; + } + return 'observed' === $outcome && 'verified' === $e['verification'] && is_string( $e['protocol'] ) + && in_array( $e['protocol'], array( 'TLSv1', 'TLSv1.1', 'TLSv1.2', 'TLSv1.3' ), true ) && is_string( $e['not_after'] ) + && ( '' === $e['not_after'] || false !== ct_scan_utc_parse( $e['not_after'] ) ); +} + +function ct_scan_validate_external_result( $result, $row, $stored_snapshot = false ) { + $bad = static function () { return new WP_Error( 'result_profile_invalid', 'result_profile_invalid' ); }; + $binding = ct_scan_job_profile_fields( $row ); + if ( ! is_array( $result ) || false === $binding || empty( $binding ) ) { return $bad(); } + foreach ( $binding as $key => $value ) { + if ( ! array_key_exists( $key, $result ) ) { return $bad(); } + if ( 'services' === $key ) { + if ( ct_scan_external_services( $result[$key], $row->target_origin ) !== $value ) { return $bad(); } + } elseif ( ! is_string( $result[$key] ) || ! hash_equals( $value, $result[$key] ) ) { return $bad(); } + } + if ( ! isset( $result['checks'], $result['manual_remainder'] ) || ! is_array( $result['checks'] ) || ! ct_scan_is_list( $result['checks'] ) ) { return $bad(); } + $core = $result; + foreach ( array( 'profile_id', 'profile_sha256', 'services', 'checks', 'manual_remainder' ) as $key ) { unset( $core[$key] ); } + $legacy = clone $row; $legacy->profile_id = 'r2-header-v1'; + // The common result validator keeps the classifier/status/date/size contract intact. + if ( is_wp_error( ct_scan_validate_result( $core, $legacy, $stored_snapshot ) ) ) { return $bad(); } + $final = wp_parse_url( $core['final_url'] ); + $service = is_array( $final ) && isset( $final['scheme'] ) ? array( 'port' => $final['port'] ?? ( 'https' === $final['scheme'] ? 443 : 80 ), 'scheme' => $final['scheme'] ) : null; + if ( ! in_array( $service, $binding['services'], true ) ) { return $bad(); } + foreach ( array( 'server', 'powered_by', 'title', 'wordpress_version' ) as $key ) { if ( '' !== $core[$key] ) { return $bad(); } } + foreach ( array( 'evidence', 'asset_version_hints', 'notes' ) as $key ) { if ( array() !== $core[$key] ) { return $bad(); } } + if ( array() !== $core['php_evidence'] && array( 'PHP was mentioned in public response headers; raw values were discarded.' ) !== $core['php_evidence'] ) { return $bad(); } + if ( count( $core['php_version_candidates'] ) > 32 ) { return $bad(); } + foreach ( $core['php_version_candidates'] as $candidate ) { if ( ! preg_match( '/\A[0-9][A-Za-z0-9.+_~:-]{0,63}\z/', $candidate ) ) { return $bad(); } } + $candidates = array_values( array_unique( $core['php_version_candidates'] ) ); sort( $candidates, SORT_STRING ); + if ( $candidates !== $core['php_version_candidates'] || $core['php_version'] !== ( 1 === count( $candidates ) ? $candidates[0] : '' ) ) { return $bad(); } + // The maintained Checker owns classification. Permit its bounded public vocabulary, + // not another target-controlled text channel disguised as interpretation or sources. + $sources = array( 'https://github.com/php/php-src/security/advisories/GHSA-w8hw-w9fx-j79w', 'https://github.com/php/php-src/security/advisories/GHSA-6wmf-4rv5-p3rp', 'https://ubuntu.com/security/notices/USN-8948-1' ); + if ( $sources !== $core['php2shell_sources'] || ! in_array( $core['php2shell_fixed_version'], array( '', '8.2.35', '8.3.36', '8.4.27', '8.5.12' ), true ) ) { return $bad(); } + $notes = array( + 'Passive metadata only. Runtime SAPI, vendor backports, configuration, and exploitability are not verified.', + 'A PHP-related indicator was observed, but no complete PHP version was disclosed.', + 'No public PHP version was disclosed; lack of a header does not establish that PHP is absent.', + 'Conflicting PHP versions: ' . implode( ', ', $candidates ), + ); + foreach ( $candidates as $candidate ) { + $notes[] = "Cannot safely compare PHP version '" . $candidate . "' with the upstream release table."; + $notes[] = 'PHP ' . $candidate . ' contains a build qualifier. Confirm the exact vendor package; its upstream base alone cannot establish patch status.'; + if ( preg_match( '/\A([0-9]+)\.([0-9]+)\.[0-9]+/', $candidate, $m ) ) { $notes[] = 'PHP ' . (int) $m[1] . '.' . (int) $m[2] . " is outside this rule snapshot's upstream 8.2-8.5 fix table. No safety conclusion is made."; } + $notes[] = 'Advertised PHP ' . $candidate . ' is older than the ' . $core['php2shell_fixed_version'] . ' upstream fix for both listed CVEs.'; + $notes[] = 'Advertised PHP ' . $candidate . ' is at or above the ' . $core['php2shell_fixed_version'] . ' upstream fix. This is not a verified clean bill of health.'; + } + if ( count( $core['php2shell_notes'] ) > 2 ) { return $bad(); } + foreach ( $core['php2shell_notes'] as $note ) { if ( ! in_array( $note, $notes, true ) ) { return $bad(); } } + $recommendations = array( + 'Ask the site administrator to verify the deployed PHP package and applicable vendor security updates.', + 'Confirm that the advertised version is the deployed, patched runtime and follow the current vendor advisory.', + 'Prioritize administrator review. Verify vendor backports and the actual runtime; install the vendor security update or upstream PHP ' . $core['php2shell_fixed_version'] . ' or newer in this branch.', + ); + if ( ! in_array( $core['php2shell_recommendation'], $recommendations, true ) ) { return $bad(); } + $expected = array(); + foreach ( $binding['services'] as $selected ) { + foreach ( array( 'robots_policy', 'public_header_triage', 'security_header_presence', 'tls_certificate' ) as $id ) { $expected[$id . '|' . $selected['scheme'] . ':' . $selected['port']] = true; } + } + if ( count( $expected ) !== count( $result['checks'] ) ) { return $bad(); } + foreach ( $result['checks'] as $check ) { + if ( ! ct_scan_external_check_valid( $check ) ) { return $bad(); } + $key = $check['id'] . '|' . $check['service']; + if ( ! isset( $expected[$key] ) ) { return $bad(); } + unset( $expected[$key] ); + } + $manual = array(); + foreach ( array( 'runtime_and_backport_confirmation', 'internal_configuration_review', 'compromise_investigation' ) as $id ) { $manual[] = array( 'id' => $id, 'status' => 'not_assessed' ); } + if ( $manual !== $result['manual_remainder'] ) { return $bad(); } + $encoded = wp_json_encode( $result ); + return is_string( $encoded ) && strlen( $encoded ) <= 32768 ? $encoded : $bad(); +} + /** Validate the small, single-result checker payload before persisting it. */ function ct_scan_validate_result( $result, $row, $stored_snapshot = false ) { + if ( 'external-web-v1' === ( $row->profile_id ?? null ) ) { return ct_scan_validate_external_result( $result, $row, $stored_snapshot ); } if ( ! is_array( $result ) || count( $result ) > 64 ) { return new WP_Error( 'result_invalid', 'result_invalid' ); } @@ -3037,7 +3505,7 @@ function ct_scan_result( WP_REST_Request $request ) { } $data = ct_scan_decode_request_body( $body ); $keys = array( 'id', 'lease', 'scope_hash', 'outcome', 'result', 'error', 'checker_revision' ); - if ( is_wp_error( $data ) || ! ct_scan_expected_keys( $data, $keys ) || ! ct_scan_callback_payload( $data ) + if ( is_wp_error( $data ) || ( ! ct_scan_expected_keys( $data, $keys ) && ! ct_scan_expected_keys( $data, array_merge( $keys, array( 'profile_id', 'profile_sha256', 'services' ) ) ) ) || ! ct_scan_callback_payload( $data ) || ! in_array( $data['outcome'], array( 'completed', 'failed' ), true ) || ! is_string( $data['checker_revision'] ) || ! preg_match( '/\A[a-f0-9]{40}\z/', $data['checker_revision'] ) ) { return ct_scan_rest_error( 400, 'request_invalid' ); @@ -3072,6 +3540,7 @@ function ct_scan_result( WP_REST_Request $request ) { if ( ! $row ) { return ct_scan_rest_error( 410, 'job_not_found' ); } + if ( ! ct_scan_result_profile_matches( $data, $row ) ) { return ct_scan_rest_error( 422, 'profile_binding_invalid' ); } if ( in_array( $row->status, array( 'completed', 'failed' ), true ) && ! empty( $row->receipt_hash ) ) { return hash_equals( $row->receipt_hash, $receipt_hash ) ? ct_scan_rest_response( 200, array( 'ok' => true ) ) @@ -3627,7 +4096,7 @@ function ct_scan_validate_report_job( $row, $token = '', $require_capability = f $fields, false, (string) $row->authorization_expires_at, - array( 'bound_required' => ! empty( $row->bound_required ), 'profile' => (string) $row->profile_id ), + array( 'bound_required' => ! empty( $row->bound_required ), 'profile' => (string) $row->profile_id, 'profile_sha256' => $row->profile_sha256 ?? null, 'services_json' => $row->services_json ?? null ), true ); if ( is_wp_error( $accepted ) diff --git a/integrations/wordpress/standalone-tests/automatic-evidence-cross.php b/integrations/wordpress/standalone-tests/automatic-evidence-cross.php new file mode 100644 index 0000000..b8eb4d7 --- /dev/null +++ b/integrations/wordpress/standalone-tests/automatic-evidence-cross.php @@ -0,0 +1,59 @@ + hash( 'sha256', $raw ), 'release_snapshot' => $raw, + 'terms_version' => $b['release']['terms_version'], 'terms_html' => $b['terms']['html'], 'terms_sha256' => $b['terms']['sha256'], 'acceptance_statement' => $b['copy']['acceptance_statement'], + 'profile_id' => 'external-web-v1', 'profile_sha256' => $b['profile']['sha256'], 'profile_snapshot' => $b['profile']['manifest_json'], + 'scope_services' => array( array( 'port' => 443, 'scheme' => 'https' ) ), 'scope_services_json' => '[{"port":443,"scheme":"https"}]', + 'target' => 'https://fixture.example.net/', 'legal_name' => "R&D Café O'Connor LLC", 'signer_name' => 'Zoë & Renée', 'signer_role' => "Owner's authorized representative", 'typed_signature' => 'Zoë & Renée', 'email' => 'fixture@example.net', 'referrer' => 'direct', + 'summary_text' => '', 'summary_sha256' => '', 'expires_at_utc' => gmdate( 'Y-m-d\TH:i:s\Z', $now + 3600 ), 'accepted_at_utc' => gmdate( 'Y-m-d\TH:i:s\Z', $now ), + 'idempotency_key' => hash( 'sha256', 'SYNTHETIC cross-source idempotency' ), 'evidence_sha256' => '', 'review_id' => hash( 'sha256', 'SYNTHETIC cross-source review' ), 'review_issued_at_utc' => gmdate( 'Y-m-d\TH:i:s\Z', $now - 10 ), 'review_expires_at_utc' => gmdate( 'Y-m-d\TH:i:s\Z', $now + 590 ), +); +$e['summary_text'] = ct_scan_target_summary( $e['target'], $e['legal_name'], $e['signer_name'], $e['signer_role'], $e['email'], $e['profile_id'], $e['scope_services'], $e['expires_at_utc'] ); +$e['summary_sha256'] = hash( 'sha256', $e['summary_text'] ); $e['evidence_sha256'] = ct_auto_scan_envelope_sha256( $e ); +check( ! is_wp_error( ct_auto_persist_validate_envelope( $e ) ), 'actual G2 validates full Unicode envelope against actual G1 summary/digests' ); +$t = array( 'client_ip' => '198.51.100.7', 'remote_addr' => '198.51.100.7', 'ip_source' => 'remote_addr', 'user_agent' => 'Synthetic offline cross fixture', 'idempotency_key' => $e['idempotency_key'], 'evidence_sha256' => $e['evidence_sha256'], 'accepted_at_utc' => $e['accepted_at_utc'] ); +$context = array( 'schema_version' => 1, 'envelope' => $e, 'technical' => $t, 'terms_url' => 'https://fixture.example.net/agreement/' ); +$body = ct_auto_persist_body( $context ); +$kses = getenv( 'CT_AUTO_CROSS_KSES' ); +if ( $kses ) { require_once $kses; $normalized = wp_kses_normalize_entities( $body ); check( $body === $normalized, 'actual WP entity normalizer preserves escaped Unicode/ampersand body' ); $body = $normalized; } +$GLOBALS['posts'][101] = (object) array( 'ID' => 101, 'post_type' => 'ct_promo_request', 'post_status' => 'private', 'post_name' => 'ct-auto-' . $e['idempotency_key'], 'post_content' => $body ); +check( true === ct_auto_persist_readback( 101, $context, true ), 'actual G2 writes and byte-checks all immutable fields into WP API doubles' ); +$fields = array( 'target' => $e['target'], 'email' => $e['email'], 'name' => $e['signer_name'], 'referrer' => $e['referrer'] ); +$frozen = array( 'bound_required' => true, 'profile' => 'external-web-v1', 'profile_sha256' => $e['profile_sha256'], 'services_json' => $e['scope_services_json'] ); +$expiry = gmdate( 'Y-m-d H:i:s', $now + 3600 ); +$check_request = function () use ( $fields, $frozen, $expiry ) { return ct_scan_validate_saved_request( 101, $fields, false, $expiry, $frozen ); }; +$accepted = $check_request(); +check( ! is_wp_error( $accepted ), 'actual G1 independently validates actual G2 retained envelope, post and metadata' . ( is_wp_error( $accepted ) ? ': ' . $accepted->get_error_code() : '' ) ); +check( $accepted['legal_name'] === $e['legal_name'] && $accepted['signer_name'] === $e['signer_name'] && $accepted['authorization_expires'] === $expiry, 'Unicode legal identity and exact signed expiry survive readback' ); +check( ct_scan_automatic_copy_v1( 101, 'email_queued' ) === $b['copy']['email_queued'], 'actual copy provider reads frozen release while activation remains off' ); +$original = $GLOBALS['meta'][101]; $original_body = $GLOBALS['posts'][101]->post_content; +foreach ( array( '_ct_terms_profile_sha256', '_ct_terms_scope_services', '_ct_terms_authorization_expires_at', '_ct_summary_text', '_ct_terms_typed_signature', '_ct_auto_technical_snapshot', '_ct_auto_envelope' ) as $key ) { + $GLOBALS['meta'][101] = $original; $GLOBALS['meta'][101][$key][0] .= 'tamper'; check( is_wp_error( $check_request() ), 'actual core refuses changed immutable ' . $key ); +} +$GLOBALS['meta'][101] = $original; $GLOBALS['meta'][101]['_ct_terms_signer_name'][] = $e['signer_name']; check( is_wp_error( $check_request() ), 'actual core refuses duplicate signer evidence' ); +$GLOBALS['meta'][101] = $original; $GLOBALS['posts'][101]->post_content .= '

injected

'; check( is_wp_error( $check_request() ), 'actual core refuses changed post representation' ); +$GLOBALS['posts'][101]->post_content = $original_body; +$wrong = $frozen; $wrong['services_json'] = '[{"port":80,"scheme":"http"}]'; check( is_wp_error( ct_scan_validate_saved_request( 101, $fields, false, $expiry, $wrong ) ), 'actual core refuses changed frozen job service' ); +check( false === ct_scan_config()['enabled'] && ! defined( 'CT_AUTO_SCAN_AUTH_ENABLED' ), 'cross-source validation never activates route or queue' ); +echo 'PASS ' . $n . " aggregate external/cross-source checks; synthetic WP stores only, no acceptance/queue/database/mail/network execution\n"; diff --git a/integrations/wordpress/standalone-tests/external-profile.php b/integrations/wordpress/standalone-tests/external-profile.php new file mode 100644 index 0000000..10d4d46 --- /dev/null +++ b/integrations/wordpress/standalone-tests/external-profile.php @@ -0,0 +1,84 @@ +code = $code; } function get_error_code() { return $this->code; } } +function is_wp_error( $v ) { return $v instanceof WP_Error; } +function add_action( ...$args ) {} +function wp_parse_url( $v ) { return parse_url( $v ); } +function wp_json_encode( $v, $flags = 0 ) { return json_encode( $v, $flags ); } +function get_post_meta( $id, $key, $single = true ) { $v = $GLOBALS['meta'][$id][$key] ?? array(); return $single ? ( $v[0] ?? '' ) : $v; } +function sanitize_text_field( $v ) { return trim( strip_tags( $v ) ); } +class ExternalDb { + public $prefix = 'wp_'; public $row; public $fail = false; public $updates = 0; + function update( $table, $data, $where, $format = null, $where_format = null ) { + if ( $this->fail ) { return 0; } + foreach ( $where as $key => $value ) { if ( (string) ( $this->row->$key ?? '' ) !== (string) $value ) { return 0; } } + foreach ( $data as $key => $value ) { $this->row->$key = $value; } + ++$this->updates; return 1; + } +} +require dirname( __DIR__ ) . '/ct-scan.php'; +$n = 0; +function check( $condition, $message ) { global $n; if ( ! $condition ) { fwrite( STDERR, 'FAIL: ' . $message . "\n" ); exit( 1 ); } ++$n; } +function row_fixture() { + return (object) array( 'job_id' => 1, 'status' => 'running', 'lease_hash' => str_repeat( 'a', 64 ), 'target_origin' => 'https://fixture.example.net/', 'profile_id' => 'external-web-v1', 'profile_sha256' => ct_scan_external_profile_digest(), 'services_json' => '[{"port":443,"scheme":"https"}]', 'http_admitted' => 0, 'tls_admitted' => 0, 'reserved_body_bytes' => 0, 'reserved_active_ms' => 0, 'reservation_version' => 1, 'last_admit_ms' => null, 'inflight_until_ms' => null, 'started_at' => '2026-11-14 00:00:00' ); +} +function result_fixture( $row ) { + $r = array( 'input_url' => $row->target_origin, 'final_url' => $row->target_origin, 'status' => 'no_evidence', 'fetch_status' => 'ok', 'http_status' => 200, 'wordpress_version' => '', 'php_version' => '', 'server' => '', 'powered_by' => '', 'title' => '', 'evidence' => array(), 'asset_version_hints' => array(), 'notes' => array(), 'checked_at' => '2026-11-14T00:00:00Z', 'php_version_candidates' => array(), 'php_evidence' => array(), 'php2shell_status' => 'unknown', 'php2shell_version_status' => 'version_not_disclosed', 'php2shell_fixed_version' => '', 'php2shell_exploitability' => 'not_tested', 'php2shell_cves' => array( 'CVE-2026-109618', 'CVE-2026-109619' ), 'php2shell_rules_as_of' => '2026-10-21', 'php2shell_sources' => array( 'https://github.com/php/php-src/security/advisories/GHSA-w8hw-w9fx-j79w', 'https://github.com/php/php-src/security/advisories/GHSA-6wmf-4rv5-p3rp', 'https://ubuntu.com/security/notices/USN-8948-1' ), 'php2shell_notes' => array( 'Passive metadata only. Runtime SAPI, vendor backports, configuration, and exploitability are not verified.', 'No public PHP version was disclosed; lack of a header does not establish that PHP is absent.' ), 'php2shell_recommendation' => 'Ask the site administrator to verify the deployed PHP package and applicable vendor security updates.' ); + $r = array_merge( $r, ct_scan_job_profile_fields( $row ) ); + $r['checks'] = array( + array( 'id' => 'robots_policy', 'service' => 'https:443', 'outcome' => 'observed', 'evidence' => array( 'http_status' => 200, 'allowed' => true ) ), + array( 'id' => 'public_header_triage', 'service' => 'https:443', 'outcome' => 'observed', 'evidence' => array( 'http_status' => 200, 'fetch_status' => 'ok' ) ), + array( 'id' => 'security_header_presence', 'service' => 'https:443', 'outcome' => 'observed', 'evidence' => array( 'http_status' => 200, 'hsts' => true, 'csp' => false, 'frame_control' => true, 'mime_options' => false ) ), + array( 'id' => 'tls_certificate', 'service' => 'https:443', 'outcome' => 'observed', 'evidence' => array( 'verification' => 'verified', 'protocol' => 'TLSv1.3', 'not_after' => '2027-01-01T00:00:00Z' ) ), + ); + $r['manual_remainder'] = array(); foreach ( array( 'runtime_and_backport_confirmation', 'internal_configuration_review', 'compromise_investigation' ) as $id ) { $r['manual_remainder'][] = array( 'id' => $id, 'status' => 'not_assessed' ); } return $r; +} +$row = row_fixture(); $binding = ct_scan_job_profile_fields( $row ); +check( false !== ct_scan_external_profile_manifest(), 'exact immutable manifest loads' ); +check( 'r2-header-v1' === ct_scan_current_profile() && false === ct_scan_config()['enabled'], 'legacy remains disabled default' ); +check( 3 === count( $binding ) && 443 === $binding['services'][0]['port'], 'external claim has exact additive binding' ); +$legacy = clone $row; $legacy->profile_id = 'r2-header-v1'; check( array() === ct_scan_job_profile_fields( $legacy ), 'legacy claim has no new fields' ); +$bad = clone $row; $bad->profile_sha256 = str_repeat( '0', 64 ); check( false === ct_scan_job_profile_fields( $bad ), 'digest tamper refuses claim' ); +check( false === ct_scan_external_services( array( array( 'scheme' => 'http', 'port' => 80 ) ), $row->target_origin ), 'primary service cannot be omitted' ); +$base = array( 'id' => 1, 'lease' => str_repeat( 'a', 64 ), 'scope_hash' => str_repeat( 'b', 64 ), 'outcome' => 'completed', 'result' => array(), 'error' => '', 'checker_revision' => str_repeat( 'c', 40 ) ); +check( ct_scan_result_profile_matches( array_merge( $base, $binding ), $row ), 'callback exact binding accepted' ); +check( ! ct_scan_result_profile_matches( $base, $row ), 'external callback cannot shed binding' ); +check( ct_scan_result_profile_matches( $base, $legacy ), 'legacy callback unchanged' ); +check( ! ct_scan_result_profile_matches( array_merge( $base, $binding ), $legacy ), 'legacy cannot gain external binding' ); +$valid = result_fixture( $row ); check( ! is_wp_error( ct_scan_validate_result( $valid, $row ) ), 'typed external result accepted' ); +foreach ( array( 'title', 'server', 'powered_by', 'wordpress_version' ) as $key ) { $bad = $valid; $bad[$key] = 'RAW PRIVATE'; check( is_wp_error( ct_scan_validate_result( $bad, $row ) ), 'raw text refused ' . $key ); } +foreach ( array( 'evidence', 'notes', 'asset_version_hints', 'php_evidence' ) as $key ) { $bad = $valid; $bad[$key] = array( 'RAW PRIVATE' ); check( is_wp_error( ct_scan_validate_result( $bad, $row ) ), 'raw list refused ' . $key ); } +foreach ( array( + 'missing check' => function ( &$r ) { array_pop( $r['checks'] ); }, + 'duplicate check' => function ( &$r ) { $r['checks'][1] = $r['checks'][0]; }, + 'unselected service' => function ( &$r ) { $r['checks'][0]['service'] = 'http:80'; }, + 'raw interpretation' => function ( &$r ) { $r['php2shell_notes'] = array( 'RAW PRIVATE' ); }, + 'raw recommendation' => function ( &$r ) { $r['php2shell_recommendation'] = 'RAW PRIVATE'; }, + 'raw source' => function ( &$r ) { $r['php2shell_sources'][] = 'https://secret.invalid/'; }, + 'unknown evidence' => function ( &$r ) { $r['checks'][0]['evidence']['secret'] = 'raw'; }, + 'boolean status' => function ( &$r ) { $r['checks'][0]['evidence']['http_status'] = true; }, + 'unknown TLS' => function ( &$r ) { $r['checks'][3]['evidence']['protocol'] = 'raw'; }, + 'invalid TLS date' => function ( &$r ) { $r['checks'][3]['evidence']['not_after'] = '2027-02-31T00:00:00Z'; }, + 'manual assessment' => function ( &$r ) { $r['manual_remainder'][0]['status'] = 'safe'; }, + 'extra schema' => function ( &$r ) { $r['secret'] = true; }, + 'unaccepted redirect' => function ( &$r ) { $r['final_url'] = 'http://fixture.example.net/'; }, +) as $label => $change ) { $bad = $valid; $change( $bad ); check( is_wp_error( ct_scan_validate_result( $bad, $row ) ), $label . ' refused' ); } +$core = $valid; foreach ( array( 'profile_id', 'profile_sha256', 'services', 'checks', 'manual_remainder' ) as $key ) { unset( $core[$key] ); } +check( ! is_wp_error( ct_scan_validate_result( $core, $legacy ) ), 'legacy result schema unchanged' ); +check( is_wp_error( ct_scan_validate_result( $core, $row ) ), 'external result cannot use legacy schema' ); +$now = strtotime( '2026-11-14 00:00:00 UTC' ) * 1000; +$GLOBALS['wpdb'] = new ExternalDb(); $GLOBALS['wpdb']->row = clone $row; +$decision = ct_scan_external_admit_locked( clone $GLOBALS['wpdb']->row, array( 'kind' => 'http' ), $now ); +check( 200 === $decision[0] && 262144 === $GLOBALS['wpdb']->row->reserved_body_bytes && 10000 === $GLOBALS['wpdb']->row->reserved_active_ms, 'HTTP reservation durably charged before grant' ); +check( 11 === count( $decision[1] ) && 99 === $decision[1]['http_remaining'], 'exact grant schema' ); +check( 429 === ct_scan_external_admit_locked( clone $GLOBALS['wpdb']->row, array( 'kind' => 'tls' ), $now + 1000 )[0], 'inflight connection cannot overlap' ); +check( 200 === ct_scan_external_admit_locked( clone $GLOBALS['wpdb']->row, array( 'kind' => 'tls' ), $now + 11000 )[0], 'separate TLS reservation accepted after prior timeout' ); +check( 1 === $GLOBALS['wpdb']->row->tls_admitted && 262144 === $GLOBALS['wpdb']->row->reserved_body_bytes, 'TLS never refunds or consumes body' ); +foreach ( array( true, -1, '00', '1e2', null, 101 ) as $value ) { $bad = clone $row; $bad->http_admitted = $value; check( 410 === ct_scan_external_admit_locked( $bad, array( 'kind' => 'http' ), $now )[0], 'malformed counter refuses' ); } +$bad = clone $row; $bad->http_admitted = 20; $bad->reserved_body_bytes = 5242880; $bad->reserved_active_ms = 200000; $bad->last_admit_ms = $now; $bad->inflight_until_ms = $now + 11000; +check( 409 === ct_scan_external_admit_locked( $bad, array( 'kind' => 'http' ), $now + 11000 )[0], 'aggregate body ceiling cannot be reset' ); +$GLOBALS['wpdb']->fail = true; +check( 503 === ct_scan_external_admit_locked( clone $row, array( 'kind' => 'http' ), $now )[0], 'failed CAS grants nothing' ); +echo 'PASS ' . $n . " external-profile source checks; no WordPress/database/network/mail execution\n"; diff --git a/integrations/wordpress/standalone-tests/manual-lifecycle.php b/integrations/wordpress/standalone-tests/manual-lifecycle.php index 912ef2b..143cc3c 100644 --- a/integrations/wordpress/standalone-tests/manual-lifecycle.php +++ b/integrations/wordpress/standalone-tests/manual-lifecycle.php @@ -6,7 +6,7 @@ class WP_Error { public $code; function __construct( $code, $message = '' ) { $this->code = $code; } } function add_action( ...$args ) {} function add_filter( ...$args ) {} -function get_option( $key ) { return $GLOBALS['schema'] ?? '8'; } +function get_option( $key ) { return $GLOBALS['schema'] ?? CT_SCAN_SCHEMA_VERSION; } function wp_cache_delete( $id, $group ) { ++$GLOBALS['cache_deletes']; unset( $GLOBALS['cached_markers'] ); } function get_post_meta( $id, $key, $single = true ) { if ( array_key_exists( 'cached_markers', $GLOBALS ) ) { return $GLOBALS['cached_markers']; } @@ -42,7 +42,7 @@ function check_manual( $ok, $message ) { global $count; ++$count; if ( ! $ok ) { } $GLOBALS['schema'] = '7'; check_manual( false === ct_scan_manual_closure_ready( 805 ), 'Manual close accepted stale schema.' ); -$GLOBALS['schema'] = '8'; +$GLOBALS['schema'] = CT_SCAN_SCHEMA_VERSION; $db->row = (object) array( 'job_id' => 1 ); check_manual( false === ct_scan_manual_closure_ready( 805 ), 'Manual close accepted an existing queued request.' ); $db->row = null; $db->query_error = true; diff --git a/integrations/wordpress/tests/external-reservations.php b/integrations/wordpress/tests/external-reservations.php new file mode 100644 index 0000000..d08b45f --- /dev/null +++ b/integrations/wordpress/tests/external-reservations.php @@ -0,0 +1,40 @@ +get_col( 'SHOW COLUMNS FROM ' . ct_scan_jobs_table(), 0 ); +foreach ( array( 'profile_sha256', 'services_json', 'tls_admitted', 'reserved_body_bytes', 'reserved_active_ms', 'inflight_until_ms', 'reservation_version' ) as $column ) { + ct_test_assert( in_array( $column, $columns, true ), 'Missing external reservation column ' . $column ); +} +ct_test_assert( '9' === CT_SCAN_SCHEMA_VERSION && CT_SCAN_SCHEMA_VERSION === get_option( 'ct_scan_schema_version' ), 'Schema9 upgrade incomplete.' ); +ct_test_assert( false !== ct_scan_external_profile_manifest(), 'Reviewed immutable external manifest unavailable.' ); +$fixture = ct_test_running_job( 'external-reservation.example.net' ); +$id = $fixture['job_id']; +try { + ct_test_assert( ct_scan_lock_acquire(), 'Cannot own reservation fixture queue lock.' ); + try { + $now = (int) floor( microtime( true ) * 1000 ); + $services = wp_json_encode( ct_scan_canonicalize( array( array( 'port' => 443, 'scheme' => 'https' ) ) ), JSON_UNESCAPED_SLASHES ); + ct_test_assert( 1 === $wpdb->update( ct_scan_jobs_table(), array( 'profile_id' => 'external-web-v1', 'profile_sha256' => ct_scan_external_profile_digest(), 'services_json' => $services, 'reservation_version' => 1, 'started_at' => gmdate( 'Y-m-d H:i:s', (int) floor( $now / 1000 ) ) ), array( 'job_id' => $id ) ), 'Cannot prepare owned synthetic profile row.' ); + $before = ct_scan_row( $id ); + $grant = ct_scan_external_admit_locked( $before, array( 'kind' => 'http' ), $now ); + $after = ct_scan_row( $id ); + ct_test_assert( 200 === $grant[0] && 1 === (int) $after->http_admitted && 262144 === (int) $after->reserved_body_bytes && 10000 === (int) $after->reserved_active_ms, 'Database did not commit reservations before grant.' ); + ct_test_assert( 503 === ct_scan_external_admit_locked( $before, array( 'kind' => 'http' ), $now )[0] && 1 === (int) ct_scan_row( $id )->http_admitted, 'Stale concurrent CAS granted or recharged a reservation.' ); + ct_test_assert( 429 === ct_scan_external_admit_locked( $after, array( 'kind' => 'tls' ), $now + 1000 )[0], 'An inflight HTTP reservation overlapped TLS.' ); + $tls = ct_scan_external_admit_locked( $after, array( 'kind' => 'tls' ), $now + 11000 ); + $after = ct_scan_row( $id ); + ct_test_assert( 200 === $tls[0] && 1 === (int) $after->tls_admitted && 262144 === (int) $after->reserved_body_bytes && 20000 === (int) $after->reserved_active_ms, 'TLS reservation did not preserve cumulative database body charge.' ); + $bad = clone $after; $bad->reserved_body_bytes = 0; + ct_test_assert( 410 === ct_scan_external_admit_locked( $bad, array( 'kind' => 'http' ), $now + 22000 )[0], 'Inconsistent restored reservation state received new budget.' ); + $bad = clone $after; $bad->reservation_version = 0; + ct_test_assert( 410 === ct_scan_external_admit_locked( $bad, array( 'kind' => 'http' ), $now + 22000 )[0], 'Legacy row was silently promoted to external reservation protocol.' ); + } finally { ct_scan_lock_release(); } +} finally { ct_test_purge_fixtures(); } +echo "External reservation database fixture passed; no external authorization or target traffic.\n"; From 4920fd5edfdf805ab66d6cf0b57e0d9adda9fb05 Mon Sep 17 00:00:00 2001 From: Timothy Wayne Gregg Date: Sat, 14 Nov 2026 02:10:44 -0500 Subject: [PATCH 14/19] fix(wordpress): refuse non-string execution profiles before lookup --- integrations/wordpress/ct-scan.php | 1 + integrations/wordpress/standalone-tests/external-profile.php | 1 + 2 files changed, 2 insertions(+) diff --git a/integrations/wordpress/ct-scan.php b/integrations/wordpress/ct-scan.php index 1ae4d01..e1eb32a 100644 --- a/integrations/wordpress/ct-scan.php +++ b/integrations/wordpress/ct-scan.php @@ -220,6 +220,7 @@ function ct_scan_control_limits() { } function ct_scan_profile_limits( $profile_id ) { + if ( ! is_string( $profile_id ) ) { return false; } if ( 'external-web-v1' === $profile_id ) { $manifest = ct_scan_external_profile_manifest(); if ( ! $manifest || ! isset( $manifest['limits'] ) || ! is_array( $manifest['limits'] ) ) { return false; } diff --git a/integrations/wordpress/standalone-tests/external-profile.php b/integrations/wordpress/standalone-tests/external-profile.php index 10d4d46..c1458f6 100644 --- a/integrations/wordpress/standalone-tests/external-profile.php +++ b/integrations/wordpress/standalone-tests/external-profile.php @@ -35,6 +35,7 @@ function result_fixture( $row ) { ); $r['manual_remainder'] = array(); foreach ( array( 'runtime_and_backport_confirmation', 'internal_configuration_review', 'compromise_investigation' ) as $id ) { $r['manual_remainder'][] = array( 'id' => $id, 'status' => 'not_assessed' ); } return $r; } +foreach ( array( null, true, array( 'external-web-v1' ), 1 ) as $invalid ) { check( false === ct_scan_profile_limits( $invalid ), 'non-string profile cannot become a configuration permission' ); } $row = row_fixture(); $binding = ct_scan_job_profile_fields( $row ); check( false !== ct_scan_external_profile_manifest(), 'exact immutable manifest loads' ); check( 'r2-header-v1' === ct_scan_current_profile() && false === ct_scan_config()['enabled'], 'legacy remains disabled default' ); From 48ae54bf8e2097fbafa865ddc60ae30cf8e88695 Mon Sep 17 00:00:00 2001 From: Timothy Wayne Gregg Date: Sat, 14 Nov 2026 02:12:45 -0500 Subject: [PATCH 15/19] test(worker): scope scanner entry regressions to Linux --- test_external_pipeline.py | 2 ++ test_external_worker_wire.py | 2 ++ 2 files changed, 4 insertions(+) diff --git a/test_external_pipeline.py b/test_external_pipeline.py index f8cd0df..0b08c85 100644 --- a/test_external_pipeline.py +++ b/test_external_pipeline.py @@ -10,6 +10,7 @@ import json import select import socket +import sys import threading import time from types import SimpleNamespace @@ -27,6 +28,7 @@ HOME = b'' +@unittest.skipUnless(sys.platform.startswith("linux"), "scanner entry requires Linux isolation") class ExternalPipelineTests(unittest.TestCase): def run_scanner(self, *, home=HOME): expiry = datetime.fromtimestamp(time.time() + 300, timezone.utc) diff --git a/test_external_worker_wire.py b/test_external_worker_wire.py index 75902b9..5b97669 100644 --- a/test_external_worker_wire.py +++ b/test_external_worker_wire.py @@ -5,6 +5,7 @@ import hashlib import io import json +import sys from pathlib import Path import tempfile import unittest @@ -199,6 +200,7 @@ def test_service_failure_also_carries_profile_and_retry_never_rescans(self): self.assertEqual(payload["error"], "admission_refused") self.assertEqual(payload["profile_sha256"], PROFILE_SHA256) + @unittest.skipUnless(sys.platform.startswith("linux"), "scanner entry requires Linux isolation") def test_scanner_dispatch_shares_one_meter_transport_and_admission_channel(self): from integrations.worker import scanner, network, external selected = replace(claim(), services=(("http", 80), ("https", 443))) From dc9a2d25bed608ce102ad0b1b56040f7241dca8c Mon Sep 17 00:00:00 2001 From: Timothy Wayne Gregg Date: Sat, 14 Nov 2026 02:13:26 -0500 Subject: [PATCH 16/19] fix(deployment): package pinned external profile and exercise fixture gates --- .dockerignore | 2 ++ .github/workflows/tests.yml | 6 ++++-- deployment/Dockerfile | 1 + deployment/compose.test.yaml | 3 +++ deployment/test-runtime-image.py | 8 ++++++++ 5 files changed, 18 insertions(+), 2 deletions(-) diff --git a/.dockerignore b/.dockerignore index d2f5708..77e4a0b 100644 --- a/.dockerignore +++ b/.dockerignore @@ -6,5 +6,7 @@ !integrations/__init__.py !integrations/worker/ !integrations/worker/*.py +!integrations/profiles/ +!integrations/profiles/external-web-v1.json !deployment/ !deployment/Dockerfile diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 220f3fc..98c7690 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -44,7 +44,7 @@ jobs: run: >- docker run --rm --network none -v "$PWD:/work:ro" -w /work php:8.3-cli@sha256:aafe21201943a8a6e497ddfb471e2ce68ada76adede38c7d61fede8918b24319 - sh -c 'set -e; php -d zend.assertions=1 integrations/wordpress/standalone-tests/report-access.php; php -d zend.assertions=1 integrations/wordpress/standalone-tests/retention.php integrations/wordpress/ct-scan.php; php -d zend.assertions=1 integrations/wordpress/standalone-tests/tombstones.php integrations/wordpress/ct-scan.php; php -d zend.assertions=1 integrations/wordpress/standalone-tests/manual-lifecycle.php' + sh -c 'set -e; php -d zend.assertions=1 integrations/wordpress/standalone-tests/report-access.php; php -d zend.assertions=1 integrations/wordpress/standalone-tests/retention.php integrations/wordpress/ct-scan.php; php -d zend.assertions=1 integrations/wordpress/standalone-tests/tombstones.php integrations/wordpress/ct-scan.php; php -d zend.assertions=1 integrations/wordpress/standalone-tests/manual-lifecycle.php; php -d zend.assertions=1 integrations/wordpress/standalone-tests/external-profile.php' wordpress: name: WordPress queue and authorization @@ -63,6 +63,8 @@ jobs: run: docker compose -p php2shell-ci -f deployment/compose.test.yaml exec -T wordpress php /work/integrations/wordpress/tests/integration.php - name: Test per-request admission, cancellation race and fail-closed behavior run: docker compose -p php2shell-ci -f deployment/compose.test.yaml exec -T wordpress php /work/integrations/wordpress/tests/admission.php + - name: Test external profile reservation database compare-and-set + run: docker compose -p php2shell-ci -f deployment/compose.test.yaml exec -T wordpress php /work/integrations/wordpress/tests/external-reservations.php - name: Test bound acceptance evidence and the accepted copy run: docker compose -p php2shell-ci -f deployment/compose.test.yaml exec -T wordpress php /work/integrations/wordpress/tests/evidence.php - name: Test Python to WordPress over HTTP with synthetic scan responses @@ -87,7 +89,7 @@ jobs: --tmpfs /tmp:rw,noexec,nosuid,size=32m,mode=1777 -v "$PWD:/app:ro" -w /app -e PYTHONDONTWRITEBYTECODE=1 python:3.12-slim-bookworm@sha256:6a0e06f5ba0b373d051aabd3fffb9b873f27f9a2a3a14bcbe37dee6f0e3bbc70 - python -m unittest -v test_scan_worker test_admission test_control + python -m unittest -v test_scan_worker test_admission test_control test_external_boundary test_external_pipeline test_external_worker_wire - name: Build and check packaged isolation run: | docker build --build-arg CHECKER_REVISION="$GITHUB_SHA" -f deployment/Dockerfile -t ct-worker-fixture . diff --git a/deployment/Dockerfile b/deployment/Dockerfile index 3ce838c..b3444d6 100644 --- a/deployment/Dockerfile +++ b/deployment/Dockerfile @@ -3,6 +3,7 @@ WORKDIR /app COPY wordpress_check.py LICENSE LICENSING.md /app/ COPY integrations/__init__.py /app/integrations/__init__.py COPY integrations/worker /app/integrations/worker +COPY integrations/profiles/external-web-v1.json /app/integrations/profiles/external-web-v1.json ARG CHECKER_REVISION ENV PYTHONDONTWRITEBYTECODE=1 PYTHONUNBUFFERED=1 CT_SCAN_REVISION=${CHECKER_REVISION} RUN test -n "$CHECKER_REVISION" && mkdir -p /var/lib/ct-scan /run/ct-scan && chmod 700 /var/lib/ct-scan /run/ct-scan diff --git a/deployment/compose.test.yaml b/deployment/compose.test.yaml index 6feae5a..26da158 100644 --- a/deployment/compose.test.yaml +++ b/deployment/compose.test.yaml @@ -33,6 +33,9 @@ services: volumes: - wp:/var/www/html - ../integrations/wordpress:/var/www/html/wp-content/mu-plugins:ro + # The plugin's canonical sibling fallback avoids a nested mount inside + # the read-only MU-plugin directory. + - ../integrations/profiles:/var/www/html/wp-content/profiles:ro - ..:/work:ro networks: [fixture] volumes: diff --git a/deployment/test-runtime-image.py b/deployment/test-runtime-image.py index 0714567..8cf27fd 100644 --- a/deployment/test-runtime-image.py +++ b/deployment/test-runtime-image.py @@ -8,18 +8,25 @@ from integrations.worker.admission import AdmissionDenied from integrations.worker.runner import SubprocessRunner, _linux_preexec_limits from integrations.worker.service import load_secret +from integrations.worker.profiles import MANIFEST_PATH, PROFILE_ID, PROFILE_SHA256, profile_manifest, validate_binding assert os.geteuid() == 0 +assert MANIFEST_PATH == Path('/app/integrations/profiles/external-web-v1.json') +manifest = profile_manifest() +assert manifest['profile_id'] == PROFILE_ID +assert validate_binding(PROFILE_ID, PROFILE_SHA256, manifest['allowed_services']) == (('http', 80), ('https', 443)) with tempfile.TemporaryDirectory() as directory: secret = Path(directory) / 'key' secret.write_text('synthetic-fixture-key-' * 3) secret.chmod(0o600) assert load_secret(secret) == b'synthetic-fixture-key-' * 3 code = '''import os,sys +from integrations.worker.profiles import PROFILE_ID, PROFILE_SHA256, profile_manifest, validate_binding assert os.getuid() == os.getgid() == 65534 assert not os.getgroups() assert 'CT_SCAN_SECRET_FILE' not in os.environ +assert validate_binding(PROFILE_ID, PROFILE_SHA256, profile_manifest()['allowed_services']) == (('http', 80), ('https', 443)) try: open(sys.argv[1]).read() except PermissionError: @@ -50,3 +57,4 @@ def probe(self): assert result['php2shell_status'] == 'unknown' assert result['php2shell_exploitability'] == 'not_tested' print('Packaged checker ran without network access and retained unknown outcome.') +print('Packaged immutable profile loaded and validated for controller and scanner identities; no external profile scan ran.') From 3d6fc1a50e75ec82f72064179b591fc0133dbe17 Mon Sep 17 00:00:00 2001 From: Timothy Wayne Gregg Date: Sat, 14 Nov 2026 02:15:19 -0500 Subject: [PATCH 17/19] test(wordpress): bridge actual external executor callbacks to PHP validation Refs #27. Use injected transports and DNS/TCP tripwires; cover selected services, uncertainty, bounded failures and tampering without network or database execution. --- .../external-profile-bootstrap.php | 21 +++ .../standalone-tests/external-profile.php | 22 +-- .../standalone-tests/external-result-wire.php | 23 +++ test_external_php_bridge.py | 134 ++++++++++++++++++ 4 files changed, 180 insertions(+), 20 deletions(-) create mode 100644 integrations/wordpress/standalone-tests/external-profile-bootstrap.php create mode 100644 integrations/wordpress/standalone-tests/external-result-wire.php create mode 100644 test_external_php_bridge.py diff --git a/integrations/wordpress/standalone-tests/external-profile-bootstrap.php b/integrations/wordpress/standalone-tests/external-profile-bootstrap.php new file mode 100644 index 0000000..245bf84 --- /dev/null +++ b/integrations/wordpress/standalone-tests/external-profile-bootstrap.php @@ -0,0 +1,21 @@ +code = $code; } function get_error_code() { return $this->code; } } +function is_wp_error( $v ) { return $v instanceof WP_Error; } +function add_action( ...$args ) {} +function wp_parse_url( $v ) { return parse_url( $v ); } +function wp_json_encode( $v, $flags = 0 ) { return json_encode( $v, $flags ); } +function get_post_meta( $id, $key, $single = true ) { $v = $GLOBALS['meta'][$id][$key] ?? array(); return $single ? ( $v[0] ?? '' ) : $v; } +function sanitize_text_field( $v ) { return trim( strip_tags( $v ) ); } +class ExternalDb { + public $prefix = 'wp_'; public $row; public $fail = false; public $updates = 0; + function update( $table, $data, $where, $format = null, $where_format = null ) { + if ( $this->fail ) { return 0; } + foreach ( $where as $key => $value ) { if ( (string) ( $this->row->$key ?? '' ) !== (string) $value ) { return 0; } } + foreach ( $data as $key => $value ) { $this->row->$key = $value; } + ++$this->updates; return 1; + } +} +require dirname( __DIR__ ) . '/ct-scan.php'; diff --git a/integrations/wordpress/standalone-tests/external-profile.php b/integrations/wordpress/standalone-tests/external-profile.php index c1458f6..fa6f919 100644 --- a/integrations/wordpress/standalone-tests/external-profile.php +++ b/integrations/wordpress/standalone-tests/external-profile.php @@ -1,24 +1,6 @@ code = $code; } function get_error_code() { return $this->code; } } -function is_wp_error( $v ) { return $v instanceof WP_Error; } -function add_action( ...$args ) {} -function wp_parse_url( $v ) { return parse_url( $v ); } -function wp_json_encode( $v, $flags = 0 ) { return json_encode( $v, $flags ); } -function get_post_meta( $id, $key, $single = true ) { $v = $GLOBALS['meta'][$id][$key] ?? array(); return $single ? ( $v[0] ?? '' ) : $v; } -function sanitize_text_field( $v ) { return trim( strip_tags( $v ) ); } -class ExternalDb { - public $prefix = 'wp_'; public $row; public $fail = false; public $updates = 0; - function update( $table, $data, $where, $format = null, $where_format = null ) { - if ( $this->fail ) { return 0; } - foreach ( $where as $key => $value ) { if ( (string) ( $this->row->$key ?? '' ) !== (string) $value ) { return 0; } } - foreach ( $data as $key => $value ) { $this->row->$key = $value; } - ++$this->updates; return 1; - } -} -require dirname( __DIR__ ) . '/ct-scan.php'; +// Pure synthetic source tests; shared bootstrap never performs external operations. +require __DIR__ . '/external-profile-bootstrap.php'; $n = 0; function check( $condition, $message ) { global $n; if ( ! $condition ) { fwrite( STDERR, 'FAIL: ' . $message . "\n" ); exit( 1 ); } ++$n; } function row_fixture() { diff --git a/integrations/wordpress/standalone-tests/external-result-wire.php b/integrations/wordpress/standalone-tests/external-result-wire.php new file mode 100644 index 0000000..1e9287e --- /dev/null +++ b/integrations/wordpress/standalone-tests/external-result-wire.php @@ -0,0 +1,23 @@ + $v['target'], 'profile_id' => $v['binding']['profile_id'], 'profile_sha256' => $v['binding']['profile_sha256'], 'services_json' => wp_json_encode( ct_scan_canonicalize( $v['binding']['services'] ), JSON_UNESCAPED_SLASHES ) ); + $callback = $v['callback']; $error = 'profile_binding_invalid'; + if ( ct_scan_callback_payload( $callback ) && ct_scan_result_profile_matches( $callback, $row ) ) { + $error = 'callback_semantics_invalid'; + if ( 'completed' === $callback['outcome'] && '' === $callback['error'] ) { + $result = ct_scan_validate_result( $callback['result'], $row ); + $accepted = ! is_wp_error( $result ); $error = $accepted ? '' : $result->get_error_code(); + } elseif ( 'failed' === $callback['outcome'] && null === $callback['result'] && is_string( $callback['error'] ) && preg_match( '/\A[a-z][a-z0-9_]{0,63}\z/', $callback['error'] ) ) { + $accepted = true; $error = ''; + } + } +} +echo json_encode( array( 'accepted' => $accepted, 'error' => $error ) ) . "\n"; diff --git a/test_external_php_bridge.py b/test_external_php_bridge.py new file mode 100644 index 0000000..76b69a9 --- /dev/null +++ b/test_external_php_bridge.py @@ -0,0 +1,134 @@ +"""Actual executor -> runner -> callback -> PHP validator, injected transports only. + +Run with CT_EXTERNAL_BRIDGE_PHP=/trusted/php python -m unittest -v + test_external_php_bridge. PHP absence is an explicit optional-platform skip; +CI can make it mandatory by setting the executable path. +""" +from copy import deepcopy +from dataclasses import replace +import json +import os +from pathlib import Path +import shutil +import socket +import subprocess +import unittest +from unittest.mock import patch + +from integrations.worker.external import ProfileStopped +from integrations.worker.network import RequestOrigin +from integrations.worker.profiles import PROFILE_ID, PROFILE_SHA256 +from integrations.worker.protocol import profile_fields +from integrations.worker.runner import ScannerFailure, SubprocessRunner +from integrations.worker.service import WorkerService +import test_external_executor as fixtures +from test_external_executor import Transport, Response, success_responses +from test_scan_worker import _claim + +PHP = os.environ.get("CT_EXTERNAL_BRIDGE_PHP") or shutil.which("php") +BRIDGE = Path(__file__).parent / "integrations/wordpress/standalone-tests/external-result-wire.php" +SERVICES = [{"scheme": "http", "port": 80}, {"scheme": "https", "port": 443}] +TARGET = "https://fixture.example.net/" +REVISION = "1" * 40 + + +@unittest.skipUnless(PHP, "PHP is unavailable; set CT_EXTERNAL_BRIDGE_PHP for required bridge checks") +class ExternalPhpBridgeTests(unittest.TestCase): + def setUp(self): + # Every transport is injected. A future regression must never fall back + # to real DNS, TCP or TLS while generating this fixture. + self.no_network = patch.object(socket, "create_connection", side_effect=AssertionError("fixture attempted TCP")) + self.no_dns = patch.object(socket, "getaddrinfo", side_effect=AssertionError("fixture attempted DNS")) + self.no_network.start(); self.no_dns.start() + self.addCleanup(self.no_network.stop); self.addCleanup(self.no_dns.stop) + self.service = WorkerService(None, None, None, REVISION) + + def claim(self, target=TARGET, services=None): + return replace(_claim(target=target), lease="a" * 64, profile_id=PROFILE_ID, profile_sha256=PROFILE_SHA256, + services=tuple((v["scheme"], v["port"]) for v in (services or [SERVICES[1]]))) + + def execute(self, responses, *, claim=None, tls=None): + claim = claim or self.claim() + fixture = fixtures.ExecutorTests() + transport = Transport(responses) + result = fixture.run_profile(transport, target=claim.target, services=profile_fields(claim)["services"], tls=tls) + scope = RequestOrigin.from_target(claim.target)[0] + self.assertIs(SubprocessRunner.validate_result(result, claim.target, scope, claim=claim), result) + self.assertFalse(transport.responses, "fixture left unused responses") + return result + + def callback(self, claim, result=None, *, error=""): + return json.loads(self.service._callback_body(claim.job_id, claim.lease, claim.scope_hash, + "failed" if error else "completed", result, error, claim=claim)) + + def php(self, claim, callback): + wire = {"target": claim.target, "binding": profile_fields(claim), "callback": callback} + proc = subprocess.run([PHP, str(BRIDGE)], input=json.dumps(wire, ensure_ascii=False), + capture_output=True, text=True, timeout=10, check=True) + self.assertEqual(proc.stderr, "", "PHP emitted a warning or private fixture data") + return json.loads(proc.stdout) + + def accepted(self, claim, result): + callback = self.callback(claim, result) + text = json.dumps(callback) + for private in ("PRIVATE-TITLE-SENTINEL", "RAW-SERVER-SENTINEL", "PRIVATE-KEY-SENTINEL"): + self.assertNotIn(private, text) + self.assertEqual(self.php(claim, callback), {"accepted": True, "error": ""}) + + def test_selected_service_variants(self): + for services, target in (([SERVICES[0]], "http://fixture.example.net/"), + ([SERVICES[1]], TARGET), + (SERVICES, TARGET), + (SERVICES, "http://fixture.example.net/")): + with self.subTest(services=services, target=target): + claim = self.claim(target, services) + result = self.execute(success_responses() * len(services), claim=claim) + self.accepted(claim, result) + + def test_reported_version_and_safe_unknown_variants(self): + for version in ("8.3.35", "8.3.36", "8.4.27", "7.4.33", "8.3.36-vendor1", "8.3.36-rc1", "8.3"): + with self.subTest(version=version): + claim = self.claim(); self.accepted(claim, self.execute(success_responses(version), claim=claim)) + claim = self.claim() + self.accepted(claim, self.execute([Response(body=b"User-agent: *\nAllow: /\n"), + Response(body=b"Public page without PHP metadata"), Response(no_read=True)], claim=claim)) + self.accepted(claim, self.execute([Response(body=b"User-agent: *\nDisallow: /\n")], claim=claim)) + both = self.claim(services=SERVICES) + self.accepted(both, self.execute(success_responses("8.3.35") + success_responses("8.3.36"), claim=both)) + self.accepted(both, self.execute(success_responses() + [Response(status=404), Response(status=404)], claim=both)) + def tls_failure(_service): raise OSError("SYNTHETIC PRIVATE TLS DETAIL") + self.accepted(claim, self.execute(success_responses(), claim=claim, tls=tls_failure)) + + def test_possible_indicator_stops_emit_only_safe_bound_failure(self): + for body, reason in ((b"-----BEGIN PRIVATE KEY-----\nPRIVATE-KEY-SENTINEL", "possible_sensitive_indicator"), + (b"This page says Hacked by PRIVATE-COMPROMISE-SENTINEL", "possible_compromise_indicator")): + with self.subTest(reason=reason): + claim = self.claim() + with self.assertRaises(ProfileStopped) as caught: + self.execute([Response(body=b"User-agent: *\nAllow: /\n"), Response(body=body)], claim=claim) + self.assertEqual(str(caught.exception), reason) + callback = self.callback(claim, error=reason) + self.assertIsNone(callback["result"]) + self.assertNotIn("SENTINEL", json.dumps(callback)) + self.assertEqual(self.php(claim, callback), {"accepted": True, "error": ""}) + + def test_actual_executor_result_tampering_refused_by_both_boundaries(self): + claim = self.claim(); result = self.execute(success_responses(), claim=claim) + for edit in (lambda r: r.update(title="PRIVATE-TITLE-SENTINEL"), + lambda r: r.update(profile_sha256="0" * 64), + lambda r: r["checks"][0]["evidence"].update(raw="PRIVATE"), + lambda r: r["checks"][0].update(service="http:80"), + lambda r: r["manual_remainder"].pop()): + bad = deepcopy(result); edit(bad) + with self.assertRaises(ScannerFailure): + SubprocessRunner.validate_result(bad, claim.target, RequestOrigin.from_target(claim.target)[0], claim=claim) + self.assertFalse(self.php(claim, self.callback(claim, bad))["accepted"]) + callback = self.callback(claim, result) + callback.pop("profile_sha256") + self.assertFalse(self.php(claim, callback)["accepted"]) + callback = self.callback(claim, result); callback["services"] = [SERVICES[0]] + self.assertFalse(self.php(claim, callback)["accepted"]) + + +if __name__ == "__main__": + unittest.main() From 74778bece454bcc3f20a7c1892f261e9438cda6d Mon Sep 17 00:00:00 2001 From: Timothy Wayne Gregg Date: Sat, 14 Nov 2026 02:15:40 -0500 Subject: [PATCH 18/19] ci: require the offline external callback PHP bridge --- .github/workflows/tests.yml | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 98c7690..bdb0b46 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -45,6 +45,19 @@ jobs: docker run --rm --network none -v "$PWD:/work:ro" -w /work php:8.3-cli@sha256:aafe21201943a8a6e497ddfb471e2ce68ada76adede38c7d61fede8918b24319 sh -c 'set -e; php -d zend.assertions=1 integrations/wordpress/standalone-tests/report-access.php; php -d zend.assertions=1 integrations/wordpress/standalone-tests/retention.php integrations/wordpress/ct-scan.php; php -d zend.assertions=1 integrations/wordpress/standalone-tests/tombstones.php integrations/wordpress/ct-scan.php; php -d zend.assertions=1 integrations/wordpress/standalone-tests/manual-lifecycle.php; php -d zend.assertions=1 integrations/wordpress/standalone-tests/external-profile.php' + - name: Set up Python for the cross-language profile fixture + uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6 + with: + python-version: '3.12' + - name: Check actual executor callbacks against PHP without network + run: | + bridge_php="$RUNNER_TEMP/ct-external-bridge-php" + cat > "$bridge_php" <<'SH' + #!/bin/sh + exec docker run --rm -i --network none --read-only --cap-drop ALL --security-opt no-new-privileges --pids-limit 16 --memory 128m -v "$PWD:$PWD:ro" -w "$PWD" php:8.3-cli@sha256:aafe21201943a8a6e497ddfb471e2ce68ada76adede38c7d61fede8918b24319 php "$@" + SH + chmod 700 "$bridge_php" + CT_EXTERNAL_BRIDGE_PHP="$bridge_php" python -m unittest -v test_external_php_bridge wordpress: name: WordPress queue and authorization From a1e4f7c9ab31d07bafc8fb37dca47b260e351cab Mon Sep 17 00:00:00 2001 From: Timothy Wayne Gregg Date: Sat, 14 Nov 2026 02:20:40 -0500 Subject: [PATCH 19/19] test(worker): scope Linux isolation and deadline acceptance checks --- test_external_boundary.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/test_external_boundary.py b/test_external_boundary.py index bbd7dc6..19e9679 100644 --- a/test_external_boundary.py +++ b/test_external_boundary.py @@ -8,6 +8,7 @@ import socket import socketserver import subprocess +import sys import threading import time import unittest @@ -316,6 +317,10 @@ def test_malformed_kind_is_a_refusal_not_an_exception(self): class DNSAndTLSBoundaryTests(unittest.TestCase): + # The worker's runner/scanner refuse non-Linux execution. These exercise + # its clean resolver environment and POSIX process/descriptor lifecycle, + # unlike the portable pure admission, TLS metadata and budget tests below. + @unittest.skipUnless(sys.platform.startswith("linux"), "Linux worker resolver process isolation") def test_dns_timeout_kills_and_reaps_the_helper(self): processes = [] real_popen = subprocess.Popen @@ -334,6 +339,7 @@ def start(*args, **kwargs): with self.assertRaises(ChildProcessError): os.waitpid(processes[0].pid, os.WNOHANG) + @unittest.skipUnless(sys.platform.startswith("linux"), "Linux worker resolver process isolation") def test_dns_helper_gets_no_parent_secrets_or_inherited_channel(self): read_fd, write_fd = os.pipe() self.addCleanup(os.close, read_fd) @@ -429,6 +435,10 @@ def connect(connection, deadline): return raw return connect + # Cross-thread shutdown of a duplicated socket enforces the Linux worker's + # hard deadline. Preserve its strict bound on the supported runtime rather + # than relaxing it to cover Windows socket cancellation behavior. + @unittest.skipUnless(sys.platform.startswith("linux"), "Linux worker hard socket deadline") def test_body_deadline_survives_connection_sock_handoff(self): transport = opener() started = time.monotonic() @@ -441,6 +451,7 @@ def test_body_deadline_survives_connection_sock_handoff(self): self.assertLess(time.monotonic() - started, 0.8) self.assertFalse(transport.meter.inflight) + @unittest.skipUnless(sys.platform.startswith("linux"), "Linux worker hard socket deadline") def test_slow_header_has_total_not_inactivity_deadline(self): transport = opener() started = time.monotonic()