From 3bc2c41c9119797d95da713a17e51e6453a2f584 Mon Sep 17 00:00:00 2001 From: svonava Date: Fri, 16 Oct 2026 16:32:21 -0700 Subject: [PATCH 01/16] feat(cross-encoder): add declared runtime LoRA lifecycle --- .../adapters/cross_encoder/__init__.py | 74 ++- .../adapters/cross_encoder/_lora.py | 384 ++++++++++++ .../tests/adapters/test_cross_encoder_lora.py | 583 ++++++++++++++++++ 3 files changed, 1040 insertions(+), 1 deletion(-) create mode 100644 packages/sie_server/src/sie_server/adapters/cross_encoder/_lora.py create mode 100644 packages/sie_server/tests/adapters/test_cross_encoder_lora.py diff --git a/packages/sie_server/src/sie_server/adapters/cross_encoder/__init__.py b/packages/sie_server/src/sie_server/adapters/cross_encoder/__init__.py index 4104fffe2..9b5c96ba1 100644 --- a/packages/sie_server/src/sie_server/adapters/cross_encoder/__init__.py +++ b/packages/sie_server/src/sie_server/adapters/cross_encoder/__init__.py @@ -17,7 +17,7 @@ """ from pathlib import Path -from typing import Any +from typing import TYPE_CHECKING, Any import torch from sentence_transformers import CrossEncoder @@ -28,6 +28,10 @@ from sie_server.core.inference_output import ScoreOutput from sie_server.types.inputs import Item +if TYPE_CHECKING: + from sie_server.adapters.cross_encoder._lora import CrossEncoderLora + from sie_server.adapters.lora import LoraRuntimeConfig + class CrossEncoderAdapter(BaseAdapter): """Adapter for cross-encoder reranker models. @@ -40,6 +44,15 @@ class CrossEncoderAdapter(BaseAdapter): These models take (query, document) pairs and output relevance scores. Higher scores indicate more relevant documents. + Runtime LoRA is opt-in through ``lora_paths={served_name: source}``. + Only ordinary A/B LoRA matrices targeting encoder Linear modules in a + standard, unquantized ``BertForSequenceClassification`` are supported. + The scoring head and activation stay those of the base checkpoint; + trained replacement heads, causal rerankers and custom model classes + are rejected when LoRA is enabled. Adapters load on demand under the + registry's resident limit. ``options.lora=None`` selects the base model. + Configurations without declarations use the original model and scoring path. + Performance optimizations: - SDPA attention (default): Uses PyTorch's scaled_dot_product_attention which can dispatch to flash attention when available @@ -61,6 +74,9 @@ def __init__( compute_precision: ComputePrecision | None = None, attn_implementation: AttnImplementation = "sdpa", revision: str | None = None, + lora_paths: dict[str, str] | None = None, + lora_revisions: dict[str, str | None] | None = None, + max_loaded_loras: int = 10, **kwargs: Any, # Accept extra args from loader (e.g., pooling) ) -> None: """Initialize the adapter. @@ -76,6 +92,12 @@ def __init__( revision: Optional HuggingFace revision/branch/commit SHA to pin when loading the model. If None, the default branch is used. Forwarded to ``CrossEncoder(..., revision=...)``. + lora_paths: Declared adapter aliases/sources. Enables deferred runtime + loading for standard BERT encoder projections only. Served names + use letters, digits, underscores, dots and hyphens; sources may + be local directories or Hugging Face repositories. + lora_revisions: Source revisions supplied by the model loader. + max_loaded_loras: Resident adapter limit, enforced with registry LRU eviction. **kwargs: Additional arguments (ignored, for compatibility). """ _ = kwargs # Unused, but accepted for loader compatibility @@ -88,6 +110,12 @@ def __init__( self._model: CrossEncoder | None = None self._device: str | None = None + self._lora: CrossEncoderLora | None = None + if lora_paths: + # Keep optional dependency imports and all LoRA state off the ordinary path. + from sie_server.adapters.cross_encoder._lora import CrossEncoderLora as LoraManager + + self._lora = LoraManager(lora_paths, lora_revisions, max_loaded_loras) def load(self, device: str) -> None: """Load the model onto the specified device. @@ -134,6 +162,50 @@ def load(self, device: str) -> None: if self._max_length is not None: self._model.max_length = self._max_length + if self._lora is not None: + self._lora.bind(self._model) + + def unload(self) -> None: + try: + if self._lora is not None: + self._lora.close() + finally: + super().unload() + + def supports_lora(self) -> bool: + return self._lora is not None + + def supports_hot_lora_reload(self) -> bool: + return self._lora is not None + + def lora_runtime_config(self) -> "LoraRuntimeConfig | None": + return self._lora.runtime_config() if self._lora is not None else None + + def fetch_lora(self, lora_path: str, revision: str | None = None) -> str: + if self._lora is None: + return super().fetch_lora(lora_path, revision) + return self._lora.fetch(lora_path, revision) + + def discard_lora_source(self, lora_path: str) -> None: + if self._lora is not None: + self._lora.discard(lora_path) + + def load_lora(self, lora_path: str, revision: str | None = None, *, source: str | None = None) -> int: + if self._lora is None: + return super().load_lora(lora_path, revision, source=source) + return self._lora.load(lora_path, revision, source=source) + + def unload_lora(self, lora_name: str) -> None: + if self._lora is None: + super().unload_lora(lora_name) + else: + self._lora.unload(lora_name) + + def set_active_lora(self, lora_name: str | None) -> None: + if self._lora is not None: + self._lora.select(lora_name) + elif lora_name is not None: + raise ValueError("No LoRA sources are declared for this cross-encoder") def score( self, diff --git a/packages/sie_server/src/sie_server/adapters/cross_encoder/_lora.py b/packages/sie_server/src/sie_server/adapters/cross_encoder/_lora.py new file mode 100644 index 000000000..e3d330407 --- /dev/null +++ b/packages/sie_server/src/sie_server/adapters/cross_encoder/_lora.py @@ -0,0 +1,384 @@ +"""Opt-in LoRA lifecycle for the ordinary sentence-transformers CrossEncoder. + +The outer wrapper, tokenizer, classifier and activation are never replaced. +Transformers injects ordinary LoRA layers into declared BERT encoder Linear +modules. The worker owns selection/mutation exclusion; only background source +preparation needs a separate lock here. No scoring code calls into this module. +""" + +from __future__ import annotations + +import math +import re +import shutil +import tempfile +import threading +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +import torch +from huggingface_hub import snapshot_download +from peft import LoraConfig, PeftConfig +from peft.functional import delete_adapter +from safetensors.torch import load_file +from sentence_transformers import CrossEncoder +from transformers import BertForSequenceClassification + +from sie_server.adapters.lora import LoraRuntimeConfig, LoraStateUncertainError, LoraUnavailableError +from sie_server.adapters.peft_lora_mixin import PEFTLoRAMixin + +_FILES = ("adapter_config.json", "adapter_model.safetensors") +_UNSUPPORTED = ( + "use_dora", + "use_rslora", + "use_qalora", + "use_bdlora", + "lora_bias", + "fan_in_fan_out", + "rank_pattern", + "alpha_pattern", + "layers_to_transform", + "layers_pattern", + "exclude_modules", + "target_parameters", + "alora_invocation_tokens", + "arrow_config", + "ensure_weight_tying", + "loftq_config", + "eva_config", + "corda_config", + "lora_ga_config", +) + + +@dataclass +class _Prepared: + directory: tempfile.TemporaryDirectory[str] + config: LoraConfig + weights: dict[str, torch.Tensor] + + +class CrossEncoderLora: + """Declared, bounded inference adapters; lifecycle calls run under the worker guard.""" + + def __init__( + self, + paths: dict[str, str], + revisions: dict[str, str | None] | None, + max_loaded: int, + ) -> None: + if isinstance(max_loaded, bool) or not isinstance(max_loaded, int) or max_loaded < 1: + raise ValueError("max_loaded_loras must be a positive integer") + if not isinstance(paths, dict): + raise ValueError("CrossEncoder lora_paths must map served aliases to declared sources") + aliases = dict(paths) + if any(not isinstance(k, str) or not k or not isinstance(v, str) or not v for k, v in aliases.items()): + raise ValueError("LoRA declarations must map nonempty names to nonempty sources") + if any(not re.fullmatch(r"[A-Za-z0-9_][A-Za-z0-9_.-]*", alias) for alias in aliases): + raise ValueError("LoRA served aliases must contain only letters, numbers, underscores, dots and hyphens") + self._sources = set(aliases.values()) + self._revisions = {source: (revisions or {}).get(source) for source in self._sources} + for source in self._sources: + if source in aliases and aliases[source] != source: + raise ValueError(f"LoRA alias {source!r} shadows a different declared source") + self._aliases = aliases + self._max_loaded = max_loaded + self._source_lock = threading.Lock() + self._prepared: dict[str, _Prepared] = {} + self._loaded: dict[str, int] = {} + self._parameters: dict[str, list[torch.nn.Parameter]] = {} + self._active: str | None = None + self._outer: CrossEncoder | None = None + self._model: Any = None + self._originals: dict[str, torch.nn.Linear] = {} + self._grad_flags: list[tuple[torch.nn.Parameter, bool]] = [] + self._closed = True + self._uncertain = False + self._generation = 0 + + def runtime_config(self) -> LoraRuntimeConfig: + return LoraRuntimeConfig(dict(self._aliases), frozenset(), frozenset(), self._max_loaded) + + def bind(self, outer: CrossEncoder) -> None: + model = outer.model + if type(model) is not BertForSequenceClassification or getattr(model, "is_quantized", False): + raise ValueError( + "CrossEncoder LoRA currently requires a standard, unquantized BertForSequenceClassification" + ) + if getattr(model, "peft_config", None) or getattr(model, "_hf_peft_config_loaded", False): + raise ValueError("The cross-encoder must start from a base checkpoint without attached adapters") + self._outer, self._model = outer, model + self._originals = {name: module for name, module in model.named_modules() if type(module) is torch.nn.Linear} + self._grad_flags = [(parameter, parameter.requires_grad) for parameter in model.parameters()] + self._active = None + self._uncertain = False + with self._source_lock: + self._generation += 1 + self._closed = False + + def _source(self, name: str) -> str: + source = self._aliases.get(name, name) + if source not in self._sources: + raise ValueError(f"LoRA {name!r} is not declared for this cross-encoder") + return source + + def _ready(self) -> None: + if self._uncertain: + raise LoraStateUncertainError("CrossEncoder LoRA state is uncertain; reload the model") + if self._closed or self._outer is None: + raise RuntimeError("CrossEncoder is not loaded") + + def _check_config(self, source: str, config: PeftConfig) -> LoraConfig: + PEFTLoRAMixin()._check_lora_config(source, config) + if not isinstance(config, LoraConfig): + raise ValueError("Only ordinary LoRA configurations are supported") + for field in _UNSUPPORTED: + if getattr(config, field, None): + raise ValueError(f"CrossEncoder LoRA does not support {field}") + if not isinstance(config.init_lora_weights, bool): + raise ValueError("CrossEncoder LoRA requires a plain boolean init_lora_weights") + if isinstance(config.r, bool) or not isinstance(config.r, int) or config.r < 1: + raise ValueError("LoRA rank must be a positive integer") + if not math.isfinite(config.lora_alpha) or config.lora_alpha <= 0: + raise ValueError("LoRA alpha must be finite and positive") + if not math.isfinite(config.lora_dropout) or not 0 <= config.lora_dropout < 1: + raise ValueError("LoRA dropout must be finite and in [0, 1)") + targets = config.target_modules + if not isinstance(targets, (set, list)) or not targets or any(not isinstance(t, str) or not t for t in targets): + raise ValueError("CrossEncoder LoRA requires an explicit list of target module names") + config.inference_mode = True + return config + + def _validate_weights(self, config: LoraConfig, weights: dict[str, torch.Tensor]) -> dict[str, torch.Tensor]: + expected: dict[str, tuple[int, int]] = {} + for target in config.target_modules or []: + matches = [name for name in self._originals if name == target or name.endswith(f".{target}")] + if not matches or any(not name.startswith("bert.encoder.") for name in matches): + raise ValueError(f"LoRA target {target!r} must resolve only to BERT encoder Linear modules") + for name in matches: + module = self._originals[name] + expected[f"{name}.lora_A.weight"] = (config.r, module.in_features) + expected[f"{name}.lora_B.weight"] = (module.out_features, config.r) + normalized: dict[str, torch.Tensor] = {} + for key, tensor in weights.items(): + name = key.removeprefix("base_model.model.") + if name in normalized: + raise ValueError(f"Duplicate LoRA weight {name!r}") + normalized[name] = tensor + if set(normalized) != set(expected): + raise ValueError("LoRA weights must exactly match every declared target's A/B matrices") + for name, tensor in normalized.items(): + if ( + tuple(tensor.shape) != expected[name] + or not tensor.is_floating_point() + or not torch.isfinite(tensor).all() + ): + raise ValueError(f"Invalid shape, dtype or nonfinite values in LoRA weight {name!r}") + module_name = name.rsplit(".", 2)[0] + dtype = self._originals[module_name].weight.dtype + if not torch.isfinite(tensor.to(dtype=dtype)).all(): + raise ValueError(f"LoRA weight {name!r} overflows the base model's compute dtype") + return normalized + + def fetch(self, name: str, revision: str | None = None) -> str: + self._ready() + source = self._source(name) + pinned = self._revisions[source] + if revision is not None and revision != pinned: + raise ValueError("LoRA revision differs from the declared source revision") + with self._source_lock: + generation = self._generation + if source in self._prepared: + return self._prepared[source].directory.name + directory = Path(source) + if directory.is_dir(): + if pinned is not None: + raise ValueError("A local LoRA directory cannot honor a Hugging Face revision pin") + else: + directory = Path(snapshot_download(source, revision=pinned, allow_patterns=list(_FILES))) + temporary = tempfile.TemporaryDirectory(prefix="sie-cross-encoder-lora-") + try: + # A private snapshot prevents later source edits or alternative pickle files + # from changing the checkpoint after validation. + for filename in _FILES: + shutil.copyfile(directory / filename, Path(temporary.name) / filename) + config = self._check_config(source, PeftConfig.from_pretrained(temporary.name, local_files_only=True)) + weights = self._validate_weights(config, load_file(str(Path(temporary.name) / _FILES[1]))) + with self._source_lock: + self._ready() + if generation != self._generation: + raise RuntimeError("CrossEncoder reloaded while LoRA source was being fetched") + existing = self._prepared.get(source) + if existing is not None: + temporary.cleanup() + return existing.directory.name + self._prepared[source] = _Prepared(temporary, config, weights) + return temporary.name + except BaseException: + temporary.cleanup() + raise + + def discard(self, name: str) -> None: + source = self._source(name) + with self._source_lock: + prepared = self._prepared.pop(source, None) + if prepared is not None: + try: + prepared.directory.cleanup() + except OSError: + # Keep ownership for model teardown to retry, rather than losing + # the path when filesystem cleanup fails. + with self._source_lock: + self._prepared[source] = prepared + raise + + def _restore_grad_flags(self) -> None: + for parameter, requires_grad in self._grad_flags: + parameter.requires_grad_(requires_grad) + + def _restore_tree(self) -> None: + for name, original in self._originals.items(): + parent_name, _, child = name.rpartition(".") + setattr(self._model.get_submodule(parent_name), child, original) + if hasattr(self._model, "peft_config"): + del self._model.peft_config + self._model._hf_peft_config_loaded = False + self._restore_grad_flags() + self._active = None + + def _remove(self, adapter_name: str) -> None: + # The low-level deletion also handles an injection that failed before + # Transformers set its configuration-loaded flag. + delete_adapter(self._model, adapter_name=adapter_name, prefix="lora_") + getattr(self._model, "peft_config", {}).pop(adapter_name, None) + + def load(self, name: str, revision: str | None = None, *, source: str | None = None) -> int: + self._ready() + canonical = self._source(name) + if canonical in self._loaded: + return self._loaded[canonical] + if len(self._loaded) >= self._max_loaded: + raise ValueError("CrossEncoder resident LoRA capacity is exhausted; evict an adapter first") + if source is None: + source = self.fetch(canonical, revision) + with self._source_lock: + prepared = self._prepared.get(canonical) + if prepared is None or prepared.directory.name != source: + raise ValueError("LoRA source must be the validated snapshot fetched for this declaration") + adapter_name = PEFTLoRAMixin._peft_adapter_name(canonical) + try: + assert self._outer is not None + self._outer.load_adapter( + adapter_name=adapter_name, + peft_config=prepared.config, + adapter_state_dict=prepared.weights, + is_trainable=False, + ) + memory = 0 + parameters = [] + for key, weight in prepared.weights.items(): + module_name, part, _ = key.rsplit(".", 2) + parameter = getattr(self._model.get_submodule(module_name), part)[adapter_name].weight + # Transformers only warns on missing/unexpected keys. Verify the + # actual installed tensors before admitting the adapter. + if not torch.isfinite(parameter).all() or not torch.equal( + parameter.detach(), weight.to(device=parameter.device, dtype=parameter.dtype) + ): + raise ValueError(f"LoRA weight {key!r} was not installed exactly") + parameter.requires_grad_(False) + parameters.append(parameter) + memory += parameter.numel() * parameter.element_size() + self._outer.disable_adapters() + self._restore_grad_flags() + self._active = None + self._loaded[canonical] = memory + self._parameters[canonical] = parameters + except BaseException: + try: + if self._loaded: + self._remove(adapter_name) + assert self._outer is not None + self._outer.disable_adapters() + self._restore_grad_flags() + self._active = None + else: + self._restore_tree() + except BaseException as rollback: + self._uncertain = True + raise LoraStateUncertainError("Failed to roll back CrossEncoder LoRA attachment") from rollback + raise + finally: + try: + self.discard(canonical) + except OSError as exc: + self._uncertain = True + raise LoraStateUncertainError("LoRA source cleanup failed; retain capacity until model reload") from exc + return memory + + def select(self, name: str | None) -> None: + self._ready() + canonical = self._source(name) if name is not None else None + if canonical is not None and canonical not in self._loaded: + raise LoraUnavailableError(f"Declared LoRA {name!r} is not resident") + if canonical == self._active: + return + assert self._outer is not None + try: + if canonical is None: + self._outer.disable_adapters() + else: + self._outer.set_adapter(PEFTLoRAMixin._peft_adapter_name(canonical)) + self._outer.enable_adapters() + # Transformers' enable/set hooks default to trainable adapters. + for parameter in self._parameters[canonical]: + parameter.requires_grad_(False) + self._restore_grad_flags() + self._active = canonical + except BaseException as exc: + self._uncertain = True + raise LoraStateUncertainError("Failed to select CrossEncoder LoRA; reload the model") from exc + + def unload(self, name: str) -> None: + self._ready() + canonical = self._source(name) + if canonical not in self._loaded: + raise LoraUnavailableError(f"LoRA {name!r} is not resident") + try: + self._remove(PEFTLoRAMixin._peft_adapter_name(canonical)) + if len(self._loaded) == 1: + self._restore_tree() + else: + assert self._outer is not None + self._outer.disable_adapters() + self._restore_grad_flags() + self._active = None + except BaseException as exc: + self._uncertain = True + raise LoraStateUncertainError("Failed to unload CrossEncoder LoRA; reload the model") from exc + del self._loaded[canonical] + del self._parameters[canonical] + self.discard(canonical) + + def close(self) -> None: + with self._source_lock: + self._closed = True + prepared, self._prepared = self._prepared, {} + errors = [] + try: + for item in prepared.values(): + try: + item.directory.cleanup() + except OSError as exc: + errors.append(exc) + if self._model is not None: + self._restore_tree() + finally: + self._loaded.clear() + self._parameters.clear() + self._originals.clear() + self._grad_flags.clear() + self._outer = self._model = None + self._uncertain = False + if errors: + raise ExceptionGroup("Failed to remove owned LoRA source snapshots", errors) diff --git a/packages/sie_server/tests/adapters/test_cross_encoder_lora.py b/packages/sie_server/tests/adapters/test_cross_encoder_lora.py new file mode 100644 index 000000000..59c19fbdd --- /dev/null +++ b/packages/sie_server/tests/adapters/test_cross_encoder_lora.py @@ -0,0 +1,583 @@ +"""Real BERT/Transformers/PEFT lifecycle checks without model downloads.""" + +from __future__ import annotations + +import builtins +import json +import time +from concurrent.futures import ThreadPoolExecutor +from pathlib import Path + +import httpx +import numpy as np +import pytest +import torch +import yaml +from peft import LoraConfig +from peft.tuners.tuners_utils import BaseTunerLayer +from safetensors.torch import load_file, save_file +from sie_server.adapters.cross_encoder import CrossEncoderAdapter +from sie_server.adapters.lora import LoraStateUncertainError, LoraUnavailableError +from sie_server.app.app_factory import AppFactory +from sie_server.app.app_state_config import AppStateConfig +from sie_server.core.registry import ModelRegistry +from transformers import BertConfig, BertForSequenceClassification, BertTokenizerFast + +QUERY = {"text": "how do i reset my password"} +DOCS = [{"text": "reset your password by email"}, {"text": "the capital is paris"}] + + +@pytest.fixture(scope="module") +def checkpoint(tmp_path_factory): + directory = tmp_path_factory.mktemp("cross-encoder-base") + vocab = [ + "[PAD]", + "[UNK]", + "[CLS]", + "[SEP]", + "[MASK]", + *["how", "do", "i", "reset", "my", "password", "your", "by", "email", "the", "capital", "is", "paris"], + ] + vocab = list(dict.fromkeys(vocab)) + vocab_path = directory / "vocab.txt" + vocab_path.write_text("\n".join(vocab)) + BertTokenizerFast(vocab_file=str(vocab_path), model_max_length=64).save_pretrained(directory) + config = BertConfig( + vocab_size=len(vocab), + hidden_size=16, + num_hidden_layers=2, + num_attention_heads=2, + intermediate_size=32, + max_position_embeddings=64, + num_labels=1, + hidden_dropout_prob=0, + attention_probs_dropout_prob=0, + ) + config.sbert_ce_default_activation_function = "torch.nn.modules.linear.Identity" + with torch.random.fork_rng(): + torch.manual_seed(29) + BertForSequenceClassification(config).save_pretrained(directory) + return directory + + +def save_adapter(path, seed=1, **overrides): + path.mkdir() + LoraConfig( + task_type="SEQ_CLS", + r=4, + lora_alpha=8, + target_modules=["query", "value"], + bias="none", + lora_dropout=0, + inference_mode=True, + ).save_pretrained(path) + if overrides: + config_path = path / "adapter_config.json" + config = json.loads(config_path.read_text()) + config.update(overrides) + config_path.write_text(json.dumps(config)) + generator = torch.Generator().manual_seed(seed) + weights = {} + for layer in range(2): + for target in ("query", "value"): + prefix = f"base_model.model.bert.encoder.layer.{layer}.attention.self.{target}" + weights[f"{prefix}.lora_A.weight"] = torch.randn(4, 16, generator=generator) * 0.4 + weights[f"{prefix}.lora_B.weight"] = torch.randn(16, 4, generator=generator) * 0.4 + save_file(weights, path / "adapter_model.safetensors") + return str(path) + + +@pytest.fixture +def adapter(checkpoint, tmp_path): + paths = {name: save_adapter(tmp_path / name, seed) for seed, name in enumerate(("a", "b", "c"), 1)} + adapter = CrossEncoderAdapter(checkpoint, lora_paths=paths, max_loaded_loras=2) + adapter.load("cpu") + yield adapter + adapter.unload() + + +def scores(adapter): + from sie_server.types.inputs import Item + + query, docs = Item(**QUERY), [Item(**doc) for doc in DOCS] + direct = np.asarray(adapter.score(query, docs)) + pairs = adapter.score_pairs([query] * len(docs), docs) + assert np.isfinite(direct).all() + np.testing.assert_allclose(pairs.scores, direct, atol=1e-7, rtol=0) + assert pairs.input_token_counts + assert all(count > 0 for count in pairs.input_token_counts) + return direct + + +def base_snapshot(adapter): + outer = adapter._model + return ( + outer, + outer.model, + outer.model.classifier, + outer.tokenizer, + outer.activation_fn, + dict(outer.model.named_modules()), + {name: value.detach().clone() for name, value in outer.model.state_dict().items()}, + [(parameter, parameter.requires_grad) for parameter in outer.model.parameters()], + ) + + +def assert_base_preserved(adapter, snapshot, *, unwrapped=False): + outer, model, head, tokenizer, activation, modules, weights, grad = snapshot + assert adapter._model is outer + assert outer.model is model + assert model.classifier is head + assert outer.tokenizer is tokenizer + assert outer.activation_fn is activation + for name, expected in weights.items(): + parent, _, leaf = name.rpartition(".") + module = model.get_submodule(parent) + if isinstance(module, BaseTunerLayer): + module = module.get_base_layer() + assert torch.equal(getattr(module, leaf), expected), name + assert all(parameter.requires_grad == flag for parameter, flag in grad) + if unwrapped: + assert dict(model.named_modules()) == modules + assert not hasattr(model, "peft_config") + assert not model._hf_peft_config_loaded + + +def test_never_lora_has_no_manager_or_optional_imports(checkpoint, monkeypatch): + original = builtins.__import__ + + def guarded_import(name, *args, **kwargs): + assert name != "sie_server.adapters.cross_encoder._lora" + return original(name, *args, **kwargs) + + monkeypatch.setattr(builtins, "__import__", guarded_import) + adapter = CrossEncoderAdapter(checkpoint) + assert adapter._lora is None + assert not adapter.supports_lora() + assert adapter.lora_capability() is None + assert not adapter.supports_hot_lora_reload() + assert adapter.lora_runtime_config() is None + adapter.load("cpu") + before = base_snapshot(adapter) + baseline = scores(adapter) + adapter.set_active_lora(None) + np.testing.assert_array_equal(scores(adapter), baseline) + assert_base_preserved(adapter, before) + with pytest.raises(ValueError, match="No LoRA sources"): + adapter.set_active_lora("a") + adapter.unload() + + +def test_actual_effect_restoration_head_and_final_module_identity(adapter): + snapshot = base_snapshot(adapter) + baseline = scores(adapter) + np.testing.assert_array_equal(scores(adapter), baseline) + runtime = adapter.lora_runtime_config() + assert runtime is not None + assert runtime.max_loaded == 2 + assert set(runtime.aliases) == {"a", "b", "c"} + assert not runtime.preloaded + assert not runtime.pinned + with pytest.raises(LoraUnavailableError): + adapter.set_active_lora("a") + assert adapter.load_lora("a") == 2048 + # Successful attachment must leave base selected, including SEQ_CLS configs. + np.testing.assert_array_equal(scores(adapter), baseline) + adapter.set_active_lora("a") + a = scores(adapter) + assert not any(parameter.requires_grad for parameter in adapter._lora._parameters[adapter._lora._aliases["a"]]) + assert np.max(np.abs(a - baseline)) > 1e-6 + assert_base_preserved(adapter, snapshot) + adapter.load_lora("b") + np.testing.assert_array_equal(scores(adapter), baseline) + adapter.set_active_lora("b") + b = scores(adapter) + assert np.max(np.abs(b - a)) > 1e-6 + adapter.set_active_lora(None) + np.testing.assert_array_equal(scores(adapter), baseline) + with pytest.raises(ValueError, match="capacity"): + adapter.load_lora("c") + adapter.unload_lora("a") + with pytest.raises(LoraUnavailableError): + adapter.set_active_lora("a") + adapter.set_active_lora("b") + np.testing.assert_array_equal(scores(adapter), b) + adapter.unload_lora("b") + assert_base_preserved(adapter, snapshot, unwrapped=True) + np.testing.assert_array_equal(scores(adapter), baseline) + adapter.load_lora("a") + adapter.set_active_lora("a") + np.testing.assert_array_equal(scores(adapter), a) + + +@pytest.mark.parametrize("existing", [False, True]) +def test_failed_attachment_rolls_back_entire_base_or_only_failed_adapter(adapter, monkeypatch, existing): + snapshot = base_snapshot(adapter) + baseline = scores(adapter) + if existing: + adapter.load_lora("a") + adapter.set_active_lora("a") + a = scores(adapter) + original = adapter._model.load_adapter + + def fail_after_injection(*args, **kwargs): + original(*args, **kwargs) + raise RuntimeError("injected post-load failure") + + with monkeypatch.context() as context: + context.setattr(adapter._model, "load_adapter", fail_after_injection) + with pytest.raises(RuntimeError, match="injected post-load"): + adapter.load_lora("b") + assert_base_preserved(adapter, snapshot, unwrapped=not existing) + np.testing.assert_array_equal(scores(adapter), baseline) + assert adapter._lora._prepared == {} + with pytest.raises(LoraUnavailableError): + adapter.set_active_lora("b") + if existing: + adapter.set_active_lora("a") + np.testing.assert_array_equal(scores(adapter), a) + adapter.load_lora("b") + adapter.set_active_lora("b") + assert np.max(np.abs(scores(adapter) - baseline)) > 1e-6 + + +@pytest.mark.parametrize("damage", ["missing", "empty", "foreign", "shape", "nan", "overflow", "duplicate"]) +def test_bad_weights_rejected_before_model_mutation(adapter, damage): + source = Path(adapter._lora._aliases["a"]) + weights_path = source / "adapter_model.safetensors" + weights = load_file(weights_path) + key = next(iter(weights)) + if damage == "missing": + del weights[key] + elif damage == "empty": + weights.clear() + elif damage == "foreign": + weights["base_model.model.classifier.weight"] = torch.ones(1, 16) + elif damage == "shape": + weights[key] = torch.ones(3, 16) + elif damage == "nan": + weights[key][0, 0] = float("nan") + elif damage == "overflow": + weights[key] = torch.full(weights[key].shape, 1e300, dtype=torch.float64) + else: + weights[key.removeprefix("base_model.model.")] = weights[key].clone() + save_file(weights, weights_path) + snapshot = base_snapshot(adapter) + with pytest.raises(ValueError, match="LoRA"): + adapter.load_lora("a") + assert_base_preserved(adapter, snapshot, unwrapped=True) + assert adapter._lora._prepared == {} + assert not adapter._lora._loaded + + +@pytest.mark.parametrize( + "change", + [ + {"modules_to_save": ["classifier"]}, + {"bias": "all"}, + {"target_modules": ["classifier"]}, + {"target_modules": ["dense"]}, + {"target_modules": ["does_not_exist"]}, + {"target_modules": "query"}, + {"init_lora_weights": "pissa"}, + {"use_dora": True}, + {"rank_pattern": {"query": 2}}, + {"target_parameters": ["classifier.weight"]}, + {"lora_alpha": float("inf")}, + ], +) +def test_unsafe_or_unqualified_config_is_rejected(adapter, change): + path = Path(adapter._lora._aliases["a"]) / "adapter_config.json" + config = json.loads(path.read_text()) + config.update(change) + path.write_text(json.dumps(config)) + snapshot = base_snapshot(adapter) + with pytest.raises(ValueError, match="LoRA"): + adapter.load_lora("a") + assert_base_preserved(adapter, snapshot, unwrapped=True) + + +def test_only_declared_private_snapshot_can_attach(adapter, tmp_path): + unknown = save_adapter(tmp_path / "unknown") + with pytest.raises(ValueError, match="not declared"): + adapter.fetch_lora(unknown) + with pytest.raises(ValueError, match="validated snapshot"): + adapter.load_lora("a", source=unknown) + prepared = Path(adapter.fetch_lora("a")) + assert prepared.is_dir() + assert prepared != Path(adapter._lora._aliases["a"]) + (Path(adapter._lora._aliases["a"]) / "adapter_model.safetensors").write_bytes(b"corrupted after validation") + assert adapter.load_lora("a", source=str(prepared)) > 0 + assert not prepared.exists() + + +def test_uncertain_mutation_refuses_even_base_until_whole_model_unload(adapter, monkeypatch): + adapter.load_lora("a") + snapshot = base_snapshot(adapter) + + def fail(*args, **kwargs): + raise RuntimeError("injected rollback failure") + + with monkeypatch.context() as context: + context.setattr(adapter._model, "load_adapter", fail) + context.setattr(adapter._lora, "_remove", fail) + with pytest.raises(LoraStateUncertainError): + adapter.load_lora("b") + with pytest.raises(LoraStateUncertainError): + adapter.set_active_lora(None) + adapter.unload() + assert not any(isinstance(module, BaseTunerLayer) for module in snapshot[1].modules()) + assert adapter._lora._outer is None + assert adapter._lora._originals == {} + adapter.load("cpu") + adapter.load_lora("a") + adapter.set_active_lora("a") + assert np.isfinite(scores(adapter)).all() + + +def test_discard_and_model_unload_release_owned_sources(adapter): + a = Path(adapter.fetch_lora("a")) + adapter.discard_lora_source("a") + assert not a.exists() + b = Path(adapter.fetch_lora("b")) + model = adapter._model.model + adapter.unload() + assert not b.exists() + assert not adapter._lora._prepared + assert not any(isinstance(module, BaseTunerLayer) for module in model.modules()) + with pytest.raises(RuntimeError, match="not loaded"): + adapter.fetch_lora("a") + + +def test_revision_and_capacity_validation(checkpoint, tmp_path): + path = save_adapter(tmp_path / "a") + with pytest.raises(ValueError, match="positive integer"): + CrossEncoderAdapter(checkpoint, lora_paths={"a": path}, max_loaded_loras=0) + adapter = CrossEncoderAdapter(checkpoint, lora_paths={"a": path}, lora_revisions={path: "a" * 40}) + adapter.load("cpu") + try: + with pytest.raises(ValueError, match="differs"): + adapter.fetch_lora("a", revision="b" * 40) + with pytest.raises(ValueError, match="local LoRA"): + adapter.fetch_lora("a") + finally: + adapter.unload() + + +def test_revisions_alone_do_not_enable_or_authorize_lora(checkpoint): + adapter = CrossEncoderAdapter(checkpoint, lora_revisions={"other-profile/source": "a" * 40}) + assert adapter._lora is None + assert adapter.lora_capability() is None + + +@pytest.mark.parametrize("paths", [["org/source"], {"/private/source": "org/source"}, {"a": "b", "b": "c"}]) +def test_aliases_are_public_names_and_cannot_shadow_sources(checkpoint, paths): + with pytest.raises(ValueError, match=r"aliases|shadows"): + CrossEncoderAdapter(checkpoint, lora_paths=paths) + + +def test_fetch_rechecks_model_generation_before_publishing_source(adapter, monkeypatch): + from sie_server.adapters.cross_encoder import _lora + + original = _lora.load_file + + def reload_during_fetch(path): + weights = original(path) + adapter.unload() + adapter.load("cpu") + return weights + + monkeypatch.setattr(_lora, "load_file", reload_during_fetch) + with pytest.raises(RuntimeError, match="reloaded while"): + adapter.fetch_lora("a") + assert adapter._lora._prepared == {} + + +@pytest.mark.parametrize("existing", [False, True]) +def test_partial_injection_failure_preserves_old_adapters_and_base(adapter, monkeypatch, existing): + from peft.tuners.lora.model import LoraModel + + snapshot = base_snapshot(adapter) + baseline = scores(adapter) + if existing: + adapter.load_lora("a") + adapter.set_active_lora("a") + a = scores(adapter) + original = LoraModel._create_and_replace + calls = 0 + + def fail_second_layer(self, *args, **kwargs): + nonlocal calls + calls += 1 + if calls == 2: + raise RuntimeError("injected mid-attachment failure") + return original(self, *args, **kwargs) + + with monkeypatch.context() as context: + context.setattr(LoraModel, "_create_and_replace", fail_second_layer) + with pytest.raises(RuntimeError, match="mid-attachment"): + adapter.load_lora("b") + assert calls == 2 + assert_base_preserved(adapter, snapshot, unwrapped=not existing) + np.testing.assert_array_equal(scores(adapter), baseline) + if existing: + adapter.set_active_lora("a") + np.testing.assert_array_equal(scores(adapter), a) + adapter.load_lora("b") + + +def test_source_cleanup_failure_retains_capacity_and_blocks_selection(adapter, monkeypatch): + path = adapter.fetch_lora("a") + canonical = adapter._lora._aliases["a"] + temporary = adapter._lora._prepared[canonical].directory + + def fail_cleanup(): + raise OSError("injected cleanup failure") + + with monkeypatch.context() as context: + context.setattr(temporary, "cleanup", fail_cleanup) + with pytest.raises(LoraStateUncertainError, match="cleanup failed"): + adapter.load_lora("a", source=path) + assert canonical in adapter._lora._loaded + assert canonical in adapter._lora._prepared + with pytest.raises(LoraStateUncertainError): + adapter.set_active_lora(None) + adapter.unload() + assert not Path(path).exists() + + +def test_source_cleanup_failure_cannot_skip_whole_model_teardown(adapter, monkeypatch): + adapter.load_lora("a") + original_model = adapter._model.model + path = adapter.fetch_lora("b") + temporary = adapter._lora._prepared[adapter._lora._aliases["b"]].directory + + def fail_cleanup(): + raise OSError("injected cleanup failure") + + with monkeypatch.context() as context: + context.setattr(temporary, "cleanup", fail_cleanup) + with pytest.raises(ExceptionGroup, match="owned LoRA"): + adapter.unload() + assert adapter._model is None + assert adapter._lora._model is None + assert not adapter._lora._loaded + assert not any(isinstance(module, BaseTunerLayer) for module in original_model.modules()) + assert Path(path).exists() + temporary.cleanup() + + +@pytest.mark.usefixtures("_offline_apps") +def test_real_score_routes_authorize_aliases_isolate_batches_and_evict( + checkpoint, + tmp_path, + monkeypatch, + serve_on_loopback, +): + from sie_server.adapters.cross_encoder._lora import CrossEncoderLora + + paths = {name: save_adapter(tmp_path / name, index) for index, name in enumerate(("a", "b", "c", "other"), 1)} + models = tmp_path / "models" + models.mkdir() + config = { + "sie_id": "test/cross-encoder-lora", + "weights_path": str(checkpoint), + "inputs": {"text": True}, + "tasks": {"score": {}}, + "max_sequence_length": 64, + "profiles": { + "default": { + "adapter_path": "sie_server.adapters.cross_encoder:CrossEncoderAdapter", + "max_batch_tokens": 8192, + "adapter_options": {"loadtime": {"lora_paths": {name: paths[name] for name in ("a", "b", "c")}}}, + }, + "other": {"extends": "default", "adapter_options": {"loadtime": {"lora_paths": {"other": paths["other"]}}}}, + }, + } + (models / "model.yaml").write_text(yaml.safe_dump(config)) + monkeypatch.setenv("SIE_MAX_LORAS_PER_MODEL", "2") + monkeypatch.setenv("SIE_DISK_CACHE_ENABLED", "false") + + async def no_watcher(self): + pass + + monkeypatch.setattr(ModelRegistry, "start_hot_reload", no_watcher) + fetched = [] + original_fetch = CrossEncoderLora.fetch + + def observed_fetch(self, name, revision=None): + fetched.append(name) + return original_fetch(self, name, revision) + + monkeypatch.setattr(CrossEncoderLora, "fetch", observed_fetch) + app = AppFactory.create_app(AppStateConfig(models_dir=str(models), device="cpu", remote_serving=False)) + model = config["sie_id"] + with ( + serve_on_loopback(app) as url, + httpx.Client(base_url=url, timeout=15, headers={"Accept": "application/json"}) as client, + ): + + def request(selection=None, *, profile=None): + options = {"lora": selection} + if profile is not None: + options["profile"] = profile + return client.post(f"/v1/score/{model}", json={"query": QUERY, "items": DOCS, "options": options}) + + def ready(selection=None): + deadline = time.monotonic() + 20 + response = request(selection) + statuses = [response.status_code] + if selection is not None and response.status_code == 503: + assert response.json()["detail"]["code"] == "LORA_LOADING" + assert float(response.headers["Retry-After"]) > 0 + while response.status_code == 503 and time.monotonic() < deadline: + time.sleep(0.02) + response = request(selection) + statuses.append(response.status_code) + assert response.status_code == 200, response.text + values = np.asarray( + [entry["score"] for entry in sorted(response.json()["scores"], key=lambda row: row["item_id"])] + ) + assert np.isfinite(values).all() + return values, statuses + + baseline, _ = ready() + assert fetched == [] + for rejected, profile in ( + ("https://example.invalid/adapter", None), + (str(tmp_path / "not-declared"), None), + ("other", None), + (paths["other"], None), + ("a", "other"), + ): + response = request(rejected, profile=profile) + assert response.status_code == 400, response.text + assert response.json()["detail"]["code"] == "INVALID_INPUT" + assert fetched == [] + a, statuses = ready("a") + assert statuses[0] == 503 + b, statuses = ready("b") + assert statuses[0] == 503 + assert np.max(np.abs(a - baseline)) > 1e-6 + assert np.max(np.abs(b - a)) > 1e-6 + assert set(app.state.registry.loaded_lora_names(model)) == {"a", "b"} + listed = client.get("/v1/models").json()["models"] + metadata = next(item for item in listed if item["name"] == model) + assert set(metadata["lora_adapters"]) == {"a", "b"} + assert not any(source in json.dumps(metadata["lora_adapters"]) for source in paths.values()) + assert len(app.state.registry._loaded[model].loras) == 2 + assert all(value.memory_bytes == 2048 for value in app.state.registry._loaded[model].loras.values()) + expected = {None: baseline, "a": a, "b": b} + selections = ["a", None, "b"] * 8 + with ThreadPoolExecutor(max_workers=6) as executor: + futures = [(selection, executor.submit(ready, selection)) for selection in selections] + for selection, future in futures: + actual, statuses = future.result(timeout=20) + assert statuses == [200] + np.testing.assert_allclose(actual, expected[selection], atol=1e-6, rtol=0) + ready("b") # A is now the least recently used resident. + ready("c") + assert set(app.state.registry.loaded_lora_names(model)) == {"b", "c"} + assert not app.state.registry.is_lora_loaded(model, "a") + np.testing.assert_allclose(ready()[0], baseline, atol=1e-6, rtol=0) + assert fetched == [paths["a"], paths["b"], paths["c"]] From 3d9993c35e8e04740cff743bcb572027d92bd8b4 Mon Sep 17 00:00:00 2001 From: svonava Date: Fri, 16 Oct 2026 16:50:52 -0700 Subject: [PATCH 02/16] test(cross-encoder): add paired runtime LoRA qualification --- .../benchmarks/cross_encoder_lora/.gitignore | 2 + tools/benchmarks/cross_encoder_lora/README.md | 154 ++++ .../benchmarks/cross_encoder_lora/__init__.py | 0 .../benchmarks/cross_encoder_lora/adapter.py | 94 ++ .../benchmarks/cross_encoder_lora/evidence.py | 63 ++ .../benchmarks/cross_encoder_lora/fixtures.py | 72 ++ .../benchmarks/cross_encoder_lora/prepare.py | 69 ++ .../benchmarks/cross_encoder_lora/protocol.py | 102 +++ tools/benchmarks/cross_encoder_lora/runner.py | 270 ++++++ .../cross_encoder_lora/tests/__init__.py | 0 .../cross_encoder_lora/tests/conftest.py | 6 + .../cross_encoder_lora/tests/test_probe.py | 334 +++++++ tools/benchmarks/cross_encoder_lora/worker.py | 867 ++++++++++++++++++ 13 files changed, 2033 insertions(+) create mode 100644 tools/benchmarks/cross_encoder_lora/.gitignore create mode 100644 tools/benchmarks/cross_encoder_lora/README.md create mode 100644 tools/benchmarks/cross_encoder_lora/__init__.py create mode 100644 tools/benchmarks/cross_encoder_lora/adapter.py create mode 100644 tools/benchmarks/cross_encoder_lora/evidence.py create mode 100644 tools/benchmarks/cross_encoder_lora/fixtures.py create mode 100644 tools/benchmarks/cross_encoder_lora/prepare.py create mode 100644 tools/benchmarks/cross_encoder_lora/protocol.py create mode 100644 tools/benchmarks/cross_encoder_lora/runner.py create mode 100644 tools/benchmarks/cross_encoder_lora/tests/__init__.py create mode 100644 tools/benchmarks/cross_encoder_lora/tests/conftest.py create mode 100644 tools/benchmarks/cross_encoder_lora/tests/test_probe.py create mode 100644 tools/benchmarks/cross_encoder_lora/worker.py diff --git a/tools/benchmarks/cross_encoder_lora/.gitignore b/tools/benchmarks/cross_encoder_lora/.gitignore new file mode 100644 index 000000000..5b3c90724 --- /dev/null +++ b/tools/benchmarks/cross_encoder_lora/.gitignore @@ -0,0 +1,2 @@ +_inputs/ +_evidence/ diff --git a/tools/benchmarks/cross_encoder_lora/README.md b/tools/benchmarks/cross_encoder_lora/README.md new file mode 100644 index 000000000..988ac98fe --- /dev/null +++ b/tools/benchmarks/cross_encoder_lora/README.md @@ -0,0 +1,154 @@ +# CrossEncoder runtime LoRA qualification + +This probe measures the ordinary `CrossEncoderAdapter` using the real SIE loader, +worker, loopback HTTP API and `sie_sdk.SIEClient`. It does not provision compute, +install dependencies, change precision, or change the production catalog. The +catalog model is overridden locally to use the ordinary adapter rather than the +catalog's BertFlash implementation. + +The fixed checkpoint is `cross-encoder/ms-marco-MiniLM-L-6-v2` at +`c5ee24cb16019beea0893ab7796b1df96625c6b8`; `model.safetensors` must hash to +`821d1aa69520101d6e0737f78a042ae25b19e5cb9160701909d10434f4aeb0ae`. +Fixtures A/B/C are deterministic, untrained rank-4, alpha-8, bias-free LoRAs on +every BERT encoder query/value projection. They establish reversible numerical +effects and lifecycle behavior, **not retrieval quality**. Source seeds, model +files and adapter files are hashed in the fixture manifest. + +## Prepare immutable sources and fixtures + +From a clean public checkout with the candidate committed: + +```console +mise tasks +mise exec -- uv run --no-sync python -m tools.benchmarks.cross_encoder_lora.prepare --candidate HEAD --output /tmp/ce-sources +mise exec -- uv run --no-sync python -m tools.benchmarks.cross_encoder_lora.fixtures --output /tmp/ce-fixtures +``` + +The preparer resolves refs once, archives baseline +`d5e024ad3ba6127b1ee176e2c85c794b76077888` and the candidate by full commit ID, +and refuses differing dependency locks. Keep the archives with their manifest. +The same benchmark code observes every arm; `PYTHONPATH` selects the archived +server and SDK. No arm runs a dependency sync. Pass `--snapshot` to both fixture +preparation and the runner when using a retained public checkpoint directory; +the worker checks its full recorded file manifest before loading it. + +Prepare one Linux L4 environment using the public lock. The qualification +preflight requires Torch `2.9.1+cu129`, Transformers `4.57.6`, +SentenceTransformers `5.4.1`, PEFT `0.19.1`, and **no installed `torchao`**. +Four Torch intra-op threads and one inter-op thread are fixed. Default CUDA +precision remains unchanged (fp16 for this adapter). Record any environment +preparation separately, including installed versions before and after; the +benchmark only inspects them and fails on a mismatch. + +```console +mise exec -- uv run --no-sync python -m tools.benchmarks.cross_encoder_lora.runner \ + --inputs /tmp/ce-sources --fixtures /tmp/ce-fixtures \ + --output /tmp/ce-evidence --deadline-s 1740 --trial-deadline-s 120 +``` + +Use a fresh output directory. Set the outer compute-session deadline before +launching, allowing bounded startup and cleanup. The runner owns and joins only +its subprocesses; it cannot terminate an external GPU session. The operator +must stop that exact session and record cleanup. A failed or incomplete trial +stops the experiment and retains its evidence. Never change thresholds or drop +failed trials to obtain a pass. + +## Frozen comparison + +There are six fresh-process blocks, in order `BNR`, `NRB`, `RBN`, `BRN`, `RNB`, +`NBR` (18 fresh servers). B uses the pinned baseline with no declared LoRA. +N uses the candidate with no declared LoRA. R uses the candidate after A/B +activity, with base selected and two adapters resident but disabled. + +Every arm uses the same password-reset query and four documents in `protocol.py`. +Each phase starts with 30 sequential warmup requests. C1 measures 200 sequential +requests; C4 and C16 each run a 12-second closed loop with one SDK client per +lane. Throughput includes draining requests admitted before the deadline. +Latencies use the nearest-rank p95. Neither warmup nor correctness calls enter +the primary statistics. + +All six metrics are required for **both** N/B and R/B: C1 mean and p95 latency, +C4/C16 requests per second and p95 latency. The statistical unit is the six +paired blocks. Latency ratios are candidate/B; throughput ratios are B/candidate. +For log ratios, the one-sided 95% t upper bound (df=5, critical +`2.0150483733330233`) must be at most 1.03, the geometric point ratio at most +1.01, and every paired ratio at most 1.10. A wide interval is inconclusive. + +Before selection, three SDK base results must be finite and stable at absolute +tolerance `1e-6`. The public fp32 checkpoint oracle has a separately fixed `0.1` +CUDA-fp16 compatibility bound. Every ordered batch shape of 4, 8, …, 64 pairs +has two preselection oracle runs, stable at `1e-6`. Actual batches must match +the corresponding ordered-workload fingerprint and shape; unexpected partial +requests or ordering fail qualification. All candidate base oracles and observed +untimed SDK replay outputs must match the corresponding B-block oracle at `1e-6`. This is +deliberately strict and is never relaxed after measurement. + +R also checks cold `503 LORA_LOADING`, bounded fresh SDK retries, duplicate-load +single-flight, undeclared-source rejection before fetch, distinct A/base/B +effects, base restoration, real concurrent base/A/B isolation and same-selection +cross-request batching. It records actual PEFT selection at model forward +boundaries. Mutation cannot overlap a score forward. Base tensors and classifier, +tokenizer and outer-wrapper identities remain unchanged. Churn exceeds the +two-adapter capacity; final removal restores the original Linear objects, +module tree and PEFT flags; same-instance reload and full model unload follow. +Production fault-injection tests separately cover malformed artifacts and +partial-attachment rollback; this probe does not pretend to reproduce them on +every GPU trial. + +The hook records deep selection/logit details during correctness checks and +two observed SDK replay rounds at C1/C4/C16. During timed phases the forward +hook is removed and production `score_pairs` and `set_active_lora` are bound +directly. The harness asserts that no observer activity occurred. Response +validation and header decoding happen after the phase clock stops; exactly one +successful HTTP response is required per warmup/measured SDK call, excluding +hidden retries. The timed client retains timer, result-list and raw-response +list-append overhead; this is an SDK workload measurement. + +Actual timed C4/C16 batch shapes are **unknown**. Their raw four-score vectors +are compared diagnostically against complete preselection oracle vectors at +`1e-6`, with matching shapes and ambiguity recorded. A diagnostic cannot combine +different oracle shapes per element or substitute for the separately observed +shape-qualified replays. Timed C1 has one outstanding four-document request and +must still match its preselection result at `1e-6`. This measurement is scoped to +the observed workload and environment; it does not establish zero overhead or +support for other CrossEncoder architectures. + +After synchronization and GC, unadapted allocated GPU bytes must match B. +R's additional allocation is bounded by measured unique adapter storage bytes +rounded up to 512 bytes per storage. Final adapter removal must restore the +initial allocated bytes. Reserved allocator memory, RSS, load duration and +GPU clock/temperature/utilization are recorded separately, not conflated with +live parameter allocation. The memory reference is captured after finite +preselection base and shape-oracle warmup, before any LoRA attachment, to keep +one-time CUDA library initialization separate from adapter storage. + +These observation changes are an explicit **preallocation protocol amendment** +to the original plan of observing every timed forward. All trial orders, +warmup/request counts, phase durations, numerical tolerances and statistical +gates remain unchanged. The original timed batch-attribution claim is withdrawn; +the observed untimed replays supply that evidence instead. + +## Evidence and CPU checks + +`summary.json` retains trial outcomes, statistical gates and source hashes. +The summary also hashes the complete benchmark file manifest. Each trial +captures the full installed-distribution manifest; its digest must remain +identical across arms. An optional `--environment-receipt` attaches a separate +operator preparation receipt and its SHA-256 (never include credentials). +Each trial contains incremental `events.jsonl`, atomic `trial.json`, generated +model configuration and a separate process log. It preserves raw request +scores, ranks, IDs, usage, latencies, errors, batch membership, logits, tensor +hashes, library versions and shutdown status. Failure paths retain completed +phases and partial request samples. Hash evidence files when archiving them; +the runner records each terminal trial file hash. + +```console +mise run test -- tools/benchmarks/cross_encoder_lora/tests/test_probe.py +mise run typecheck -- tools/benchmarks/cross_encoder_lora +``` + +The CPU suite serves a tiny real BERT model through the actual loader and SDK, +uses real PEFT tensors, and checks correctness plus positive and negative +statistical/provenance controls. Its deliberately shorter phases and CPU +precision are labeled `qualification: false`; it is a harness smoke test, +not acceptance evidence for the pinned MiniLM checkpoint or L4 performance. diff --git a/tools/benchmarks/cross_encoder_lora/__init__.py b/tools/benchmarks/cross_encoder_lora/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tools/benchmarks/cross_encoder_lora/adapter.py b/tools/benchmarks/cross_encoder_lora/adapter.py new file mode 100644 index 000000000..c1f41ac84 --- /dev/null +++ b/tools/benchmarks/cross_encoder_lora/adapter.py @@ -0,0 +1,94 @@ +"""Identical benchmark observation on both main and candidate adapters.""" + +from __future__ import annotations + +import time +from typing import Any + +from sie_server.adapters.cross_encoder import CrossEncoderAdapter + + +class ObservedCrossEncoder(CrossEncoderAdapter): + def load(self, device: str) -> None: + self.batch_observations = [] + self.selection_observations = [] + self.observed_selection = None + self.observation_phase = "load" + self.capture_logits = True + self.active_forward = False + self.mutation_overlap = False + self.forward_observations = [] + self.fetch_observations = [] + super().load(device) + assert self._model is not None + self._observer_handle = self._model.model.register_forward_hook(self._observe_forward, with_kwargs=True) + + def _observe_forward(self, model: Any, _args: Any, kwargs: Any, result: Any) -> None: + enabled = [] + if self.capture_logits: + for name, module in model.named_modules(): + if hasattr(module, "lora_A") and not getattr(module, "disable_adapters", False): + enabled.append({"module": name, "adapters": list(module.active_adapters)}) + inputs = kwargs.get("input_ids") + row = { + "selection": self.observed_selection, + "enabled_adapters": enabled, + "input_shape": list(inputs.shape) if inputs is not None else None, + } + if self.capture_logits: + row["logits"] = result.logits.detach().float().cpu().tolist() + self.forward_observations.append(row) + + def set_active_lora(self, lora_name: str | None) -> None: + if self.active_forward: + self.mutation_overlap = True + super().set_active_lora(lora_name) + self.observed_selection = lora_name + self.selection_observations.append({"time_ns": time.time_ns(), "selection": lora_name}) + + def score_pairs(self, queries: list, docs: list, **kwargs: Any) -> Any: + started = time.time_ns() + first_forward = len(self.forward_observations) + self.active_forward = True + try: + result = super().score_pairs(queries, docs, **kwargs) + self.batch_observations.append( + { + "start_ns": started, + "end_ns": time.time_ns(), + "selection": self.observed_selection, + "phase": self.observation_phase, + "queries": [item.text for item in queries], + "documents": [item.text for item in docs], + "item_ids": [item.id for item in docs], + "scores": result.scores.tolist(), + "input_token_counts": result.input_token_counts, + "forwards": self.forward_observations[first_forward:], + } + ) + return result + finally: + self.active_forward = False + + def load_lora(self, *args: Any, **kwargs: Any) -> int: + if self.active_forward: + self.mutation_overlap = True + return super().load_lora(*args, **kwargs) + + def fetch_lora(self, *args: Any, **kwargs: Any) -> str: + self.fetch_observations.append( + {"time_ns": time.time_ns(), "name": args[0] if args else kwargs.get("lora_name")} + ) + return super().fetch_lora(*args, **kwargs) + + def unload_lora(self, *args: Any, **kwargs: Any) -> None: + if self.active_forward: + self.mutation_overlap = True + super().unload_lora(*args, **kwargs) + + def unload(self) -> None: + handle = getattr(self, "_observer_handle", None) + if handle is not None: + handle.remove() + self._observer_handle = None + super().unload() diff --git a/tools/benchmarks/cross_encoder_lora/evidence.py b/tools/benchmarks/cross_encoder_lora/evidence.py new file mode 100644 index 000000000..cc9e0e080 --- /dev/null +++ b/tools/benchmarks/cross_encoder_lora/evidence.py @@ -0,0 +1,63 @@ +"""Incremental observations and atomic terminal records for the bounded probe.""" + +from __future__ import annotations + +import hashlib +import json +import math +import os +import threading +import time +from pathlib import Path +from typing import Any + +from tools.benchmarks.cross_encoder_lora.protocol import require + + +def digest(path: Path) -> str: + return hashlib.sha256(path.read_bytes()).hexdigest() + + +def fingerprint(value: Any) -> str: + return hashlib.sha256(json.dumps(value, sort_keys=True).encode()).hexdigest() + + +def atomic_json(path: Path, value: Any) -> None: + temporary = path.with_suffix(path.suffix + ".pending") + with temporary.open("w") as stream: + stream.write(json.dumps(value, indent=2, allow_nan=False) + "\n") + stream.flush() + os.fsync(stream.fileno()) + temporary.replace(path) + + +def vector_delta(left: list[float], right: list[float]) -> float: + require(len(left) == len(right) and bool(left), "score vector shape changed") + require(all(math.isfinite(value) for value in left + right), "nonfinite score") + return max(abs(x - y) for x, y in zip(left, right, strict=True)) + + +class Evidence: + def __init__(self, directory: Path) -> None: + self.directory = directory + directory.mkdir(parents=True, exist_ok=False) + self.lock = threading.Lock() + + def emit(self, event: str, **fields: Any) -> dict[str, Any]: + row = {"event": event, "time_ns": time.time_ns(), **fields} + + # Preserve nonfinite observations as explicit strings in failed evidence. + def finite(value: Any) -> Any: + if isinstance(value, float) and not math.isfinite(value): + return str(value) + if isinstance(value, dict): + return {key: finite(item) for key, item in value.items()} + if isinstance(value, (list, tuple)): + return [finite(item) for item in value] + return value + + row = finite(row) + with self.lock, (self.directory / "events.jsonl").open("a") as stream: + stream.write(json.dumps(row, sort_keys=True, allow_nan=False) + "\n") + stream.flush() + return row diff --git a/tools/benchmarks/cross_encoder_lora/fixtures.py b/tools/benchmarks/cross_encoder_lora/fixtures.py new file mode 100644 index 000000000..3e6e15dba --- /dev/null +++ b/tools/benchmarks/cross_encoder_lora/fixtures.py @@ -0,0 +1,72 @@ +"""Pinned public model and deterministic, deliberately untrained LoRA fixtures.""" + +from __future__ import annotations + +import argparse +import json +from pathlib import Path + +import torch +from huggingface_hub import snapshot_download +from peft import LoraConfig +from safetensors.torch import save_file + +from tools.benchmarks.cross_encoder_lora.evidence import digest +from tools.benchmarks.cross_encoder_lora.protocol import MODEL, REVISION, WEIGHTS_SHA256, require + + +def prepare(output: Path, snapshot: Path | None = None, *, smoke: bool = False) -> dict: + if snapshot is None: + snapshot = Path( + snapshot_download( + MODEL, + revision=REVISION, + token=False, + allow_patterns=["*.json", "vocab.txt", "model.safetensors"], + ) + ) + if not smoke: + require(digest(snapshot / "model.safetensors") == WEIGHTS_SHA256, "pinned model weight hash differs") + config = json.loads((snapshot / "config.json").read_text()) + require(config["model_type"] == "bert", "the fixture protocol requires BERT") + output.mkdir(parents=True, exist_ok=False) + manifest = { + "model": MODEL, + "revision": REVISION, + "snapshot": str(snapshot.resolve()), + "smoke": smoke, + "model_files": {p.name: digest(p) for p in sorted(snapshot.iterdir()) if p.is_file()}, + "adapters": {}, + } + for index, alias in enumerate(("a", "b", "c")): + seed = 20261016 + index + directory = output / alias + directory.mkdir() + LoraConfig( + r=4, lora_alpha=8, target_modules=["query", "value"], lora_dropout=0, bias="none", inference_mode=True + ).save_pretrained(str(directory)) + generator = torch.Generator().manual_seed(seed) + weights = {} + width = config["hidden_size"] + for layer in range(config["num_hidden_layers"]): + for target in ("query", "value"): + prefix = f"base_model.model.bert.encoder.layer.{layer}.attention.self.{target}" + scale = 0.4 if smoke else 0.04 + weights[f"{prefix}.lora_A.weight"] = torch.randn(4, width, generator=generator) * scale + weights[f"{prefix}.lora_B.weight"] = torch.randn(width, 4, generator=generator) * scale + save_file(weights, directory / "adapter_model.safetensors") + manifest["adapters"][alias] = {"seed": seed, "files": {p.name: digest(p) for p in directory.iterdir()}} + (output / "manifest.json").write_text(json.dumps(manifest, indent=2) + "\n") + return manifest + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--output", type=Path, required=True) + parser.add_argument("--snapshot", type=Path) + args = parser.parse_args() + prepare(args.output, args.snapshot) + + +if __name__ == "__main__": + main() diff --git a/tools/benchmarks/cross_encoder_lora/prepare.py b/tools/benchmarks/cross_encoder_lora/prepare.py new file mode 100644 index 000000000..f38c4e143 --- /dev/null +++ b/tools/benchmarks/cross_encoder_lora/prepare.py @@ -0,0 +1,69 @@ +"""Archive immutable B/N/R sources without installing or changing dependencies.""" + +# Fixed git subcommands, argv-only execution; refs resolve before archive reads. +# ruff: noqa: S603, S607 + +from __future__ import annotations + +import argparse +import json +import subprocess +from pathlib import Path + +from tools.benchmarks.cross_encoder_lora.evidence import atomic_json, digest +from tools.benchmarks.cross_encoder_lora.protocol import BASE_REVISION, require + +SOURCE_PATHS = ( + "packages/sie_server/src", + "packages/sie_server/models", + "packages/sie_sdk/src", + "packages/sie_config/src", + "packages/sie_bundles", + "bundles", + "uv.lock", + "pyproject.toml", +) + + +def git(root: Path, *arguments: str) -> bytes: + return subprocess.check_output(["git", "-C", str(root), *arguments]) + + +def prepare(root: Path, output: Path, candidate: str, baseline: str = BASE_REVISION) -> dict: + # Resolve before collecting any bytes: moving symbolic refs are never + # reused when archiving, hashing the lock, or writing the manifest. + commits = { + "B": git(root, "rev-parse", "--verify", "--end-of-options", f"{baseline}^{{commit}}").decode().strip(), + "candidate": git(root, "rev-parse", "--verify", "--end-of-options", f"{candidate}^{{commit}}").decode().strip(), + } + require(commits["B"] == BASE_REVISION, "baseline differs from the frozen protocol") + output.mkdir(parents=True, exist_ok=False) + records = {} + locks = [] + for name, commit in commits.items(): + existing = [path for path in SOURCE_PATHS if git(root, "ls-tree", "--name-only", commit, path).strip()] + archive = output / f"{name}.tar" + with archive.open("wb") as stream: + subprocess.run( + ["git", "-C", str(root), "archive", "--format=tar", commit, *existing], stdout=stream, check=True + ) + lock = git(root, "show", f"{commit}:uv.lock") + locks.append(lock) + records[name] = {"commit": commit, "archive": archive.name, "sha256": digest(archive), "paths": existing} + require(locks[0] == locks[1], "baseline and candidate dependency locks differ") + manifest = {"schema": 1, "sources": records, "dependency_lock_identical": True} + atomic_json(output / "manifest.json", manifest) + return manifest + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--repo", type=Path, default=Path.cwd()) + parser.add_argument("--output", type=Path, required=True) + parser.add_argument("--candidate", required=True) + args = parser.parse_args() + print(json.dumps(prepare(args.repo, args.output, args.candidate), indent=2)) + + +if __name__ == "__main__": + main() diff --git a/tools/benchmarks/cross_encoder_lora/protocol.py b/tools/benchmarks/cross_encoder_lora/protocol.py new file mode 100644 index 000000000..6a40f19bc --- /dev/null +++ b/tools/benchmarks/cross_encoder_lora/protocol.py @@ -0,0 +1,102 @@ +"""Preregistered workload and paired non-inferiority gates (2026-10-16).""" + +from __future__ import annotations + +import math +import statistics +from typing import Any + +MODEL = "cross-encoder/ms-marco-MiniLM-L-6-v2" +REVISION = "c5ee24cb16019beea0893ab7796b1df96625c6b8" +WEIGHTS_SHA256 = "821d1aa69520101d6e0737f78a042ae25b19e5cb9160701909d10434f4aeb0ae" +BASE_REVISION = "d5e024ad3ba6127b1ee176e2c85c794b76077888" +QUERY = {"text": "How do I reset my password?"} +DOCUMENTS = [ + "Use the password reset link on the sign-in page to receive a recovery email.", + "The capital of France is Paris.", + "Contact account support if you cannot access your recovery email.", + "To change a billing address, open the payment settings page.", +] +ITEMS = [{"id": str(i), "text": text} for i, text in enumerate(DOCUMENTS)] +CPU_ORACLE = [3.89416241645813, -11.230953216552734, -10.699657440185547, -9.410902976989746] +ORACLE_ATOL = 0.1 # Existing finite fp32 checkpoint oracle versus the default fp16 CUDA path. +RESTORE_ATOL = 1e-6 +EFFECT_FLOOR = 1e-5 +EFFECT_DRIFT_MULTIPLIER = 5 +ORDERS = ("BNR", "NRB", "RBN", "BRN", "RNB", "NBR") +WARMUP = 30 +SEQUENTIAL = 200 +STEADY_SECONDS = 12 +CONCURRENCIES = (4, 16) +THREADS = 4 +MAX_BATCH_ITEMS = 64 +MAX_TRIAL_SECONDS = 300 +MAX_EXPERIMENT_SECONDS = 3600 +EXPECTED_VERSIONS = { + "torch": "2.9.1+cu129", + "transformers": "4.57.6", + "sentence-transformers": "5.4.1", + "peft": "0.19.1", + "safetensors": "0.7.0", + "huggingface-hub": "0.36.2", +} +T95_DF5 = 2.0150483733330233 +MAX_POINT_RATIO = 1.01 +MAX_UPPER_RATIO = 1.03 +MAX_PAIR_RATIO = 1.10 + + +def require(condition: Any, message: str) -> None: + if not condition: + raise AssertionError(message) + + +def percentile(values: list[float], fraction: float) -> float: + require(bool(values) and all(math.isfinite(x) and x > 0 for x in values), "invalid latency samples") + return sorted(values)[math.ceil(len(values) * fraction) - 1] + + +def paired_gate(ratios: list[float]) -> dict[str, Any]: + require(len(ratios) == len(ORDERS), "six fresh-process pairs are required") + require(all(math.isfinite(x) and x > 0 for x in ratios), "invalid paired ratios") + logs = [math.log(value) for value in ratios] + mean = statistics.mean(logs) + point = math.exp(mean) + upper = math.exp(mean + T95_DF5 * statistics.stdev(logs) / math.sqrt(len(logs))) + return { + "paired_ratios": ratios, + "geometric_ratio": point, + "one_sided_95_upper": upper, + "passed": point <= MAX_POINT_RATIO and upper <= MAX_UPPER_RATIO and max(ratios) <= MAX_PAIR_RATIO, + } + + +def compare(trials: list[dict[str, Any]]) -> dict[str, Any]: + require(len(trials) == len(ORDERS) * 3, "all eighteen fresh-process trials are required") + require( + [(t["pair"], t["arm"]) for t in trials] == [(i, arm) for i, order in enumerate(ORDERS) for arm in order], + "trial identities/order differ from the six preregistered triplets", + ) + require(all(trial["passed"] and trial["shutdown_complete"] for trial in trials), "a trial failed") + result = {} + for arm in ("N", "R"): + metrics = {} + for concurrency, field in ( + (1, "mean_ms"), + (1, "p95_ms"), + (4, "rps"), + (4, "p95_ms"), + (16, "rps"), + (16, "p95_ms"), + ): + ratios = [] + for pair in range(len(ORDERS)): + baseline = next(t for t in trials if t["pair"] == pair and t["arm"] == "B") + candidate = next(t for t in trials if t["pair"] == pair and t["arm"] == arm) + b = baseline["performance"][str(concurrency)][field] + c = candidate["performance"][str(concurrency)][field] + require(all(math.isfinite(value) and value > 0 for value in (b, c)), "invalid raw performance metric") + ratios.append(b / c if field == "rps" else c / b) + metrics[f"C{concurrency}_{field}"] = paired_gate(ratios) + result[arm] = {"metrics": metrics, "passed": all(value["passed"] for value in metrics.values())} + return {"arms": result, "passed": all(value["passed"] for value in result.values())} diff --git a/tools/benchmarks/cross_encoder_lora/runner.py b/tools/benchmarks/cross_encoder_lora/runner.py new file mode 100644 index 000000000..c381f71ee --- /dev/null +++ b/tools/benchmarks/cross_encoder_lora/runner.py @@ -0,0 +1,270 @@ +"""Run the frozen six-block experiment in eighteen owned fresh processes.""" + +# Only the current Python executable and this checkout's benchmark module run. +# ruff: noqa: S603 + +from __future__ import annotations + +import argparse +import json +import os +import signal +import subprocess +import sys +import tarfile +import time +from pathlib import Path +from typing import Any + +from tools.benchmarks.cross_encoder_lora.evidence import atomic_json, digest, fingerprint, vector_delta +from tools.benchmarks.cross_encoder_lora.protocol import ( + BASE_REVISION, + MAX_EXPERIMENT_SECONDS, + MAX_TRIAL_SECONDS, + ORDERS, + RESTORE_ATOL, + compare, + require, +) + +ROOT = Path(__file__).resolve().parents[3] + + +def harness_manifest() -> dict[str, Any]: + directory = Path(__file__).resolve().parent + files = { + str(path.relative_to(directory)): digest(path) + for path in sorted(directory.rglob("*")) + if path.is_file() + and path.suffix in {".py", ".md"} + and not any(part in {"_inputs", "_evidence", "__pycache__"} for part in path.relative_to(directory).parts) + } + return {"files": files, "file_manifest_sha256": fingerprint(files)} + + +def source_environment(source: Path) -> dict[str, str]: + # The harness stays fixed; both server and SDK imports come from the + # selected immutable archive. Do not run uv sync per arm. + paths = [source / f"packages/{package}/src" for package in ("sie_server", "sie_sdk", "sie_config", "sie_bundles")] + return { + **os.environ, + "PYTHONPATH": os.pathsep.join(str(path) for path in [*paths, ROOT]), + "TOKENIZERS_PARALLELISM": "false", + "SIE_BUNDLE": "default", + } + + +def extract(inputs: Path, output: Path) -> dict[str, Any]: + manifest = json.loads((inputs / "manifest.json").read_text()) + require(manifest["schema"] == 1 and manifest["dependency_lock_identical"], "unexpected source manifest") + require(manifest["sources"]["B"]["commit"] == BASE_REVISION, "baseline differs from protocol") + for name, record in manifest["sources"].items(): + require(name in {"B", "candidate"}, "unknown source arm") + path = inputs / record["archive"] + require(path.resolve().is_relative_to(inputs.resolve()), "archive path escapes inputs") + require(digest(path) == record["sha256"], "source archive hash differs") + target = output / name + target.mkdir(parents=True) + with tarfile.open(path) as archive: + archive.extractall(target, filter="data") + require( + (output / "B/uv.lock").read_bytes() == (output / "candidate/uv.lock").read_bytes(), "extracted locks differ" + ) + return manifest + + +def cross_checks(trials: list[dict[str, Any]]) -> dict[str, Any]: + results = {} + for pair in range(len(ORDERS)): + rows = {trial["arm"]: trial for trial in trials if trial["pair"] == pair} + baseline = rows["B"] + for arm in ("N", "R"): + candidate = rows[arm] + require( + candidate["environment"]["versions"] == baseline["environment"]["versions"], + "libraries changed between arms", + ) + require( + candidate["environment"]["installed_distributions_sha256"] + == baseline["environment"]["installed_distributions_sha256"], + "installed distributions changed between arms", + ) + require(candidate["environment"]["device"] == baseline["environment"]["device"], "GPU changed between arms") + require( + vector_delta(candidate["base_scores"], baseline["base_scores"]) <= RESTORE_ATOL, + "cross-process C1 base scores differ", + ) + require( + candidate["initial_memory"]["allocated"] == baseline["initial_memory"]["allocated"], + "unadapted allocated bytes changed", + ) + for key, oracle in candidate["shape_oracles"].items(): + require( + vector_delta(oracle["scores"], baseline["shape_oracles"][key]["scores"]) <= RESTORE_ATOL, + "cross-process shape oracle differs", + ) + # Actual shape attribution is available only in observed untimed + # replays. Timed C4/C16 diagnostics are not shape-qualified evidence. + for phase in candidate["untimed_replays"].values(): + for batch in phase["batches"]: + key = fingerprint({"queries": batch["queries"], "documents": batch["documents"]}) + require( + vector_delta(batch["scores"], baseline["shape_oracles"][key]["scores"]) <= RESTORE_ATOL, + "candidate untimed SDK replay differs from the matched B shape", + ) + delta = candidate["resident_memory"]["allocated"] - baseline["resident_memory"]["allocated"] + allowance = candidate.get("adapter_storage", {}).get("rounded_storage_bytes", 0) + require(0 <= delta <= allowance, "resident allocated bytes exceed measured adapter storage allowance") + results[f"{pair}-{arm}"] = {"allocated_delta": delta, "allowance": allowance, "passed": True} + return results + + +def stop(process: subprocess.Popen) -> None: + if process.poll() is None: + process.send_signal(signal.SIGTERM) + try: + process.wait(timeout=35) + except subprocess.TimeoutExpired: + process.kill() + process.wait(timeout=10) + + +def run(args: argparse.Namespace) -> dict[str, Any]: + args.output.mkdir(parents=True, exist_ok=False) + manifest = extract(args.inputs.resolve(), args.output / "sources") + summary: dict[str, Any] = { + "schema": 1, + "passed": False, + "source_manifest": manifest, + "harness_manifest": harness_manifest(), + "smoke": args.smoke, + "orders": ORDERS, + "trials": [], + } + summary_path = args.output / "summary.json" + absolute_deadline = time.monotonic() + args.deadline_s + if args.environment_receipt is not None: + summary["environment_preparation"] = { + "sha256": digest(args.environment_receipt), + "receipt": json.loads(args.environment_receipt.read_text()), + } + try: + schedule = [(0, "R")] if args.smoke else [(pair, arm) for pair, order in enumerate(ORDERS) for arm in order] + for pair, arm in schedule: + require( + time.monotonic() + args.trial_deadline_s + 45 < absolute_deadline, + "insufficient time for a complete bounded next trial and cleanup", + ) + directory = args.output / f"trial-{pair}-{arm}" + source = (args.output / "sources" / ("B" if arm == "B" else "candidate")).resolve() + command = [ + sys.executable, + "-m", + "tools.benchmarks.cross_encoder_lora.worker", + "--source", + str(source), + "--fixtures", + str(args.fixtures.resolve()), + "--output", + str(directory.resolve()), + "--arm", + arm, + "--pair", + str(pair), + "--deadline-s", + str(args.trial_deadline_s), + ] + if args.snapshot: + command.extend(["--snapshot", str(args.snapshot.resolve())]) + if args.smoke: + command.append("--smoke") + print(f"Starting block {pair + 1}, arm {arm}", flush=True) + process = None + pending_error: BaseException | None = None + returncode = None + with (args.output / f"trial-{pair}-{arm}.log").open("w") as log: + try: + process = subprocess.Popen( + command, env=source_environment(source), cwd=source, stdout=log, stderr=subprocess.STDOUT + ) + returncode = process.wait(timeout=args.trial_deadline_s + 45) + except BaseException as error: # noqa: BLE001 -- persist the trial, then re-raise after owned cleanup + pending_error = error + finally: + if process is not None: + stop(process) + returncode = process.returncode + trial: dict[str, Any] + if (directory / "trial.json").is_file(): + trial = json.loads((directory / "trial.json").read_text()) + else: + trial = { + "pair": pair, + "arm": arm, + "passed": False, + "shutdown_complete": False, + "error": "worker produced no terminal record", + } + trial["process_returncode"] = returncode + trial["source_commit"] = manifest["sources"]["B" if arm == "B" else "candidate"]["commit"] + trial["terminal_record_sha256"] = ( + digest(directory / "trial.json") if (directory / "trial.json").exists() else None + ) + if pending_error is not None: + trial["passed"] = False + trial["runner_error"] = {"type": type(pending_error).__name__, "message": str(pending_error)} + summary["trials"].append(trial) + atomic_json(summary_path, summary) + if pending_error is not None: + raise pending_error + require( + returncode == 0 and trial["passed"] and trial["shutdown_complete"], + "trial failed; raw evidence retained", + ) + print(f"Finished block {pair + 1}, arm {arm}", flush=True) + if args.smoke: + summary["passed"] = True + summary["qualification"] = False + else: + summary["correctness_memory"] = cross_checks(summary["trials"]) + summary["statistics"] = compare(summary["trials"]) + summary["passed"] = summary["statistics"]["passed"] + require(summary["passed"], "fixed non-inferiority gates did not pass") + except BaseException as error: + summary["error"] = {"type": type(error).__name__, "message": str(error)} + raise + finally: + atomic_json(summary_path, summary) + return summary + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--inputs", type=Path, required=True) + parser.add_argument("--fixtures", type=Path, required=True) + parser.add_argument("--snapshot", type=Path) + parser.add_argument( + "--environment-receipt", + type=Path, + help="Optional operator-owned JSON receipt of environment preparation (no secrets)", + ) + parser.add_argument("--output", type=Path, required=True) + parser.add_argument("--deadline-s", type=float, default=2400) + parser.add_argument("--trial-deadline-s", type=float, default=120) + parser.add_argument("--smoke", action="store_true") + args = parser.parse_args() + require( + 0 < args.deadline_s <= MAX_EXPERIMENT_SECONDS and 0 < args.trial_deadline_s <= MAX_TRIAL_SECONDS, + "deadline outside bounded protocol", + ) + signal.signal(signal.SIGTERM, terminate) + run(args) + + +def terminate(signum: int, _frame: Any) -> None: + # Enter the ordinary owned-child cleanup and atomic evidence finally paths. + raise SystemExit(128 + signum) + + +if __name__ == "__main__": + main() diff --git a/tools/benchmarks/cross_encoder_lora/tests/__init__.py b/tools/benchmarks/cross_encoder_lora/tests/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tools/benchmarks/cross_encoder_lora/tests/conftest.py b/tools/benchmarks/cross_encoder_lora/tests/conftest.py new file mode 100644 index 000000000..08a014d75 --- /dev/null +++ b/tools/benchmarks/cross_encoder_lora/tests/conftest.py @@ -0,0 +1,6 @@ +"""Make the public source-only tools namespace importable under pytest's importlib mode.""" + +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parents[4])) diff --git a/tools/benchmarks/cross_encoder_lora/tests/test_probe.py b/tools/benchmarks/cross_encoder_lora/tests/test_probe.py new file mode 100644 index 000000000..eb820a77a --- /dev/null +++ b/tools/benchmarks/cross_encoder_lora/tests/test_probe.py @@ -0,0 +1,334 @@ +"""Real CPU smoke plus negative controls for the public acceptance harness.""" + +# Owned Python subprocesses run only this checkout's benchmark, without a shell. +# ruff: noqa: S603 + +from __future__ import annotations + +import json +import subprocess +import sys +import tarfile +from pathlib import Path + +import httpx +import pytest +import torch +from transformers import BertConfig, BertForSequenceClassification, BertTokenizerFast + +from tools.benchmarks.cross_encoder_lora import prepare as source_preparer +from tools.benchmarks.cross_encoder_lora.evidence import Evidence, atomic_json, vector_delta +from tools.benchmarks.cross_encoder_lora.fixtures import prepare +from tools.benchmarks.cross_encoder_lora.protocol import BASE_REVISION, ORDERS, compare, paired_gate, percentile +from tools.benchmarks.cross_encoder_lora.runner import ROOT, extract, source_environment +from tools.benchmarks.cross_encoder_lora.worker import ORDER, describe_response, matching_vectors, validate_response + + +def tiny_snapshot(directory: Path) -> Path: + directory.mkdir() + vocabulary = [ + "[PAD]", + "[UNK]", + "[CLS]", + "[SEP]", + "[MASK]", + "the", + "password", + "reset", + "link", + "email", + "paris", + "account", + "support", + "billing", + "address", + ".", + "?", + "how", + "do", + "i", + "my", + "use", + "to", + "a", + "is", + "of", + "france", + ] + (directory / "vocab.txt").write_text("\n".join(vocabulary) + "\n") + tokenizer = BertTokenizerFast(vocab_file=str(directory / "vocab.txt")) + tokenizer.save_pretrained(directory) + config = BertConfig( + vocab_size=len(vocabulary), + hidden_size=16, + num_hidden_layers=2, + num_attention_heads=2, + intermediate_size=32, + max_position_embeddings=64, + num_labels=1, + hidden_dropout_prob=0, + attention_probs_dropout_prob=0, + ) + config.sbert_ce_default_activation_function = "torch.nn.modules.linear.Identity" + torch.manual_seed(29) + BertForSequenceClassification(config).save_pretrained(directory) + return directory + + +def test_gate_positive_negative_and_uncertainty() -> None: + assert paired_gate([1.0] * 6)["passed"] + assert not paired_gate([1.02] * 6)["passed"] # point gate + uncertain = paired_gate([0.94, 0.94, 0.94, 1.06, 1.06, 1.06]) + assert uncertain["geometric_ratio"] < 1.01 + assert uncertain["one_sided_95_upper"] > 1.03 + assert not uncertain["passed"] + assert not paired_gate([0.97] * 5 + [1.11])["passed"] + with pytest.raises(AssertionError, match="six"): + paired_gate([1.0] * 5) + with pytest.raises(AssertionError, match="invalid"): + paired_gate([float("nan")] * 6) + + +def test_nearest_rank() -> None: + assert percentile(list(range(1, 21)), 0.95) == 19 + assert percentile([1, 2, 3], 0.95) == 3 + + +def trials() -> list[dict]: + performance = {str(c): {"mean_ms": 10.0, "p95_ms": 12.0, "rps": 100.0} for c in (1, 4, 16)} + return [ + {"pair": i, "arm": arm, "passed": True, "shutdown_complete": True, "performance": performance} + for i, order in enumerate(ORDERS) + for arm in order + ] + + +def test_full_schedule_and_cleanup_are_required() -> None: + assert compare(trials())["passed"] + wrong = trials() + wrong[0]["arm"] = "R" + with pytest.raises(AssertionError, match="identities"): + compare(wrong) + incomplete = trials() + incomplete[0]["shutdown_complete"] = False + with pytest.raises(AssertionError, match="failed"): + compare(incomplete) + + +@pytest.mark.parametrize("value", [-1.0, 0.0, float("nan"), float("inf")]) +def test_raw_metrics_must_be_positive_finite(value: float) -> None: + invalid = trials() + for trial in invalid: + trial["performance"]["1"]["mean_ms"] = value + with pytest.raises(AssertionError, match="raw performance"): + compare(invalid) + + +def test_nonfinite_evidence_retained_but_never_accepted(tmp_path: Path) -> None: + evidence = Evidence(tmp_path / "out") + evidence.emit("bad_output", values=[float("nan"), float("inf")]) + assert json.loads((tmp_path / "out/events.jsonl").read_text())["values"] == ["nan", "inf"] + with pytest.raises(AssertionError, match="nonfinite"): + vector_delta([float("nan")], [1.0]) + atomic_json(tmp_path / "out/final.json", {"passed": False}) + assert json.loads((tmp_path / "out/final.json").read_text()) == {"passed": False} + + +@pytest.mark.parametrize("arm", ["B", "R"]) +def test_real_loader_sdk_lifecycle(tmp_path: Path, arm: str) -> None: + snapshot = tiny_snapshot(tmp_path / "model") + fixtures = tmp_path / "fixtures" + prepare(fixtures, snapshot, smoke=True) + # Use real historical B and committed candidate imports, not the editable + # checkout accidentally standing in for both arms. Full clones contain B; + # a shallow CI checkout can still run the other deterministic controls. + available = subprocess.run(["git", "cat-file", "-e", BASE_REVISION], cwd=ROOT, capture_output=True, check=False) # noqa: S607 + if available.returncode: + pytest.skip("immutable archive smoke requires the documented baseline commit in git history") + inputs = tmp_path / "source-inputs" + source_preparer.prepare(ROOT, inputs, "HEAD") + sources = tmp_path / "sources" + extract(inputs, sources) + source = sources / ("B" if arm == "B" else "candidate") + output = tmp_path / "evidence" + command = [ + sys.executable, + "-m", + "tools.benchmarks.cross_encoder_lora.worker", + "--source", + str(source), + "--fixtures", + str(fixtures), + "--output", + str(output), + "--arm", + arm, + "--smoke", + "--deadline-s", + "90", + ] + result = subprocess.run( + command, env=source_environment(source), cwd=source, capture_output=True, text=True, timeout=130, check=False + ) + assert result.returncode == 0, result.stdout + result.stderr + summary = json.loads((output / "trial.json").read_text()) + assert summary["passed"] + assert summary["shutdown_complete"] + assert summary["manager_present"] == (arm == "R") + assert summary["environment"]["qualification"] is False + assert Path(summary["environment"]["adapter_source"]).is_relative_to(source) + events = [json.loads(line) for line in (output / "events.jsonl").read_text().splitlines()] + assert {"shape_oracles", "performance", "finished"} <= {event["event"] for event in events} + if arm == "R": + assert {"effects", "concurrent_batches", "resident_set", "restored_state"} <= { + event["event"] for event in events + } + + +def test_worker_failure_persists_terminal_record(tmp_path: Path) -> None: + output = tmp_path / "failed" + command = [ + sys.executable, + "-m", + "tools.benchmarks.cross_encoder_lora.worker", + "--source", + str(ROOT), + "--fixtures", + str(tmp_path / "missing"), + "--output", + str(output), + "--arm", + "R", + "--smoke", + ] + result = subprocess.run( + command, env=source_environment(ROOT), cwd=ROOT, capture_output=True, text=True, timeout=45, check=False + ) + assert result.returncode != 0 + summary = json.loads((output / "trial.json").read_text()) + assert not summary["passed"] + assert summary["error"]["type"] == "FileNotFoundError" + assert (output / "events.jsonl").read_text().endswith("\n") + + +def test_source_archive_resolves_moving_refs_once(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + root = tmp_path / "repo" + root.mkdir() + + def git(*args: str) -> bytes: + return subprocess.check_output(["git", "-C", str(root), *args]) # noqa: S607 + + git("init", "-q") + source = root / "packages/sie_server/src" + source.mkdir(parents=True) + (source / "value.txt").write_text("baseline") + (root / "uv.lock").write_text("fixed lock") + git("add", ".") + + def commit() -> None: + git("-c", "user.name=Harness Test", "-c", "user.email=harness@example.invalid", "commit", "-qm", "fixture") + + commit() + baseline = git("rev-parse", "HEAD").decode().strip() + git("checkout", "-qb", "candidate") + (source / "value.txt").write_text("candidate before move") + git("add", ".") + commit() + candidate = git("rev-parse", "HEAD").decode().strip() + original = source_preparer.git + + def moving_git(path: Path, *arguments: str) -> bytes: + value = original(path, *arguments) + if arguments[-1] == "candidate^{commit}": + (source / "value.txt").write_text("candidate after move") + git("add", ".") + commit() + return value + + monkeypatch.setattr(source_preparer, "BASE_REVISION", baseline) + monkeypatch.setattr(source_preparer, "git", moving_git) + output = tmp_path / "archives" + manifest = source_preparer.prepare(root, output, "candidate", baseline) + assert manifest["sources"]["candidate"]["commit"] == candidate + assert git("rev-parse", "candidate").decode().strip() != candidate + with tarfile.open(output / "candidate.tar") as archive: + stream = archive.extractfile("packages/sie_server/src/value.txt") + assert stream is not None + assert stream.read() == b"candidate before move" + + +def test_tampered_archive_refused_before_extract(tmp_path: Path) -> None: + inputs = tmp_path / "input" + inputs.mkdir() + (inputs / "B.tar").write_bytes(b"not the declared archive") + atomic_json( + inputs / "manifest.json", + { + "schema": 1, + "dependency_lock_identical": True, + "sources": {"B": {"commit": BASE_REVISION, "archive": "B.tar", "sha256": "0" * 64}}, + }, + ) + with pytest.raises(AssertionError, match="archive hash"): + extract(inputs, tmp_path / "output") + + +def test_timed_diagnostics_require_one_complete_oracle_vector() -> None: + first = [0.0, 1.0, 2.0, 3.0] + second = [10.0, 11.0, 12.0, 13.0] + oracles = { + "first": {"copies": 1, "scores": [first[index] for index in ORDER]}, + "second": {"copies": 2, "scores": [second[index] for index in ORDER for _ in range(2)]}, + } + assert matching_vectors(first, oracles) == [{"workload_sha256": "first", "copies": 1, "request_position": 0}] + assert matching_vectors([0.0, 11.0, 2.0, 13.0], oracles) == [] + + +def test_hidden_retry_rejected_and_malformed_error_retained() -> None: + row = { + "label": "test", + "responses": [httpx.Response(200), httpx.Response(200)], + "result": {"scores": [{"item_id": f"test:{i}", "score": float(4 - i), "rank": i} for i in range(4)]}, + } + with pytest.raises(AssertionError, match="hidden SDK retry"): + validate_response(row) + bad = describe_response(httpx.Response(503, content=b"not JSON")) + assert bad["body_decode_error"] == "JSONDecodeError" + assert bad["status"] == 503 + + +def test_live_server_failure_and_failed_evidence_emit_still_cleanup(tmp_path: Path) -> None: + snapshot = tiny_snapshot(tmp_path / "model") + fixtures = tmp_path / "fixtures" + prepare(fixtures, snapshot, smoke=True) + output = tmp_path / "failed-live" + script = """ +import argparse, asyncio, sys +from pathlib import Path +from tools.benchmarks.cross_encoder_lora import worker +original = worker.Evidence.emit +def failing_emit(self, event, **fields): + if event == 'original_state': + raise OSError('injected post-load evidence failure') + if event == 'client_wire_responses': + raise OSError('injected cleanup evidence failure') + return original(self, event, **fields) +worker.Evidence.emit = failing_emit +asyncio.run(worker.run(argparse.Namespace(source=Path(sys.argv[1]), fixtures=Path(sys.argv[2]), + output=Path(sys.argv[3]), snapshot=None, arm='R', pair=0, smoke=True, deadline_s=60))) +""" + result = subprocess.run( + [sys.executable, "-c", script, str(ROOT), str(fixtures), str(output)], + env=source_environment(ROOT), + cwd=ROOT, + capture_output=True, + text=True, + timeout=100, + check=False, + ) + assert result.returncode != 0 + summary = json.loads((output / "trial.json").read_text()) + assert not summary["passed"] + assert summary["shutdown_complete"] + assert summary["error"]["message"] == "injected cleanup evidence failure" + assert {row["message"] for row in summary["error"]["context"]} == {"injected post-load evidence failure"} diff --git a/tools/benchmarks/cross_encoder_lora/worker.py b/tools/benchmarks/cross_encoder_lora/worker.py new file mode 100644 index 000000000..b8dcd9c8a --- /dev/null +++ b/tools/benchmarks/cross_encoder_lora/worker.py @@ -0,0 +1,867 @@ +"""One fresh-process SDK/worker trial; invoked by runner.py, never provisions compute.""" + +from __future__ import annotations + +import argparse +import asyncio +import gc +import hashlib +import importlib.metadata +import json +import math +import os +import resource +import socket +import statistics +import subprocess +import sys +import threading +import time +import weakref +from collections.abc import Iterator +from concurrent.futures import ThreadPoolExecutor +from contextlib import ExitStack, contextmanager +from http import HTTPStatus +from pathlib import Path +from typing import Any + +import httpx +import torch +import uvicorn +import yaml +from sie_sdk import SIEClient +from sie_sdk._msgpack import unpackb +from sie_sdk.client.errors import RequestError, ServerError +from sie_server.adapters.cross_encoder import CrossEncoderAdapter +from sie_server.adapters.peft_lora_mixin import PEFTLoRAMixin +from sie_server.app.app_factory import AppFactory +from sie_server.app.app_state_config import AppStateConfig +from sie_server.types.inputs import Item + +from tools.benchmarks.cross_encoder_lora.evidence import Evidence, atomic_json, digest, fingerprint, vector_delta +from tools.benchmarks.cross_encoder_lora.protocol import ( + CONCURRENCIES, + CPU_ORACLE, + DOCUMENTS, + EFFECT_DRIFT_MULTIPLIER, + EFFECT_FLOOR, + EXPECTED_VERSIONS, + MAX_BATCH_ITEMS, + MAX_TRIAL_SECONDS, + MODEL, + ORACLE_ATOL, + QUERY, + RESTORE_ATOL, + SEQUENTIAL, + STEADY_SECONDS, + THREADS, + WARMUP, + WEIGHTS_SHA256, + percentile, + require, +) + +HERE = Path(__file__).resolve().parent +ORDER = sorted(range(len(DOCUMENTS)), key=lambda index: len(DOCUMENTS[index])) +MAX_LOADING_RETRIES = 10 +MAX_LOADING_DELAY = 5 +RESIDENT_CAPACITY = 2 +REQUEST_ITEMS = len(DOCUMENTS) + + +def error_record(error: BaseException) -> dict[str, Any]: + record: dict[str, Any] = {"type": type(error).__name__, "message": str(error), "context": []} + seen = {id(error)} + cause = error.__cause__ or error.__context__ + while cause is not None and id(cause) not in seen: + seen.add(id(cause)) + record["context"].append({"type": type(cause).__name__, "message": str(cause)}) + cause = cause.__cause__ or cause.__context__ + return record + + +def ordered_workload(copies: int) -> tuple[list[Item], list[Item]]: + indices = [index for index in ORDER for _ in range(copies)] + return ( + [Item(text=QUERY["text"]) for _ in indices], + [Item(id=f"oracle:{j}", text=DOCUMENTS[index]) for j, index in enumerate(indices)], + ) + + +def batch_key(row: dict[str, Any]) -> str: + require(len(row["documents"]) % 4 == 0, "unexpected partial four-document request batch") + copies = len(row["documents"]) // 4 + require(1 <= copies <= MAX_BATCH_ITEMS // 4, "unexpected batch size") + queries, documents = ordered_workload(copies) + workload = {"queries": [item.text for item in queries], "documents": [item.text for item in documents]} + require( + row["queries"] == workload["queries"] and row["documents"] == workload["documents"], + "batch order/workload differs from its preregistered oracle", + ) + return fingerprint(workload) + + +def memory() -> dict[str, Any]: + gc.collect() + if torch.cuda.is_available(): + torch.cuda.synchronize() + return { + "allocated": torch.cuda.memory_allocated() if torch.cuda.is_available() else 0, + "reserved": torch.cuda.memory_reserved() if torch.cuda.is_available() else 0, + "max_rss_platform_units": resource.getrusage(resource.RUSAGE_SELF).ru_maxrss, + } + + +def telemetry() -> dict[str, Any]: + if not torch.cuda.is_available(): + return {"device": "cpu", "qualification": False} + command = [ + "nvidia-smi", + "--query-gpu=name,uuid,driver_version,temperature.gpu,clocks.sm,clocks.mem,power.draw,utilization.gpu,memory.used", + "--format=csv,noheader,nounits", + ] + result = subprocess.run(command, capture_output=True, text=True, timeout=10, check=False) # noqa: S603 -- fixed nvidia-smi argv + return { + "query": command[1], + "returncode": result.returncode, + "csv": result.stdout.strip(), + "stderr": result.stderr.strip(), + } + + +def tensor_state(adapter: Any) -> dict[str, Any]: + outer = adapter._model + model = outer.model + tensors = {} + for name, tensor in model.named_parameters(): + if ".lora_" not in name: + value = tensor.detach().cpu().contiguous() + tensors[str(id(tensor))] = { + "name": name.replace(".base_layer.", "."), + "dtype": str(tensor.dtype), + "shape": list(tensor.shape), + "sha256": hashlib.sha256(value.view(torch.uint8).numpy().tobytes()).hexdigest(), + } + return { + "wrapper_id": id(outer), + "model_id": id(model), + "tokenizer_id": id(outer.tokenizer), + "head_id": id(model.classifier), + "activation": str(outer.activation_fn), + "tensors": tensors, + "tree": { + name: [type(module).__module__ + "." + type(module).__qualname__, id(module)] + for name, module in model.named_modules() + }, + "hf_peft_config_loaded": bool(getattr(model, "_hf_peft_config_loaded", False)), + "peft_config_present": bool(getattr(model, "peft_config", None)), + } + + +def base_unchanged(before: dict[str, Any], after: dict[str, Any], *, restored: bool = False) -> None: + for key in ("wrapper_id", "model_id", "tokenizer_id", "head_id", "activation", "tensors"): + require(before[key] == after[key], f"base state changed: {key}") + if restored: + for key in ("tree", "hf_peft_config_loaded", "peft_config_present"): + require(before[key] == after[key], f"final unload did not restore {key}") + + +def adapter_storage(adapter: Any) -> dict[str, int]: + storages = {} + for name, tensor in adapter._model.model.named_parameters(): + if ".lora_" in name: + storage = tensor.untyped_storage() + storages[(str(tensor.device), storage.data_ptr())] = storage.nbytes() + return { + "storage_count": len(storages), + "tensor_storage_bytes": sum(storages.values()), + "rounded_storage_bytes": sum(math.ceil(size / 512) * 512 for size in storages.values()), + } + + +class Client: + def __init__(self, url: str) -> None: + self.responses: list[httpx.Response] = [] + http = httpx.Client(base_url=url, timeout=30, event_hooks={"response": [self.observe]}) + self.sdk = SIEClient(url, timeout_s=30, http_client=http) + + def observe(self, response: httpx.Response) -> None: + # The only extra HTTP-hook work in timed phases. Decode after clocks stop. + self.responses.append(response) + + def score(self, selection: str | None, label: str, *, cold: bool = False, timed: bool = False) -> dict[str, Any]: + started = time.perf_counter_ns() + wall_started = time.time_ns() + first = len(self.responses) + # score() currently does not retry LORA_LOADING. These are explicitly + # fresh SDK calls for this pre-execution status, never transport retries. + for attempt in range(MAX_LOADING_RETRIES + 1): + try: + result = self.sdk.score( + MODEL, + {"text": QUERY["text"]}, + [{"id": f"{label}:{index}", "text": text} for index, text in enumerate(DOCUMENTS)], + options={"lora": selection}, + wait_for_capacity=False, + max_oom_retries=0, + provision_timeout_s=30, + ) + break + except ServerError as error: + if ( + not cold + or error.status_code != HTTPStatus.SERVICE_UNAVAILABLE + or error.code != "LORA_LOADING" + or attempt == MAX_LOADING_RETRIES + ): + raise + delay = error.retry_after if error.retry_after is not None else 0.2 + require(0 <= delay <= MAX_LOADING_DELAY, "unexpected loading retry delay") + time.sleep(delay) + elapsed = (time.perf_counter_ns() - started) / 1e6 + row = { + "label": label, + "selection": selection, + "latency_ms": elapsed, + "start_ns": wall_started, + "end_ns": time.time_ns(), + "result": result, + "responses": self.responses[first:], + } + if not timed: + validate_response(row, allow_loading=cold) + return row + + def close(self) -> None: + self.sdk.close() + + +def describe_response(response: httpx.Response) -> dict[str, Any]: + row: dict[str, Any] = { + "status": response.status_code, + "timing_headers": { + name: response.headers[name] + for name in ( + "x-queue-time", + "x-tokenization-time", + "x-inference-time", + "x-postprocessing-time", + "x-total-time", + ) + if name in response.headers + }, + } + if response.status_code >= HTTPStatus.BAD_REQUEST: + response.read() + try: + body = ( + unpackb(response.content, numeric_arrays=False) + if "msgpack" in response.headers.get("content-type", "") + else response.json() + ) + except ValueError as error: + row["body_decode_error"] = type(error).__name__ + body = None + error = body.get("error", body.get("detail", body)) if isinstance(body, dict) else None + row.update( + code=error.get("code") if isinstance(error, dict) else None, retry_after=response.headers.get("retry-after") + ) + return row + + +def serialize_responses(row: dict[str, Any]) -> None: + row["responses"] = [ + describe_response(value) if isinstance(value, httpx.Response) else value for value in row["responses"] + ] + + +def validate_response(row: dict[str, Any], *, allow_loading: bool = False) -> None: + serialize_responses(row) + statuses = row["responses"] + require(statuses and statuses[-1]["status"] == HTTPStatus.OK, "missing successful HTTP response") + if allow_loading: + require( + all( + value["status"] == HTTPStatus.SERVICE_UNAVAILABLE and value.get("code") == "LORA_LOADING" + for value in statuses[:-1] + ), + "unexpected retry during cold adapter request", + ) + else: + require(len(statuses) == 1, "hidden SDK retry disqualifies this request") + scores = row["result"]["scores"] + expected_ids = [f"{row['label']}:{i}" for i in range(REQUEST_ITEMS)] + by_id = {item["item_id"]: item["score"] for item in scores} + require(len(scores) == REQUEST_ITEMS and set(by_id) == set(expected_ids), "score result lost or changed item IDs") + require([item["rank"] for item in scores] == list(range(REQUEST_ITEMS)), "score ranks changed") + values = [by_id[key] for key in expected_ids] + row["scores"] = values + require(all(math.isfinite(value) for value in values), "nonfinite SDK score") + require([item["score"] for item in scores] == sorted(values, reverse=True), "score ranks are unordered") + + +@contextmanager +def unobserved(adapter: Any) -> Iterator[None]: + """Time production methods directly; no model hook, copying or observer lock.""" + score_pairs, select = adapter.score_pairs, adapter.set_active_lora + before = (len(adapter.batch_observations), len(adapter.forward_observations), len(adapter.selection_observations)) + adapter._observer_handle.remove() + adapter._observer_handle = None + adapter.score_pairs = CrossEncoderAdapter.score_pairs.__get__(adapter) + adapter.set_active_lora = CrossEncoderAdapter.set_active_lora.__get__(adapter) + try: + require(adapter.score_pairs.__func__ is CrossEncoderAdapter.score_pairs, "timed scoring is instrumented") + require( + adapter.set_active_lora.__func__ is CrossEncoderAdapter.set_active_lora, "timed selection is instrumented" + ) + yield + after = ( + len(adapter.batch_observations), + len(adapter.forward_observations), + len(adapter.selection_observations), + ) + require(before == after, "observer activity occurred during production timing") + finally: + adapter.score_pairs, adapter.set_active_lora = score_pairs, select + adapter._observer_handle = adapter._model.model.register_forward_hook( + adapter._observe_forward, with_kwargs=True + ) + + +def matching_vectors(scores: list[float], oracles: dict[str, Any]) -> list[dict[str, Any]]: + matches = [] + for key, oracle_row in oracles.items(): + copies = oracle_row["copies"] + for position in range(copies): + vector = [oracle_row["scores"][ORDER.index(index) * copies + position] for index in range(REQUEST_ITEMS)] + if vector_delta(scores, vector) <= RESTORE_ATOL: + matches.append({"workload_sha256": key, "copies": copies, "request_position": position}) + return matches + + +def performance(url: str, concurrency: int, evidence: Evidence, *, smoke: bool) -> dict[str, Any]: + clients = [Client(url) for _ in range(concurrency)] + samples: list[dict[str, Any]] = [] + warmups: list[dict[str, Any]] = [] + lock = threading.Lock() + try: + for i in range(2 if smoke else WARMUP): + warmups.append(clients[0].score(None, f"warm-C{concurrency}-{i}", timed=True)) + barrier = threading.Barrier(concurrency + 1) + clock: dict[str, float] = {} + + def lane(index: int) -> None: + barrier.wait(timeout=30) + counter = 0 + while ( + counter < (4 if smoke else SEQUENTIAL) if concurrency == 1 else time.perf_counter() < clock["deadline"] + ): + row = clients[index].score(None, f"C{concurrency}-lane{index}-{counter}", timed=True) + with lock: + samples.append(row) + counter += 1 + + with ThreadPoolExecutor(max_workers=concurrency) as executor: + futures = [executor.submit(lane, i) for i in range(concurrency)] + clock["start"] = time.perf_counter() + clock["deadline"] = clock["start"] + (0.15 if smoke else STEADY_SECONDS) + barrier.wait(timeout=30) + for future in futures: + future.result(timeout=90) + elapsed = time.perf_counter() - clock["start"] + for row in [*warmups, *samples]: + validate_response(row) + latencies = [row["latency_ms"] for row in samples] + return { + "concurrency": concurrency, + "count": len(samples), + "elapsed_s": elapsed, + "mean_ms": statistics.mean(latencies), + "p95_ms": percentile(latencies, 0.95), + "rps": len(samples) / elapsed, + "samples": samples, + } + finally: + with ExitStack() as cleanup: + for client in clients: + cleanup.callback(client.close) + for row in [*warmups, *samples]: + serialize_responses(row) + evidence.emit( + "performance_attempt", + concurrency=concurrency, + completed_samples=samples, + warmups=warmups, + wire_responses=[describe_response(response) for client in clients for response in client.responses], + ) + + +def validate_usage(rows: list[dict[str, Any]], batches: list[dict[str, Any]]) -> None: + tokens = { + item_id: count + for batch in batches + for item_id, count in zip(batch["item_ids"], batch["input_token_counts"], strict=True) + } + for row in rows: + expected = sum(tokens[f"{row['label']}:{i}"] for i in range(4)) + require( + row["result"]["usage"]["input_tokens"] == expected, "SDK token usage differs from real tokenizer counts" + ) + + +def replay(url: str, concurrency: int) -> list[dict[str, Any]]: + clients = [Client(url) for _ in range(concurrency)] + rows = [] + try: + for repeat in range(2): + barrier = threading.Barrier(concurrency) + + def lane(index: int) -> dict[str, Any]: + barrier.wait(timeout=20) + return clients[index].score(None, f"replay-C{concurrency}-{repeat}-{index}") + + with ThreadPoolExecutor(max_workers=concurrency) as executor: + rows.extend(executor.map(lane, range(concurrency))) + return rows + finally: + for client in clients: + client.close() + + +async def oracle(adapter: Any, worker: Any, selection: str | None, *, smoke: bool) -> dict[str, Any]: + result = {} + for copies in range(1, MAX_BATCH_ITEMS // 4 + 1): + queries, documents = ordered_workload(copies) + + def forward() -> Any: + adapter.set_active_lora(selection) + return adapter.score_pairs(queries, documents) + + values = [] + for _ in range(2): + output = await worker.run_adapter_mutation(forward) + values.append(output.scores.tolist()) + require(vector_delta(*values) <= RESTORE_ATOL, "preselection shape oracle is unstable") + key = fingerprint({"queries": [item.text for item in queries], "documents": [item.text for item in documents]}) + result[key] = { + "copies": copies, + "scores": values[0], + "repeat": values[1], + "input_token_counts": output.input_token_counts, + } + return result + + +def configs(directory: Path, source: Path, fixtures: Path, arm: str, snapshot: Path, *, smoke: bool) -> None: + directory.mkdir() + config = yaml.safe_load( + (source / "packages/sie_server/models/cross-encoder__ms-marco-MiniLM-L-6-v2.yaml").read_text() + ) + config.pop("hf_id") + config["weights_path"] = str(snapshot) + if smoke: + config["max_sequence_length"] = 64 + profile = config["profiles"]["default"] + profile["adapter_path"] = f"{HERE / 'adapter.py'}:ObservedCrossEncoder" + if arm == "R": + profile["adapter_options"]["loadtime"] = { + "lora_paths": {name: str(fixtures / name) for name in ("a", "b", "c")} + } + (directory / "model.yaml").write_text(yaml.safe_dump(config)) + + +def preflight(args: argparse.Namespace, evidence: Evidence) -> dict[str, Any]: + versions = {name: importlib.metadata.version(name) for name in EXPECTED_VERSIONS} + try: + torchao = importlib.metadata.version("torchao") + except importlib.metadata.PackageNotFoundError: + torchao = None + module_path = sys.modules[CrossEncoderAdapter.__module__].__file__ + assert module_path is not None + actual_source = Path(module_path).resolve() + require(actual_source.is_relative_to(args.source.resolve()), "adapter was imported from a different source tree") + data = { + "versions": versions, + "torchao": torchao, + "adapter_source": str(actual_source), + "adapter_sha256": digest(actual_source), + "python": sys.version, + "threads": torch.get_num_threads(), + "device": str(torch.cuda.get_device_name(0)) if torch.cuda.is_available() else "cpu", + "torch_cuda": torch.version.cuda, + "qualification": not args.smoke, + "installed_distributions": sorted( + (dist.metadata["Name"], dist.version) for dist in importlib.metadata.distributions() + ), + } + data["installed_distributions_sha256"] = fingerprint(data["installed_distributions"]) + evidence.emit("environment", **data) + if not args.smoke: + require(versions == EXPECTED_VERSIONS, "installed stack differs from the exact public lock") + require(torchao is None, "torchao must be absent; benchmark does not mutate dependencies") + require(torch.cuda.is_available() and "L4" in data["device"], "qualification requires the declared L4") + return data + + +async def experiment(args: argparse.Namespace, evidence: Evidence, summary: dict[str, Any]) -> None: + fixtures = args.fixtures.resolve() + manifest = json.loads((fixtures / "manifest.json").read_text()) + snapshot = args.snapshot.resolve() if args.snapshot else Path(manifest["snapshot"]) + require(bool(manifest["smoke"]) == args.smoke, "fixture smoke/qualification mode differs") + for filename, expected in manifest["model_files"].items(): + require(digest(snapshot / filename) == expected, "model file changed after fixture preparation") + if not args.smoke: + require(digest(snapshot / "model.safetensors") == WEIGHTS_SHA256, "public checkpoint hash differs") + for name, record in manifest["adapters"].items(): + for filename, expected in record["files"].items(): + require(digest(fixtures / name / filename) == expected, "LoRA fixture changed") + summary["fixtures"] = manifest + configs(evidence.directory / "models", args.source, fixtures, args.arm, snapshot, smoke=args.smoke) + app = AppFactory.create_app( + AppStateConfig( + models_dir=evidence.directory / "models", + device="cpu" if args.smoke else "cuda:0", + remote_serving=False, + bind_host="127.0.0.1", + ) + ) + sock = socket.socket() + sock.bind(("127.0.0.1", 0)) + url = f"http://127.0.0.1:{sock.getsockname()[1]}" + server = uvicorn.Server(uvicorn.Config(app, log_level="warning", timeout_graceful_shutdown=20, access_log=False)) + task = asyncio.create_task(server.serve(sockets=[sock])) + client = Client(url) + adapter = None + try: + while not server.started: + if task.done(): + await task + raise RuntimeError("server stopped before readiness") + await asyncio.sleep(0.05) + registry = app.state.registry + load_start = time.monotonic() + adapter = await registry.load_async(MODEL, "cpu" if args.smoke else "cuda:0") + summary["load_s"] = time.monotonic() - load_start + loaded = registry._loaded[MODEL] + worker = loaded.worker + require(worker is not None, "actual model worker is missing") + original = await asyncio.to_thread(tensor_state, adapter) + evidence.emit("original_state", state=original) + expected_dtype = "torch.float32" if args.smoke else "torch.float16" + require( + {value["dtype"] for value in original["tensors"].values()} == {expected_dtype}, + "actual base tensor precision differs from the protocol", + ) + summary["actual_precision"] = expected_dtype + summary["loaded_before_forward_memory"] = memory() + summary["manager_present"] = getattr(adapter, "_lora", None) is not None + require(summary["manager_present"] == (args.arm == "R"), "unexpected manager activation") + + async def call(selection: str | None, label: str, *, cold: bool = False) -> dict[str, Any]: + row = await asyncio.to_thread(client.score, selection, label, cold=cold) + evidence.emit("request", **row) + validate_usage([row], adapter.batch_observations) + return row + + adapter.observation_phase = "preselection" + bases = [await call(None, f"base-{i}") for i in range(3)] + baseline = bases[0]["scores"] + drift = max(vector_delta(baseline, row["scores"]) for row in bases) + require(drift <= RESTORE_ATOL, "preselection SDK baseline is unstable") + if not args.smoke: + require(vector_delta(baseline, CPU_ORACLE) <= ORACLE_ATOL, "finite public-checkpoint oracle differs") + summary["base_scores"] = baseline + summary["base_drift"] = drift + summary["shape_oracles"] = await oracle(adapter, worker, None, smoke=args.smoke) + evidence.emit("shape_oracles", values=summary["shape_oracles"]) + # The fixed base memory reference is warm and precedes any adapter + # selection; CUDA library initialization is not counted as LoRA bytes. + summary["initial_memory"] = memory() + selected_oracles = {None: summary["shape_oracles"]} + if args.arm == "R": + before_fetch = len(adapter.fetch_observations) + try: + await call("undeclared", "undeclared") + except RequestError as error: + require(error.status_code == HTTPStatus.BAD_REQUEST, "undeclared selection did not return 400") + evidence.emit("undeclared_denied", status=error.status_code, code=error.code) + else: + raise AssertionError("undeclared LoRA accepted") + require(len(adapter.fetch_observations) == before_fetch, "undeclared source reached fetch") + adapted = {} + for alias in ("a", "b"): + if alias == "a": + cold_clients = [Client(url) for _ in range(4)] + cold_barrier = threading.Barrier(4) + + def cold_call(index: int) -> dict[str, Any]: + cold_barrier.wait(timeout=20) + return cold_clients[index].score("a", f"singleflight-{index}", cold=True) + + try: + cold_rows = await asyncio.gather(*(asyncio.to_thread(cold_call, index) for index in range(4))) + finally: + for other in cold_clients: + other.close() + evidence.emit("singleflight_requests", requests=cold_rows) + require( + sum(any(item.get("code") == "LORA_LOADING" for item in row["responses"]) for row in cold_rows) + >= RESIDENT_CAPACITY, + "cold duplicate requests did not overlap source loading", + ) + require( + len(adapter.fetch_observations) - before_fetch == 1, "duplicate cold requests fetched twice" + ) + adapted[alias] = await call(alias, f"cold-{alias}", cold=True) + if alias == "b": + require( + any( + row.get("code") == "LORA_LOADING" and row["status"] == HTTPStatus.SERVICE_UNAVAILABLE + for row in adapted[alias]["responses"] + ), + "cold request did not expose LORA_LOADING", + ) + restored = await call(None, f"reset-{alias}") + require(vector_delta(baseline, restored["scores"]) <= RESTORE_ATOL, "base restoration failed") + effects = { + "a_base": vector_delta(adapted["a"]["scores"], baseline), + "b_base": vector_delta(adapted["b"]["scores"], baseline), + "a_b": vector_delta(adapted["a"]["scores"], adapted["b"]["scores"]), + } + evidence.emit("effects", **effects) + require( + min(effects.values()) >= max(EFFECT_FLOOR, EFFECT_DRIFT_MULTIPLIER * drift), + "adapter effects not qualified", + ) + require( + len(adapter.fetch_observations) - before_fetch == RESIDENT_CAPACITY, + "cold adapters fetched more than once", + ) + for alias in ("a", "b"): + selected_oracles[alias] = await oracle(adapter, worker, alias, smoke=args.smoke) + adapter.observation_phase = "concurrent-isolation" + first = len(adapter.batch_observations) + clients = [Client(url) for _ in range(12)] + barrier = threading.Barrier(len(clients)) + selections = [None, "a", "b"] * 4 + + def concurrent(index: int) -> dict[str, Any]: + barrier.wait(timeout=20) + return clients[index].score(selections[index], f"isolation-{index}") + + try: + rows = await asyncio.gather(*(asyncio.to_thread(concurrent, index) for index in range(len(clients)))) + finally: + for other in clients: + other.close() + evidence.emit("concurrent_requests", requests=rows) + batches = adapter.batch_observations[first:] + evidence.emit("concurrent_batches", batches=batches) + validate_usage(rows, batches) + combined = set() + for batch in batches: + selections_in_batch = {selections[int(item.split(":")[0].split("-")[-1])] for item in batch["item_ids"]} + require(len(selections_in_batch) == 1, "one forward mixed adapter selections") + alias = next(iter(selections_in_batch)) + source = str(fixtures / alias) if alias else None + require(batch["selection"] in (alias, source), "actual worker selection differs from request") + key = batch_key(batch) + require( + vector_delta(batch["scores"], selected_oracles[alias][key]["scores"]) <= RESTORE_ATOL, + "concurrent selected scores differ from shape-matched oracle", + ) + expected_peft = PEFTLoRAMixin._peft_adapter_name(source) if source else None + for forward in batch["forwards"]: + enabled = forward["enabled_adapters"] + require(bool(enabled) == bool(alias), "actual adapter enable state differs at model forward") + require( + all(item["adapters"] == [expected_peft] for item in enabled), "wrong PEFT adapter at forward" + ) + if len({item.split(":")[0] for item in batch["item_ids"]}) > 1: + combined.add(alias) + require(combined == {None, "a", "b"}, "did not observe cross-request batches for base, A and B") + require( + max(row["start_ns"] for row in rows) < min(row["end_ns"] for row in rows), "SDK calls did not overlap" + ) + await call(None, "before-performance-base") + require(set(registry.loaded_lora_names(MODEL)) == {"a", "b"}, "resident set before R measurement differs") + summary["adapter_storage"] = adapter_storage(adapter) + base_unchanged(original, await asyncio.to_thread(tensor_state, adapter)) + + summary["untimed_replays"] = {} + adapter.observation_phase = "untimed-shape-qualification" + for concurrency in (1, *CONCURRENCIES): + first = len(adapter.batch_observations) + rows = await asyncio.to_thread(replay, url, concurrency) + batches = adapter.batch_observations[first:] + evidence.emit("untimed_replay", concurrency=concurrency, requests=rows, batches=batches) + summary["untimed_replays"][str(concurrency)] = {"requests": rows, "batches": batches} + validate_usage(rows, batches) + for batch in batches: + key = batch_key(batch) + require( + vector_delta(batch["scores"], summary["shape_oracles"][key]["scores"]) <= RESTORE_ATOL, + "untimed SDK replay differs from preselection shape oracle", + ) + require(batch["selection"] is None, "untimed base replay selected a LoRA") + require( + all(not forward["enabled_adapters"] for forward in batch["forwards"]), + "untimed base replay left a PEFT adapter enabled", + ) + if concurrency > 1: + require( + any(len({item.split(":")[0] for item in batch["item_ids"]}) > 1 for batch in batches), + "untimed concurrent replay did not combine independent SDK requests", + ) + summary["resident_memory"] = memory() + summary["performance"] = {} + with unobserved(adapter): + for concurrency in (1, *CONCURRENCIES): + phase = f"performance-C{concurrency}" + evidence.emit("gpu_before_phase", phase=phase, telemetry=telemetry()) + measured = await asyncio.to_thread(performance, url, concurrency, evidence, smoke=args.smoke) + summary["performance"][str(concurrency)] = measured + evidence.emit("performance", phase=phase, **measured) + atomic_json(evidence.directory / "trial.json", summary) + for row in measured["samples"]: + require(row["result"]["usage"] == bases[0]["result"]["usage"], "timed SDK usage differs") + if concurrency == 1: + require(vector_delta(row["scores"], baseline) <= RESTORE_ATOL, "timed C1 base output differs") + row["diagnostic_matching_oracles"] = matching_vectors(row["scores"], summary["shape_oracles"]) + measured["actual_timed_batch_shape"] = "unobserved" + evidence.emit( + "timed_output_diagnostics", + concurrency=concurrency, + matches=[ + {"label": row["label"], "matches": row["diagnostic_matching_oracles"]} + for row in measured["samples"] + ], + ) + evidence.emit("gpu_after_phase", phase=phase, telemetry=telemetry()) + adapter.observation_phase = "lifecycle" + if args.arm == "R": + previous = set(registry.loaded_lora_names(MODEL)) + evicted = set() + for index, alias in enumerate(("c", "a", "b", "c")): + await call(alias, f"churn-{index}", cold=True) + ready = set(registry.loaded_lora_names(MODEL)) + require(alias in ready and len(ready) == RESIDENT_CAPACITY, "LRU capacity/readiness differs") + evicted.update(previous - ready) + evidence.emit("resident_set", selected=alias, ready=sorted(ready)) + previous = ready + require(evicted == {"a", "b", "c"}, "LRU churn failed to evict each dynamic adapter") + + async def unload_adapters() -> None: + for name in list(loaded.loras): + await registry._unload_lora(loaded, name, model=MODEL) + loaded.loras.pop(name) + + await unload_adapters() + final_state = await asyncio.to_thread(tensor_state, adapter) + evidence.emit("restored_state", state=final_state) + base_unchanged(original, final_state, restored=True) + restored = await call(None, "final-unloaded-base") + require(vector_delta(baseline, restored["scores"]) <= RESTORE_ATOL, "final unload changed base output") + summary["unloaded_memory"] = memory() + require( + summary["unloaded_memory"]["allocated"] == summary["initial_memory"]["allocated"], + "adapter unload did not restore allocated GPU bytes", + ) + await call("a", "same-instance-reload", cold=True) + require(id(adapter._model) == original["wrapper_id"], "reload replaced outer wrapper") + await unload_adapters() + base_unchanged(original, await asyncio.to_thread(tensor_state, adapter), restored=True) + require(not adapter.mutation_overlap, "adapter mutation overlapped a score forward") + evidence.emit( + "all_batches", + batches=adapter.batch_observations, + selections=adapter.selection_observations, + fetches=adapter.fetch_observations, + ) + inner_ref = weakref.ref(adapter._model.model) + outer_ref = weakref.ref(adapter._model) + await registry.unload_async(MODEL) + gc.collect() + require( + adapter._model is None and outer_ref() is None and inner_ref() is None, + "full model unload retained wrapper or inner model", + ) + summary["model_unloaded_memory"] = memory() + summary["passed"] = True + finally: + try: + evidence.emit( + "client_wire_responses", responses=[describe_response(response) for response in client.responses] + ) + if adapter is not None: + evidence.emit( + "final_batch_snapshot", + batches=adapter.batch_observations, + fetches=adapter.fetch_observations, + mutation_overlap=adapter.mutation_overlap, + ) + finally: + try: + client.close() + finally: + server.should_exit = True + try: + await asyncio.wait_for(asyncio.shield(task), timeout=30) + summary["shutdown_complete"] = True + finally: + sock.close() + + +async def run(args: argparse.Namespace) -> dict[str, Any]: + evidence = Evidence(args.output.resolve()) + summary: dict[str, Any] = { + "arm": args.arm, + "pair": args.pair, + "passed": False, + "shutdown_complete": False, + "smoke": args.smoke, + "pid": os.getpid(), + } + os.environ.update( + { + "SIE_TELEMETRY_DISABLED": "1", + "SIE_DISK_CACHE_ENABLED": "false", + "SIE_MAX_LORAS_PER_MODEL": "2", + "SIE_MAX_BATCH_REQUESTS": str(MAX_BATCH_ITEMS), + "TOKENIZERS_PARALLELISM": "false", + } + ) + torch.set_num_threads(THREADS) + torch.set_num_interop_threads(1) + try: + summary["environment"] = preflight(args, evidence) + async with asyncio.timeout(args.deadline_s): + await experiment(args, evidence, summary) + except BaseException as error: + summary["passed"] = False + summary["error"] = error_record(error) + evidence.emit("failure", **summary["error"]) + raise + finally: + atomic_json(evidence.directory / "trial.json", summary) + evidence.emit("finished", passed=summary["passed"], shutdown_complete=summary["shutdown_complete"]) + return summary + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--source", type=Path, required=True) + parser.add_argument("--fixtures", type=Path, required=True) + parser.add_argument("--snapshot", type=Path) + parser.add_argument("--output", type=Path, required=True) + parser.add_argument("--arm", choices=("B", "N", "R"), required=True) + parser.add_argument("--pair", type=int, default=0) + parser.add_argument("--deadline-s", type=float, default=180) + parser.add_argument("--smoke", action="store_true") + args = parser.parse_args() + require(0 < args.deadline_s <= MAX_TRIAL_SECONDS, "worker deadline must be in (0, 300]") + asyncio.run(run(args)) + + +if __name__ == "__main__": + main() From c5f77f72f25ed1b186cb2c5ebfddb237023c8efb Mon Sep 17 00:00:00 2001 From: svonava Date: Fri, 16 Oct 2026 17:04:52 -0700 Subject: [PATCH 03/16] test(cross-encoder): reserve full trial cleanup budget --- tools/benchmarks/cross_encoder_lora/runner.py | 16 +- .../tests/test_runner_budget.py | 137 ++++++++++++++++++ 2 files changed, 149 insertions(+), 4 deletions(-) create mode 100644 tools/benchmarks/cross_encoder_lora/tests/test_runner_budget.py diff --git a/tools/benchmarks/cross_encoder_lora/runner.py b/tools/benchmarks/cross_encoder_lora/runner.py index c381f71ee..9195caf95 100644 --- a/tools/benchmarks/cross_encoder_lora/runner.py +++ b/tools/benchmarks/cross_encoder_lora/runner.py @@ -28,6 +28,9 @@ ) ROOT = Path(__file__).resolve().parents[3] +WAIT_GRACE_SECONDS = 45 +TERMINATE_GRACE_SECONDS = 35 +KILL_GRACE_SECONDS = 10 def harness_manifest() -> dict[str, Any]: @@ -123,10 +126,10 @@ def stop(process: subprocess.Popen) -> None: if process.poll() is None: process.send_signal(signal.SIGTERM) try: - process.wait(timeout=35) + process.wait(timeout=TERMINATE_GRACE_SECONDS) except subprocess.TimeoutExpired: process.kill() - process.wait(timeout=10) + process.wait(timeout=KILL_GRACE_SECONDS) def run(args: argparse.Namespace) -> dict[str, Any]: @@ -152,7 +155,12 @@ def run(args: argparse.Namespace) -> dict[str, Any]: schedule = [(0, "R")] if args.smoke else [(pair, arm) for pair, order in enumerate(ORDERS) for arm in order] for pair, arm in schedule: require( - time.monotonic() + args.trial_deadline_s + 45 < absolute_deadline, + time.monotonic() + + args.trial_deadline_s + + WAIT_GRACE_SECONDS + + TERMINATE_GRACE_SECONDS + + KILL_GRACE_SECONDS + < absolute_deadline, "insufficient time for a complete bounded next trial and cleanup", ) directory = args.output / f"trial-{pair}-{arm}" @@ -187,7 +195,7 @@ def run(args: argparse.Namespace) -> dict[str, Any]: process = subprocess.Popen( command, env=source_environment(source), cwd=source, stdout=log, stderr=subprocess.STDOUT ) - returncode = process.wait(timeout=args.trial_deadline_s + 45) + returncode = process.wait(timeout=args.trial_deadline_s + WAIT_GRACE_SECONDS) except BaseException as error: # noqa: BLE001 -- persist the trial, then re-raise after owned cleanup pending_error = error finally: diff --git a/tools/benchmarks/cross_encoder_lora/tests/test_runner_budget.py b/tools/benchmarks/cross_encoder_lora/tests/test_runner_budget.py new file mode 100644 index 000000000..7f9810410 --- /dev/null +++ b/tools/benchmarks/cross_encoder_lora/tests/test_runner_budget.py @@ -0,0 +1,137 @@ +"""Admission reserves the complete worker wait and both cleanup grace periods.""" + +from __future__ import annotations + +import argparse +import json +import signal +import subprocess +from pathlib import Path +from typing import Any + +import pytest + +from tools.benchmarks.cross_encoder_lora import runner + + +def arguments(tmp_path: Path, budget: float) -> argparse.Namespace: + return argparse.Namespace( + output=tmp_path / "evidence", + inputs=tmp_path / "inputs", + fixtures=tmp_path / "fixtures", + snapshot=None, + smoke=True, + environment_receipt=None, + deadline_s=budget, + trial_deadline_s=120, + ) + + +def setup_sources(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(runner, "extract", lambda *_: {"sources": {"candidate": {"commit": "frozen"}}}) + monkeypatch.setattr(runner, "harness_manifest", dict) + + +@pytest.mark.parametrize(("budget", "admitted"), [(209.999, False), (210.0, False), (210.001, True)]) +def test_trial_admission_reserves_full_cleanup( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + budget: float, + admitted: bool, +) -> None: + setup_sources(monkeypatch) + monkeypatch.setattr(runner.time, "monotonic", lambda: 100.0) + waits = [] + started = [] + + class CompletedProcess: + returncode = 0 + + def wait(self, *, timeout: float) -> int: + waits.append(timeout) + return self.returncode + + def poll(self) -> int: + return self.returncode + + def start(command: list[str], **_kwargs: Any) -> CompletedProcess: + started.append(command) + output = Path(command[command.index("--output") + 1]) + output.mkdir() + (output / "trial.json").write_text( + json.dumps({"pair": 0, "arm": "R", "passed": True, "shutdown_complete": True}) + ) + return CompletedProcess() + + monkeypatch.setattr(runner.subprocess, "Popen", start) + args = arguments(tmp_path, budget) + if admitted: + assert runner.run(args)["passed"] + assert waits == [165] + else: + with pytest.raises(AssertionError, match="insufficient time"): + runner.run(args) + terminal = json.loads((args.output / "summary.json").read_text()) + assert not terminal["passed"] + assert terminal["trials"] == [] + assert bool(started) is admitted + + +def test_full_timeout_cleanup_finishes_before_budget_and_retains_partial_trial( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + setup_sources(monkeypatch) + clock = [100.0] + monkeypatch.setattr(runner.time, "monotonic", lambda: clock[0]) + actions = [] + + class HungProcess: + returncode = None + + def wait(self, *, timeout: float) -> int: + actions.append(("wait", timeout)) + clock[0] += timeout + if len([action for action, _ in actions if action == "wait"]) < 3: + raise subprocess.TimeoutExpired("owned-worker", timeout) + self.returncode = -9 + return self.returncode + + def poll(self) -> None: + return None + + def send_signal(self, signum: int) -> None: + actions.append(("signal", signum)) + + def kill(self) -> None: + actions.append(("kill", None)) + + def start(command: list[str], **_kwargs: Any) -> HungProcess: + output = Path(command[command.index("--output") + 1]) + output.mkdir() + (output / "trial.json").write_text( + json.dumps( + { + "pair": 0, + "arm": "R", + "passed": False, + "shutdown_complete": False, + "performance": {"1": {"completed_samples": 7}}, + } + ) + ) + return HungProcess() + + monkeypatch.setattr(runner.subprocess, "Popen", start) + args = arguments(tmp_path, 211) + with pytest.raises(subprocess.TimeoutExpired): + runner.run(args) + assert actions == [("wait", 165), ("signal", signal.SIGTERM), ("wait", 35), ("kill", None), ("wait", 10)] + assert clock[0] == 310 + assert clock[0] < 100 + args.deadline_s + terminal = json.loads((args.output / "summary.json").read_text()) + assert not terminal["passed"] + trial = terminal["trials"][0] + assert trial["performance"]["1"]["completed_samples"] == 7 + assert trial["process_returncode"] == -9 + assert trial["runner_error"]["type"] == "TimeoutExpired" From 445c5b303a15e852bd094505e0021fe8eb4768e0 Mon Sep 17 00:00:00 2001 From: svonava Date: Fri, 16 Oct 2026 17:38:25 -0700 Subject: [PATCH 04/16] fix(benchmarks): bound cross-encoder evidence size --- tools/benchmarks/cross_encoder_lora/README.md | 18 +- tools/benchmarks/cross_encoder_lora/runner.py | 64 ++++- .../tests/test_evidence_size.py | 223 ++++++++++++++++++ .../cross_encoder_lora/tests/test_probe.py | 6 +- .../tests/test_runner_budget.py | 4 +- tools/benchmarks/cross_encoder_lora/worker.py | 22 +- 6 files changed, 318 insertions(+), 19 deletions(-) create mode 100644 tools/benchmarks/cross_encoder_lora/tests/test_evidence_size.py diff --git a/tools/benchmarks/cross_encoder_lora/README.md b/tools/benchmarks/cross_encoder_lora/README.md index 988ac98fe..12530dc1f 100644 --- a/tools/benchmarks/cross_encoder_lora/README.md +++ b/tools/benchmarks/cross_encoder_lora/README.md @@ -106,7 +106,10 @@ list-append overhead; this is an SDK workload measurement. Actual timed C4/C16 batch shapes are **unknown**. Their raw four-score vectors are compared diagnostically against complete preselection oracle vectors at -`1e-6`, with matching shapes and ambiguity recorded. A diagnostic cannot combine +`1e-6`. Each response records one deterministic matching oracle reference, or +null if none matches. This witness does not identify the actual shape or imply +a unique match: the retained raw vector and complete oracle corpus allow every +match and its ambiguity to be recomputed. A diagnostic cannot combine different oracle shapes per element or substitute for the separately observed shape-qualified replays. Timed C1 has one outstanding four-document request and must still match its preselection result at `1e-6`. This measurement is scoped to @@ -130,7 +133,11 @@ the observed untimed replays supply that evidence instead. ## Evidence and CPU checks -`summary.json` retains trial outcomes, statistical gates and source hashes. +`summary.json` schema 2 retains trial outcomes, aggregate metrics, statistical +gates and source hashes. Each entry links its complete `trial.json` by relative +path and SHA-256; raw request arrays and oracle corpora stay in those per-trial +files instead of being copied into the growing summary. Final cross-process +checks reopen the complete records and verify their hashes and identities. The summary also hashes the complete benchmark file manifest. Each trial captures the full installed-distribution manifest; its digest must remain identical across arms. An optional `--environment-receipt` attaches a separate @@ -142,6 +149,13 @@ hashes, library versions and shutdown status. Failure paths retain completed phases and partial request samples. Hash evidence files when archiving them; the runner records each terminal trial file hash. +The compact representation follows an incomplete run whose stdout archive was +truncated. Its old evidence stays separate and is not acceptance evidence. This +change affects only post-measurement diagnostics and storage, not the timed +path, workload, numerical tolerances, statistical gates, or raw observations. +Store the evidence directory on durable storage and verify the full archive +hash after retrieval; stdout is unsuitable for transferring large artifacts. + ```console mise run test -- tools/benchmarks/cross_encoder_lora/tests/test_probe.py mise run typecheck -- tools/benchmarks/cross_encoder_lora diff --git a/tools/benchmarks/cross_encoder_lora/runner.py b/tools/benchmarks/cross_encoder_lora/runner.py index 9195caf95..bc8c078bb 100644 --- a/tools/benchmarks/cross_encoder_lora/runner.py +++ b/tools/benchmarks/cross_encoder_lora/runner.py @@ -132,11 +132,61 @@ def stop(process: subprocess.Popen) -> None: process.wait(timeout=KILL_GRACE_SECONDS) +def trial_summary(trial: dict[str, Any]) -> dict[str, Any]: + """Keep the cumulative index independent of the number of raw requests.""" + fields = ( + "pair", + "arm", + "passed", + "shutdown_complete", + "smoke", + "pid", + "process_returncode", + "source_commit", + "terminal_record", + "terminal_record_sha256", + "error", + "runner_error", + ) + result = {field: trial[field] for field in fields if field in trial} + if "performance" in trial: + metrics = ("concurrency", "count", "elapsed_s", "mean_ms", "p95_ms", "rps", "actual_timed_batch_shape") + result["performance"] = { + concurrency: {field: phase[field] for field in metrics if field in phase} + for concurrency, phase in trial["performance"].items() + } + return result + + +def load_trial(output: Path, record: dict[str, Any]) -> dict[str, Any]: + """Reopen the complete evidence only after checking its recorded identity.""" + relative = f"trial-{record['pair']}-{record['arm']}/trial.json" + require(record["terminal_record"] == relative, "unexpected terminal record path") + path = output / relative + require(path.resolve().is_relative_to(output.resolve()), "terminal record path escapes output") + require(digest(path) == record["terminal_record_sha256"], "terminal record hash differs") + trial = json.loads(path.read_text()) + require((trial["pair"], trial["arm"]) == (record["pair"], record["arm"]), "terminal record identity differs") + for field, value in trial_summary(trial).items(): + if field == "passed" and "runner_error" in record: + # A process timeout can invalidate an otherwise successful worker + # record. The runner's failed verdict must survive that override. + require(record[field] is False, "runner error lost its failed verdict") + else: + require(record.get(field) == value, "terminal record summary differs") + if "runner_error" in record: + trial["passed"] = False + trial["runner_error"] = record["runner_error"] + if "process_returncode" in record: + trial["process_returncode"] = record["process_returncode"] + return trial + + def run(args: argparse.Namespace) -> dict[str, Any]: args.output.mkdir(parents=True, exist_ok=False) manifest = extract(args.inputs.resolve(), args.output / "sources") summary: dict[str, Any] = { - "schema": 1, + "schema": 2, "passed": False, "source_manifest": manifest, "harness_manifest": harness_manifest(), @@ -218,10 +268,15 @@ def run(args: argparse.Namespace) -> dict[str, Any]: trial["terminal_record_sha256"] = ( digest(directory / "trial.json") if (directory / "trial.json").exists() else None ) + trial["terminal_record"] = ( + str((directory / "trial.json").relative_to(args.output)) + if trial["terminal_record_sha256"] is not None + else None + ) if pending_error is not None: trial["passed"] = False trial["runner_error"] = {"type": type(pending_error).__name__, "message": str(pending_error)} - summary["trials"].append(trial) + summary["trials"].append(trial_summary(trial)) atomic_json(summary_path, summary) if pending_error is not None: raise pending_error @@ -234,8 +289,9 @@ def run(args: argparse.Namespace) -> dict[str, Any]: summary["passed"] = True summary["qualification"] = False else: - summary["correctness_memory"] = cross_checks(summary["trials"]) - summary["statistics"] = compare(summary["trials"]) + complete_trials = [load_trial(args.output, row) for row in summary["trials"]] + summary["correctness_memory"] = cross_checks(complete_trials) + summary["statistics"] = compare(complete_trials) summary["passed"] = summary["statistics"]["passed"] require(summary["passed"], "fixed non-inferiority gates did not pass") except BaseException as error: diff --git a/tools/benchmarks/cross_encoder_lora/tests/test_evidence_size.py b/tools/benchmarks/cross_encoder_lora/tests/test_evidence_size.py new file mode 100644 index 000000000..0ea45f796 --- /dev/null +++ b/tools/benchmarks/cross_encoder_lora/tests/test_evidence_size.py @@ -0,0 +1,223 @@ +"""Raw observations remain revalidatable without multiplying diagnostic copies.""" + +from __future__ import annotations + +import json +from pathlib import Path +from typing import Any + +import pytest + +from tools.benchmarks.cross_encoder_lora import runner +from tools.benchmarks.cross_encoder_lora.evidence import atomic_json, digest, vector_delta +from tools.benchmarks.cross_encoder_lora.protocol import ORDERS, RESTORE_ATOL +from tools.benchmarks.cross_encoder_lora.tests.test_runner_budget import arguments, setup_sources +from tools.benchmarks.cross_encoder_lora.worker import ORDER, matching_vector + + +def corpus() -> tuple[list[float], dict[str, Any]]: + scores = [1.0, 2.0, 3.0, 4.0] + oracles = { + f"workload-{copies:02d}": { + "copies": copies, + "scores": [scores[index] for index in ORDER for _ in range(copies)], + } + for copies in range(1, 17) + } + return scores, oracles + + +def oracle_vector(oracles: dict[str, Any], reference: dict[str, Any]) -> list[float]: + # Deliberately reconstruct independently of the matcher, in request order. + oracle = oracles[reference["workload_sha256"]] + assert reference["copies"] == oracle["copies"] + offset = reference["request_position"] + groups = { + item: oracle["scores"][index * oracle["copies"] : (index + 1) * oracle["copies"]] + for index, item in enumerate(ORDER) + } + return [groups[item][offset] for item in range(4)] + + +def test_all_136_matches_have_bounded_reconstructible_witness() -> None: + scores, oracles = corpus() + reference = matching_vector(scores, oracles) + assert reference == {"workload_sha256": "workload-01", "copies": 1, "request_position": 0} + assert matching_vector(scores, dict(reversed(list(oracles.items())))) == reference + raw = [ + { + "label": str(index), + "scores": scores, + "latency_ms": 1.0 + index / 1000, + "responses": [{"status": 200}], + "diagnostic_matching_oracle": matching_vector(scores, oracles), + } + for index in range(1000) + ] + encoded = json.dumps({"shape_oracles": oracles, "samples": raw}) + assert len(encoded) < 260_000 # Independent of repeating all 136 witnesses per response. + recovered = json.loads(encoded) + for original, row in zip(raw, recovered["samples"], strict=True): + assert row == original + witness = oracle_vector(recovered["shape_oracles"], row["diagnostic_matching_oracle"]) + assert vector_delta(row["scores"], witness) <= RESTORE_ATOL + all_matches = [ + {"workload_sha256": key, "copies": oracle["copies"], "request_position": position} + for key, oracle in recovered["shape_oracles"].items() + for position in range(oracle["copies"]) + if vector_delta( + scores, + oracle_vector( + recovered["shape_oracles"], + {"workload_sha256": key, "copies": oracle["copies"], "request_position": position}, + ), + ) + <= RESTORE_ATOL + ] + assert len(all_matches) == 136 # Ambiguity is reconstructible, not asserted away. + assert matching_vector([1.0, 2.0, 3.0, 4.0 + 2 * RESTORE_ATOL], oracles) is None + + +def terminal(directory: Path, pair: int, arm: str, *, failed: bool = False) -> dict[str, Any]: + scores, oracles = corpus() + rows = [ + { + "label": str(index), + "scores": scores, + "latency_ms": 10.0, + "responses": [ + {"status": 503, "code": "TEST_FAILURE", "body": "retained error"} if failed else {"status": 200} + ], + } + for index in range(500) + ] + trial = { + "pair": pair, + "arm": arm, + "passed": not failed, + "shutdown_complete": not failed, + "shape_oracles": oracles, + "performance": { + str(concurrency): { + "count": len(rows), + "mean_ms": 10.0, + "p95_ms": 10.0, + "rps": 100.0, + "samples": rows, + "future_raw_field": rows, + } + for concurrency in (1, 4, 16) + }, + } + if failed: + trial["error"] = {"type": "RuntimeError", "message": "retained failure"} + directory.mkdir(parents=True) + atomic_json(directory / "trial.json", trial) + return trial + + +def test_summary_size_is_bounded_and_final_checks_reopen_every_trial( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + setup_sources(monkeypatch) + monkeypatch.setattr(runner, "extract", lambda *_: {"sources": {arm: {"commit": arm} for arm in ("B", "candidate")}}) + monkeypatch.setattr(runner.time, "monotonic", lambda: 100.0) + + class CompletedProcess: + returncode = 0 + + def wait(self, *, timeout: float) -> int: + return self.returncode + + def poll(self) -> int: + return self.returncode + + def start(command: list[str], **_kwargs: Any) -> CompletedProcess: + def value(flag: str) -> str: + return command[command.index(flag) + 1] + + terminal(Path(value("--output")), int(value("--pair")), value("--arm")) + return CompletedProcess() + + full_records = [] + + def inspect(trials: list[dict[str, Any]]) -> dict[str, Any]: + full_records.extend(trials) + assert len(trials) == 18 + for trial in trials: + assert len(trial["shape_oracles"]) == 16 + for phase in trial["performance"].values(): + assert len(phase["samples"]) == 500 + assert phase["samples"][0]["scores"] == [1.0, 2.0, 3.0, 4.0] + assert phase["samples"][0]["responses"] == [{"status": 200}] + return {"passed": True} + + monkeypatch.setattr(runner.subprocess, "Popen", start) + monkeypatch.setattr(runner, "cross_checks", inspect) + args = arguments(tmp_path, 3000) + args.smoke = False + summary = runner.run(args) + assert summary["schema"] == 2 + assert summary["passed"] + assert len(full_records) == 18 + encoded = (args.output / "summary.json").read_bytes() + assert len(encoded) < 30_000 + assert b'"samples"' not in encoded + assert b'"shape_oracles"' not in encoded + assert b'"future_raw_field"' not in encoded + for record, (pair, arm) in zip( + summary["trials"], ((i, arm) for i, order in enumerate(ORDERS) for arm in order), strict=True + ): + assert record["terminal_record"] == f"trial-{pair}-{arm}/trial.json" + assert record["terminal_record_sha256"] == digest(args.output / record["terminal_record"]) + + +def test_failed_raw_record_survives_compaction_and_refs_reject_tampering(tmp_path: Path) -> None: + path = tmp_path / "trial-0-R/trial.json" + trial = terminal(path.parent, 0, "R", failed=True) + trial.update( + terminal_record="trial-0-R/trial.json", + terminal_record_sha256=digest(path), + process_returncode=-9, + runner_error={"type": "TimeoutExpired", "message": "owned worker stopped"}, + ) + record = runner.trial_summary(trial) + assert not record["passed"] + assert not record["shutdown_complete"] + assert record["error"] == trial["error"] + assert record["runner_error"] == trial["runner_error"] + assert record["process_returncode"] == -9 + raw = runner.load_trial(tmp_path, record) + assert raw["performance"]["1"]["samples"] == trial["performance"]["1"]["samples"] + changed = json.loads(json.dumps(record)) + changed["performance"]["1"]["mean_ms"] = 9.0 + with pytest.raises(AssertionError, match="summary differs"): + runner.load_trial(tmp_path, changed) + with pytest.raises(AssertionError, match="record path"): + runner.load_trial(tmp_path, {**record, "terminal_record": "../trial.json"}) + path.write_text(path.read_text() + "\n") + with pytest.raises(AssertionError, match="record hash"): + runner.load_trial(tmp_path, record) + raw["pair"] = 1 + atomic_json(path, raw) + with pytest.raises(AssertionError, match="identity"): + runner.load_trial(tmp_path, {**record, "terminal_record_sha256": digest(path)}) + + +def test_runner_failure_override_never_turns_into_success(tmp_path: Path) -> None: + path = tmp_path / "trial-0-R/trial.json" + trial = terminal(path.parent, 0, "R") + trial.update( + terminal_record="trial-0-R/trial.json", + terminal_record_sha256=digest(path), + passed=False, + runner_error={"type": "TimeoutExpired", "message": "owned worker stopped"}, + ) + record = runner.trial_summary(trial) + reopened = runner.load_trial(tmp_path, record) + assert not reopened["passed"] + assert reopened["runner_error"] == record["runner_error"] + assert json.loads(path.read_text())["passed"] # The original raw file remains unchanged. + assert not record["passed"] + with pytest.raises(AssertionError, match="lost its failed verdict"): + runner.load_trial(tmp_path, {**record, "passed": True}) diff --git a/tools/benchmarks/cross_encoder_lora/tests/test_probe.py b/tools/benchmarks/cross_encoder_lora/tests/test_probe.py index eb820a77a..5fed4ea86 100644 --- a/tools/benchmarks/cross_encoder_lora/tests/test_probe.py +++ b/tools/benchmarks/cross_encoder_lora/tests/test_probe.py @@ -21,7 +21,7 @@ from tools.benchmarks.cross_encoder_lora.fixtures import prepare from tools.benchmarks.cross_encoder_lora.protocol import BASE_REVISION, ORDERS, compare, paired_gate, percentile from tools.benchmarks.cross_encoder_lora.runner import ROOT, extract, source_environment -from tools.benchmarks.cross_encoder_lora.worker import ORDER, describe_response, matching_vectors, validate_response +from tools.benchmarks.cross_encoder_lora.worker import ORDER, describe_response, matching_vector, validate_response def tiny_snapshot(directory: Path) -> Path: @@ -280,8 +280,8 @@ def test_timed_diagnostics_require_one_complete_oracle_vector() -> None: "first": {"copies": 1, "scores": [first[index] for index in ORDER]}, "second": {"copies": 2, "scores": [second[index] for index in ORDER for _ in range(2)]}, } - assert matching_vectors(first, oracles) == [{"workload_sha256": "first", "copies": 1, "request_position": 0}] - assert matching_vectors([0.0, 11.0, 2.0, 13.0], oracles) == [] + assert matching_vector(first, oracles) == {"workload_sha256": "first", "copies": 1, "request_position": 0} + assert matching_vector([0.0, 11.0, 2.0, 13.0], oracles) is None def test_hidden_retry_rejected_and_malformed_error_retained() -> None: diff --git a/tools/benchmarks/cross_encoder_lora/tests/test_runner_budget.py b/tools/benchmarks/cross_encoder_lora/tests/test_runner_budget.py index 7f9810410..f755de8e1 100644 --- a/tools/benchmarks/cross_encoder_lora/tests/test_runner_budget.py +++ b/tools/benchmarks/cross_encoder_lora/tests/test_runner_budget.py @@ -116,7 +116,7 @@ def start(command: list[str], **_kwargs: Any) -> HungProcess: "arm": "R", "passed": False, "shutdown_complete": False, - "performance": {"1": {"completed_samples": 7}}, + "performance": {"1": {"count": 7}}, } ) ) @@ -132,6 +132,6 @@ def start(command: list[str], **_kwargs: Any) -> HungProcess: terminal = json.loads((args.output / "summary.json").read_text()) assert not terminal["passed"] trial = terminal["trials"][0] - assert trial["performance"]["1"]["completed_samples"] == 7 + assert trial["performance"]["1"]["count"] == 7 assert trial["process_returncode"] == -9 assert trial["runner_error"]["type"] == "TimeoutExpired" diff --git a/tools/benchmarks/cross_encoder_lora/worker.py b/tools/benchmarks/cross_encoder_lora/worker.py index b8dcd9c8a..33269264a 100644 --- a/tools/benchmarks/cross_encoder_lora/worker.py +++ b/tools/benchmarks/cross_encoder_lora/worker.py @@ -328,15 +328,21 @@ def unobserved(adapter: Any) -> Iterator[None]: ) -def matching_vectors(scores: list[float], oracles: dict[str, Any]) -> list[dict[str, Any]]: - matches = [] - for key, oracle_row in oracles.items(): +def matching_vector(scores: list[float], oracles: dict[str, Any]) -> dict[str, Any] | None: + """Return one deterministic diagnostic witness, never actual batch attribution. + + The full oracle corpus and raw score vector are retained separately, so all + matches (including ambiguity) can be recomputed without repeating up to 136 + equivalent descriptions on every response. + """ + for key in sorted(oracles): + oracle_row = oracles[key] copies = oracle_row["copies"] for position in range(copies): vector = [oracle_row["scores"][ORDER.index(index) * copies + position] for index in range(REQUEST_ITEMS)] if vector_delta(scores, vector) <= RESTORE_ATOL: - matches.append({"workload_sha256": key, "copies": copies, "request_position": position}) - return matches + return {"workload_sha256": key, "copies": copies, "request_position": position} + return None def performance(url: str, concurrency: int, evidence: Evidence, *, smoke: bool) -> dict[str, Any]: @@ -727,13 +733,13 @@ def concurrent(index: int) -> dict[str, Any]: require(row["result"]["usage"] == bases[0]["result"]["usage"], "timed SDK usage differs") if concurrency == 1: require(vector_delta(row["scores"], baseline) <= RESTORE_ATOL, "timed C1 base output differs") - row["diagnostic_matching_oracles"] = matching_vectors(row["scores"], summary["shape_oracles"]) + row["diagnostic_matching_oracle"] = matching_vector(row["scores"], summary["shape_oracles"]) measured["actual_timed_batch_shape"] = "unobserved" evidence.emit( "timed_output_diagnostics", concurrency=concurrency, - matches=[ - {"label": row["label"], "matches": row["diagnostic_matching_oracles"]} + witnesses=[ + {"label": row["label"], "match": row["diagnostic_matching_oracle"]} for row in measured["samples"] ], ) From ac29fa1595711af0e01e2b7c421f1600aa4087a6 Mon Sep 17 00:00:00 2001 From: svonava Date: Fri, 16 Oct 2026 18:29:50 -0700 Subject: [PATCH 05/16] fix(cross-encoder): detach resident adapters from base execution --- .../adapters/cross_encoder/_lora.py | 59 +++++- .../tests/adapters/test_cross_encoder_lora.py | 190 +++++++++++++++++- 2 files changed, 241 insertions(+), 8 deletions(-) diff --git a/packages/sie_server/src/sie_server/adapters/cross_encoder/_lora.py b/packages/sie_server/src/sie_server/adapters/cross_encoder/_lora.py index e3d330407..86285178e 100644 --- a/packages/sie_server/src/sie_server/adapters/cross_encoder/_lora.py +++ b/packages/sie_server/src/sie_server/adapters/cross_encoder/_lora.py @@ -3,7 +3,8 @@ The outer wrapper, tokenizer, classifier and activation are never replaced. Transformers injects ordinary LoRA layers into declared BERT encoder Linear modules. The worker owns selection/mutation exclusion; only background source -preparation needs a separate lock here. No scoring code calls into this module. +preparation needs a separate lock here. Resident layers stay outside the model's +module tree while base is selected. No scoring code calls into this module. """ from __future__ import annotations @@ -21,6 +22,7 @@ from huggingface_hub import snapshot_download from peft import LoraConfig, PeftConfig from peft.functional import delete_adapter +from peft.tuners.lora.layer import Linear as LoraLinear from safetensors.torch import load_file from sentence_transformers import CrossEncoder from transformers import BertForSequenceClassification @@ -92,6 +94,10 @@ def __init__( self._outer: CrossEncoder | None = None self._model: Any = None self._originals: dict[str, torch.nn.Linear] = {} + # An ordinary dict deliberately keeps resident adapters outside nn.Module's + # traversal while base is selected. CrossEncoder.predict calls to()/eval() + # on every batch, even when the model is already on the requested device. + self._resident_layers: dict[str, LoraLinear] = {} self._grad_flags: list[tuple[torch.nn.Parameter, bool]] = [] self._closed = True self._uncertain = False @@ -237,6 +243,42 @@ def _restore_grad_flags(self) -> None: for parameter, requires_grad in self._grad_flags: parameter.requires_grad_(requires_grad) + def _attach_layers(self) -> None: + """Expose all resident layers only during a guarded adapter operation.""" + for name, layer in self._resident_layers.items(): + original = self._originals[name] + current = self._model.get_submodule(name) + if current is not original and current is not layer: + raise RuntimeError(f"CrossEncoder module {name!r} changed outside the LoRA lifecycle") + if layer.get_base_layer() is not original: + raise RuntimeError(f"CrossEncoder LoRA base layer {name!r} changed") + # Serving keeps the model's device and precision fixed after loading. + # Do not silently migrate detached weights or invalidate their memory + # reservation if an external caller moves or casts the base model. + for parameter in layer.parameters(): + if parameter.device != original.weight.device or parameter.dtype != original.weight.dtype: + raise RuntimeError("CrossEncoder device or precision changed with resident LoRAs") + parent_name, _, child = name.rpartition(".") + setattr(self._model.get_submodule(parent_name), child, layer) + + def _detach_layers(self) -> None: + """Restore the exact base forward/tree while retaining adapter residency.""" + for name, original in self._originals.items(): + current = self._model.get_submodule(name) + if current is original: + continue + if not isinstance(current, LoraLinear) or current.get_base_layer() is not original: + raise RuntimeError(f"CrossEncoder module {name!r} is not its owned LoRA layer") + if current.lora_A: + self._resident_layers[name] = current + else: + # Upstream deletion leaves empty tuner layers in the tree. A + # disjoint-target eviction must release those wrappers too. + self._resident_layers.pop(name, None) + parent_name, _, child = name.rpartition(".") + setattr(self._model.get_submodule(parent_name), child, original) + self._active = None + def _restore_tree(self) -> None: for name, original in self._originals.items(): parent_name, _, child = name.rpartition(".") @@ -244,6 +286,7 @@ def _restore_tree(self) -> None: if hasattr(self._model, "peft_config"): del self._model.peft_config self._model._hf_peft_config_loaded = False + self._resident_layers.clear() self._restore_grad_flags() self._active = None @@ -269,6 +312,7 @@ def load(self, name: str, revision: str | None = None, *, source: str | None = N adapter_name = PEFTLoRAMixin._peft_adapter_name(canonical) try: assert self._outer is not None + self._attach_layers() self._outer.load_adapter( adapter_name=adapter_name, peft_config=prepared.config, @@ -291,17 +335,20 @@ def load(self, name: str, revision: str | None = None, *, source: str | None = N memory += parameter.numel() * parameter.element_size() self._outer.disable_adapters() self._restore_grad_flags() - self._active = None + self._detach_layers() self._loaded[canonical] = memory self._parameters[canonical] = parameters except BaseException: try: if self._loaded: + # A failed detach may have already saved part of the new + # injection off-tree. Include it in rollback deletion too. + self._attach_layers() self._remove(adapter_name) assert self._outer is not None self._outer.disable_adapters() self._restore_grad_flags() - self._active = None + self._detach_layers() else: self._restore_tree() except BaseException as rollback: @@ -327,7 +374,9 @@ def select(self, name: str | None) -> None: try: if canonical is None: self._outer.disable_adapters() + self._detach_layers() else: + self._attach_layers() self._outer.set_adapter(PEFTLoRAMixin._peft_adapter_name(canonical)) self._outer.enable_adapters() # Transformers' enable/set hooks default to trainable adapters. @@ -345,6 +394,7 @@ def unload(self, name: str) -> None: if canonical not in self._loaded: raise LoraUnavailableError(f"LoRA {name!r} is not resident") try: + self._attach_layers() self._remove(PEFTLoRAMixin._peft_adapter_name(canonical)) if len(self._loaded) == 1: self._restore_tree() @@ -352,7 +402,7 @@ def unload(self, name: str) -> None: assert self._outer is not None self._outer.disable_adapters() self._restore_grad_flags() - self._active = None + self._detach_layers() except BaseException as exc: self._uncertain = True raise LoraStateUncertainError("Failed to unload CrossEncoder LoRA; reload the model") from exc @@ -376,6 +426,7 @@ def close(self) -> None: finally: self._loaded.clear() self._parameters.clear() + self._resident_layers.clear() self._originals.clear() self._grad_flags.clear() self._outer = self._model = None diff --git a/packages/sie_server/tests/adapters/test_cross_encoder_lora.py b/packages/sie_server/tests/adapters/test_cross_encoder_lora.py index 59c19fbdd..26bd47253 100644 --- a/packages/sie_server/tests/adapters/test_cross_encoder_lora.py +++ b/packages/sie_server/tests/adapters/test_cross_encoder_lora.py @@ -3,8 +3,10 @@ from __future__ import annotations import builtins +import gc import json import time +import weakref from concurrent.futures import ThreadPoolExecutor from pathlib import Path @@ -60,13 +62,13 @@ def checkpoint(tmp_path_factory): return directory -def save_adapter(path, seed=1, **overrides): +def save_adapter(path, seed=1, *, targets=("query", "value"), **overrides): path.mkdir() LoraConfig( task_type="SEQ_CLS", r=4, lora_alpha=8, - target_modules=["query", "value"], + target_modules=list(targets), bias="none", lora_dropout=0, inference_mode=True, @@ -79,7 +81,7 @@ def save_adapter(path, seed=1, **overrides): generator = torch.Generator().manual_seed(seed) weights = {} for layer in range(2): - for target in ("query", "value"): + for target in targets: prefix = f"base_model.model.bert.encoder.layer.{layer}.attention.self.{target}" weights[f"{prefix}.lora_A.weight"] = torch.randn(4, 16, generator=generator) * 0.4 weights[f"{prefix}.lora_B.weight"] = torch.randn(16, 4, generator=generator) * 0.4 @@ -210,6 +212,173 @@ def test_actual_effect_restoration_head_and_final_module_identity(adapter): np.testing.assert_array_equal(scores(adapter), a) +def test_resident_adapters_leave_the_exact_base_tree_and_preserve_layer_hooks(adapter): + snapshot = base_snapshot(adapter) + baseline = scores(adapter) + original_query = snapshot[1].bert.encoder.layer[0].attention.self.query + original_calls, wrapper_calls = [], [] + base_hook = original_query.register_forward_hook(lambda *_: original_calls.append(True)) + adapter.load_lora("a") + manager = adapter._lora + assert dict(snapshot[1].named_modules()) == snapshot[5] + wrapper = manager._resident_layers["bert.encoder.layer.0.attention.self.query"] + wrapper_hook = wrapper.register_forward_hook(lambda *_: wrapper_calls.append(True)) + adapter.load_lora("b") + identities = {name: id(layer) for name, layer in manager._resident_layers.items()} + assert dict(snapshot[1].named_modules()) == snapshot[5] + assert len(manager._loaded) == 2 + assert sum(manager._loaded.values()) == 4096 + expected = {None: baseline} + for selection in ("a", None, "b", None, "a", None, "b", None): + old_wrapper_calls = len(wrapper_calls) + old_original_calls = len(original_calls) + adapter.set_active_lora(selection) + actual = scores(adapter) + assert len(original_calls) > old_original_calls + if selection is None: + assert dict(snapshot[1].named_modules()) == snapshot[5] + assert len(wrapper_calls) == old_wrapper_calls + else: + assert len(wrapper_calls) > old_wrapper_calls + assert all(id(snapshot[1].get_submodule(name)) == identity for name, identity in identities.items()) + assert np.max(np.abs(actual - baseline)) > 1e-6 + if selection in expected: + np.testing.assert_array_equal(actual, expected[selection]) + expected[selection] = actual + assert_base_preserved(adapter, snapshot) + assert {name: id(layer) for name, layer in manager._resident_layers.items()} == identities + assert np.max(np.abs(expected["a"] - expected["b"])) > 1e-6 + base_hook.remove() + wrapper_hook.remove() + + +def test_disjoint_targets_evict_only_their_wrappers_and_final_unload_releases_weights(checkpoint, tmp_path): + paths = { + "a": save_adapter(tmp_path / "a", 1, targets=("query",)), + "b": save_adapter(tmp_path / "b", 2, targets=("value",)), + } + adapter = CrossEncoderAdapter(checkpoint, lora_paths=paths, max_loaded_loras=2) + adapter.load("cpu") + try: + snapshot, baseline = base_snapshot(adapter), scores(adapter) + adapter.load_lora("a") + adapter.set_active_lora("a") + a = scores(adapter) + adapter.load_lora("b") + manager = adapter._lora + refs = {name: weakref.ref(layer) for name, layer in manager._resident_layers.items()} + parameter_refs = [weakref.ref(p) for parameters in manager._parameters.values() for p in parameters] + assert dict(snapshot[1].named_modules()) == snapshot[5] + assert set(refs) == { + f"bert.encoder.layer.{i}.attention.self.{target}" for i in range(2) for target in ("query", "value") + } + adapter.set_active_lora("b") + b = scores(adapter) + assert np.max(np.abs(a - baseline)) > 1e-6 + assert np.max(np.abs(b - baseline)) > 1e-6 + adapter.set_active_lora(None) + np.testing.assert_array_equal(scores(adapter), baseline) + adapter.unload_lora("a") + gc.collect() + assert all(ref() is None for name, ref in refs.items() if name.endswith("query")) + assert all(ref() is not None for name, ref in refs.items() if name.endswith("value")) + assert dict(snapshot[1].named_modules()) == snapshot[5] + assert sum(manager._loaded.values()) == 1024 + adapter.set_active_lora("b") + np.testing.assert_array_equal(scores(adapter), b) + adapter.unload_lora("b") + gc.collect() + assert all(ref() is None for ref in refs.values()) + assert all(ref() is None for ref in parameter_refs) + assert not manager._resident_layers + assert_base_preserved(adapter, snapshot, unwrapped=True) + adapter.load_lora("a") + adapter.set_active_lora("a") + np.testing.assert_array_equal(scores(adapter), a) + finally: + adapter.unload() + + +@pytest.mark.parametrize("operation", ["attach", "detach"]) +def test_partial_tree_swap_failure_poisoning_preserves_residency_until_teardown(adapter, monkeypatch, operation): + snapshot = base_snapshot(adapter) + adapter.load_lora("a") + adapter.load_lora("b") + manager = adapter._lora + if operation == "detach": + adapter.set_active_lora("a") + refs = [weakref.ref(layer) for layer in manager._resident_layers.values()] + + def partial_swap(): + name, layer = next(iter(manager._resident_layers.items())) + parent_name, _, child = name.rpartition(".") + setattr( + manager._model.get_submodule(parent_name), + child, + layer if operation == "attach" else manager._originals[name], + ) + raise RuntimeError("injected partial tree swap") + + with monkeypatch.context() as context: + context.setattr(manager, f"_{operation}_layers", partial_swap) + with pytest.raises(LoraStateUncertainError, match="Failed to select"): + adapter.set_active_lora("a" if operation == "attach" else None) + assert len(manager._loaded) == 2 + assert sum(manager._loaded.values()) == 4096 + with pytest.raises(LoraStateUncertainError, match="uncertain"): + adapter.set_active_lora(None) + adapter.unload() + gc.collect() + assert all(ref() is None for ref in refs) + assert dict(snapshot[1].named_modules()) == snapshot[5] + assert not manager._resident_layers + assert not manager._loaded + + +def test_external_precision_change_with_detached_adapters_fails_closed(adapter): + adapter.load_lora("a") + adapter._model.to(dtype=torch.float64) + with pytest.raises(LoraStateUncertainError, match="Failed to select"): + adapter.set_active_lora("a") + assert len(adapter._lora._loaded) == 1 + with pytest.raises(LoraStateUncertainError, match="uncertain"): + adapter.set_active_lora(None) + + +def test_partial_detach_failure_during_load_removes_all_failed_adapter_weights(adapter, monkeypatch): + snapshot, baseline = base_snapshot(adapter), scores(adapter) + adapter.load_lora("a") + adapter.set_active_lora("a") + a = scores(adapter) + manager = adapter._lora + original = manager._detach_layers + failed = False + + def fail_after_first_detached_layer(): + nonlocal failed + if failed: + return original() + failed = True + name, layer = next(iter(manager._resident_layers.items())) + parent_name, _, child = name.rpartition(".") + setattr(manager._model.get_submodule(parent_name), child, manager._originals[name]) + assert len(layer.lora_A) == 2 + raise RuntimeError("injected failure after detaching one updated layer") + + with monkeypatch.context() as context: + context.setattr(manager, "_detach_layers", fail_after_first_detached_layer) + with pytest.raises(RuntimeError, match="after detaching"): + adapter.load_lora("b") + assert len(manager._loaded) == 1 + assert all(len(layer.lora_A) == len(layer.lora_B) == 1 for layer in manager._resident_layers.values()) + assert dict(snapshot[1].named_modules()) == snapshot[5] + np.testing.assert_array_equal(scores(adapter), baseline) + adapter.set_active_lora("a") + np.testing.assert_array_equal(scores(adapter), a) + adapter.load_lora("b") + assert len(manager._loaded) == 2 + + @pytest.mark.parametrize("existing", [False, True]) def test_failed_attachment_rolls_back_entire_base_or_only_failed_adapter(adapter, monkeypatch, existing): snapshot = base_snapshot(adapter) @@ -229,6 +398,7 @@ def fail_after_injection(*args, **kwargs): with pytest.raises(RuntimeError, match="injected post-load"): adapter.load_lora("b") assert_base_preserved(adapter, snapshot, unwrapped=not existing) + assert dict(snapshot[1].named_modules()) == snapshot[5] np.testing.assert_array_equal(scores(adapter), baseline) assert adapter._lora._prepared == {} with pytest.raises(LoraUnavailableError): @@ -394,11 +564,20 @@ def reload_during_fetch(path): @pytest.mark.parametrize("existing", [False, True]) -def test_partial_injection_failure_preserves_old_adapters_and_base(adapter, monkeypatch, existing): +@pytest.mark.parametrize("disjoint", [False, True]) +def test_partial_injection_failure_preserves_old_adapters_and_base(adapter, monkeypatch, existing, disjoint): from peft.tuners.lora.model import LoraModel snapshot = base_snapshot(adapter) baseline = scores(adapter) + if disjoint: + for alias, target in (("a", "query"), ("b", "value")): + path = Path(adapter._lora._aliases[alias]) + config = json.loads((path / "adapter_config.json").read_text()) + config["target_modules"] = [target] + (path / "adapter_config.json").write_text(json.dumps(config)) + weights = {k: v for k, v in load_file(path / "adapter_model.safetensors").items() if f".{target}." in k} + save_file(weights, path / "adapter_model.safetensors") if existing: adapter.load_lora("a") adapter.set_active_lora("a") @@ -419,6 +598,9 @@ def fail_second_layer(self, *args, **kwargs): adapter.load_lora("b") assert calls == 2 assert_base_preserved(adapter, snapshot, unwrapped=not existing) + assert dict(snapshot[1].named_modules()) == snapshot[5] + if disjoint: + assert all(name.endswith("query") for name in adapter._lora._resident_layers) np.testing.assert_array_equal(scores(adapter), baseline) if existing: adapter.set_active_lora("a") From 7919cfd86a7138f1a097c88a6fdff04a17c895bb Mon Sep 17 00:00:00 2001 From: svonava Date: Fri, 16 Oct 2026 18:28:42 -0700 Subject: [PATCH 06/16] fix(benchmarks): observe direct cross-encoder forwards --- tools/benchmarks/cross_encoder_lora/README.md | 15 +- .../benchmarks/cross_encoder_lora/adapter.py | 157 +++++++++++++---- .../tests/test_forward_observer.py | 158 ++++++++++++++++++ .../cross_encoder_lora/tests/test_probe.py | 5 + tools/benchmarks/cross_encoder_lora/worker.py | 24 +-- 5 files changed, 304 insertions(+), 55 deletions(-) create mode 100644 tools/benchmarks/cross_encoder_lora/tests/test_forward_observer.py diff --git a/tools/benchmarks/cross_encoder_lora/README.md b/tools/benchmarks/cross_encoder_lora/README.md index 12530dc1f..eb0b40f3a 100644 --- a/tools/benchmarks/cross_encoder_lora/README.md +++ b/tools/benchmarks/cross_encoder_lora/README.md @@ -95,10 +95,17 @@ Production fault-injection tests separately cover malformed artifacts and partial-attachment rollback; this probe does not pretend to reproduce them on every GPU trial. -The hook records deep selection/logit details during correctness checks and -two observed SDK replay rounds at C1/C4/C16. During timed phases the forward -hook is removed and production `score_pairs` and `set_active_lora` are bound -directly. The harness asserts that no observer activity occurred. Response +The untimed observer wraps the model's bound `forward` method, because Sentence +Transformers calls that method directly and bypasses root module forward hooks. +Every observed outer score batch must have nonempty forward records, finite +logits and exact input-row coverage; selected/base tuner checks cannot pass on +an empty list. It records selection/logit details during correctness checks and +two observed SDK replay rounds at C1/C4/C16. During timed phases the original +forward attribute lookup is restored: an original instance override is restored +exactly, or the temporary instance attribute is deleted to restore descriptor +lookup. Production `score_pairs` and `set_active_lora` are bound directly. +The harness checks the original forward target and that no observer activity +occurred, including restoration after an exceptional exit. Response validation and header decoding happen after the phase clock stops; exactly one successful HTTP response is required per warmup/measured SDK call, excluding hidden retries. The timed client retains timer, result-list and raw-response diff --git a/tools/benchmarks/cross_encoder_lora/adapter.py b/tools/benchmarks/cross_encoder_lora/adapter.py index c1f41ac84..a9c6ce828 100644 --- a/tools/benchmarks/cross_encoder_lora/adapter.py +++ b/tools/benchmarks/cross_encoder_lora/adapter.py @@ -2,11 +2,49 @@ from __future__ import annotations +import math import time +from functools import wraps from typing import Any from sie_server.adapters.cross_encoder import CrossEncoderAdapter +from tools.benchmarks.cross_encoder_lora.protocol import require + +INPUT_DIMENSIONS = 2 + + +def require_forward_coverage(batch: dict[str, Any]) -> None: + """An outer score batch alone does not prove the model forward was observed.""" + forwards = batch["forwards"] + require(bool(forwards), "model forward observations are missing") + rows = 0 + for forward in forwards: + shape = forward["input_shape"] + require( + isinstance(shape, list) + and len(shape) == INPUT_DIMENSIONS + and all(isinstance(n, int) and n > 0 for n in shape), + "invalid observed forward input shape", + ) + logits = forward.get("logits") + require( + isinstance(logits, list) + and len(logits) == shape[0] + and all(isinstance(row, list) and len(row) == 1 and math.isfinite(row[0]) for row in logits), + "observed forward logits are missing, nonfinite or have the wrong shape", + ) + rows += shape[0] + require(rows == len(batch["item_ids"]), "observed forward rows do not cover the score batch") + + +def require_forward_selection(batch: dict[str, Any], expected_adapter: str | None) -> None: + require_forward_coverage(batch) + for forward in batch["forwards"]: + enabled = forward["enabled_adapters"] + require(bool(enabled) == bool(expected_adapter), "actual adapter enable state differs at model forward") + require(all(item["adapters"] == [expected_adapter] for item in enabled), "wrong PEFT adapter at forward") + class ObservedCrossEncoder(CrossEncoderAdapter): def load(self, device: str) -> None: @@ -14,30 +52,78 @@ def load(self, device: str) -> None: self.selection_observations = [] self.observed_selection = None self.observation_phase = "load" - self.capture_logits = True self.active_forward = False self.mutation_overlap = False self.forward_observations = [] self.fetch_observations = [] super().load(device) assert self._model is not None - self._observer_handle = self._model.model.register_forward_hook(self._observe_forward, with_kwargs=True) - - def _observe_forward(self, model: Any, _args: Any, kwargs: Any, result: Any) -> None: - enabled = [] - if self.capture_logits: - for name, module in model.named_modules(): - if hasattr(module, "lora_A") and not getattr(module, "disable_adapters", False): - enabled.append({"module": name, "adapters": list(module.active_adapters)}) - inputs = kwargs.get("input_ids") - row = { - "selection": self.observed_selection, - "enabled_adapters": enabled, - "input_shape": list(inputs.shape) if inputs is not None else None, - } - if self.capture_logits: + model = self._model.model + self._forward_had_override = "forward" in model.__dict__ + self._forward_override = model.__dict__.get("forward") + self._original_forward = model.forward + self._observer_wrapper = None + self.install_forward_observer() + + def assert_original_forward(self) -> None: + assert self._model is not None + model = self._model.model + if self._forward_had_override: + require(model.__dict__.get("forward") is self._forward_override, "original forward override changed") + else: + require("forward" not in model.__dict__, "forward descriptor lookup was not restored") + current = model.forward + original = self._original_forward + require( + getattr(current, "__self__", None) is getattr(original, "__self__", None) + and getattr(current, "__func__", current) is getattr(original, "__func__", original), + "original forward target changed", + ) + + def install_forward_observer(self) -> None: + assert self._model is not None + require(self._observer_wrapper is None, "forward observer is already installed") + self.assert_original_forward() + model = self._model.model + original = self._original_forward + assert callable(original) + + # Sentence Transformers calls model.forward directly, bypassing root + # Module.__call__ hooks. Preserve its signature and wrap that exact target. + @wraps(original) + def observed(*args: Any, **kwargs: Any) -> Any: + enabled = [ + {"module": name, "adapters": list(module.active_adapters)} + for name, module in model.named_modules() + if hasattr(module, "lora_A") and not getattr(module, "disable_adapters", False) + ] + inputs = kwargs.get("input_ids", args[0] if args else None) + row = { + "selection": self.observed_selection, + "enabled_adapters": enabled, + "input_shape": list(inputs.shape) if inputs is not None else None, + } + result = original(*args, **kwargs) row["logits"] = result.logits.detach().float().cpu().tolist() - self.forward_observations.append(row) + self.forward_observations.append(row) + return result + + self._observer_wrapper = observed + model.forward = observed + + def remove_forward_observer(self) -> None: + assert self._model is not None + model = self._model.model + if self._observer_wrapper is not None: + require(model.__dict__.get("forward") is self._observer_wrapper, "forward observer target changed") + if self._forward_had_override: + override = self._forward_override + assert callable(override) + model.forward = override + else: + del model.forward + self._observer_wrapper = None + self.assert_original_forward() def set_active_lora(self, lora_name: str | None) -> None: if self.active_forward: @@ -52,20 +138,20 @@ def score_pairs(self, queries: list, docs: list, **kwargs: Any) -> Any: self.active_forward = True try: result = super().score_pairs(queries, docs, **kwargs) - self.batch_observations.append( - { - "start_ns": started, - "end_ns": time.time_ns(), - "selection": self.observed_selection, - "phase": self.observation_phase, - "queries": [item.text for item in queries], - "documents": [item.text for item in docs], - "item_ids": [item.id for item in docs], - "scores": result.scores.tolist(), - "input_token_counts": result.input_token_counts, - "forwards": self.forward_observations[first_forward:], - } - ) + batch = { + "start_ns": started, + "end_ns": time.time_ns(), + "selection": self.observed_selection, + "phase": self.observation_phase, + "queries": [item.text for item in queries], + "documents": [item.text for item in docs], + "item_ids": [item.id for item in docs], + "scores": result.scores.tolist(), + "input_token_counts": result.input_token_counts, + "forwards": self.forward_observations[first_forward:], + } + self.batch_observations.append(batch) + require_forward_coverage(batch) return result finally: self.active_forward = False @@ -87,8 +173,9 @@ def unload_lora(self, *args: Any, **kwargs: Any) -> None: super().unload_lora(*args, **kwargs) def unload(self) -> None: - handle = getattr(self, "_observer_handle", None) - if handle is not None: - handle.remove() - self._observer_handle = None + if getattr(self, "_original_forward", None) is not None: + self.remove_forward_observer() + # Bound methods and closures otherwise keep the unloaded model alive. + self._original_forward = None + self._forward_override = None super().unload() diff --git a/tools/benchmarks/cross_encoder_lora/tests/test_forward_observer.py b/tools/benchmarks/cross_encoder_lora/tests/test_forward_observer.py new file mode 100644 index 000000000..46c4d0f25 --- /dev/null +++ b/tools/benchmarks/cross_encoder_lora/tests/test_forward_observer.py @@ -0,0 +1,158 @@ +"""Real Sentence Transformers forwarding and exact timed-path restoration.""" + +from __future__ import annotations + +import copy +from contextlib import nullcontext +from functools import wraps +from pathlib import Path + +import pytest +from sie_server.adapters.cross_encoder import CrossEncoderAdapter +from sie_server.types.inputs import Item + +from tools.benchmarks.cross_encoder_lora.adapter import ( + ObservedCrossEncoder, + require_forward_coverage, + require_forward_selection, +) +from tools.benchmarks.cross_encoder_lora.tests.test_probe import tiny_snapshot +from tools.benchmarks.cross_encoder_lora.worker import unobserved + + +def pairs(count: int = 4) -> tuple[list[Item], list[Item]]: + return ( + [Item(id=f"q-{index}", text="how do i reset my password ?") for index in range(count)], + [Item(id=f"d-{index}", text="use the password reset link .") for index in range(count)], + ) + + +def test_real_predict_bypasses_root_hook_but_bound_forward_covers_every_row(tmp_path: Path) -> None: + adapter = ObservedCrossEncoder(tiny_snapshot(tmp_path / "model")) + adapter.load("cpu") + try: + assert adapter._model is not None + model = adapter._model.model + hook_calls = [] + handle = model.register_forward_hook(lambda *_args: hook_calls.append(True)) + try: + # This reproduces the old observer's failure using the actual + # Sentence Transformers predict path, not a fake direct call. + with unobserved(adapter): + result = adapter.score_pairs(*pairs(70)) + assert len(result.scores) == 70 + assert not hook_calls + assert not adapter.forward_observations + with pytest.raises(AssertionError, match="observations are missing"): + require_forward_coverage({"item_ids": list(range(70)), "forwards": []}) + adapter.score_pairs(*pairs(70)) + batch = adapter.batch_observations[-1] + require_forward_selection(batch, None) + assert len(batch["forwards"]) > 1 # predict internally chunks the outer batch. + assert sum(row["input_shape"][0] for row in batch["forwards"]) == 70 + assert not hook_calls + finally: + handle.remove() + finally: + adapter.unload() + + +@pytest.mark.parametrize("instance_override", [False, True]) +@pytest.mark.parametrize("fail_inside_timing", [False, True]) +def test_timing_restores_exact_original_lookup_and_cleanup( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, instance_override: bool, fail_inside_timing: bool +) -> None: + original_load = CrossEncoderAdapter.load + original_attribute = {} + + def load(adapter, device): + original_load(adapter, device) + model = adapter._model.model + if instance_override: + target = model.forward + + @wraps(target) + def original_override(*args, **kwargs): + return target(*args, **kwargs) + + model.forward = original_override + original_attribute["had_override"] = "forward" in model.__dict__ + original_attribute["value"] = model.__dict__.get("forward") + original_attribute["target"] = model.forward + + monkeypatch.setattr(CrossEncoderAdapter, "load", load) + adapter = ObservedCrossEncoder(tiny_snapshot(tmp_path / "model")) + adapter.load("cpu") + assert adapter._model is not None + model = adapter._model.model + + def assert_original(): + assert ("forward" in model.__dict__) == original_attribute["had_override"] + if instance_override: + assert model.__dict__["forward"] is original_attribute["value"] + current, target = model.forward, original_attribute["target"] + assert getattr(current, "__self__", None) is getattr(target, "__self__", None) + assert getattr(current, "__func__", current) is getattr(target, "__func__", target) + + try: + adapter.score_pairs(*pairs()) + before = ( + len(adapter.batch_observations), + len(adapter.forward_observations), + len(adapter.selection_observations), + ) + expected_error = ( + pytest.raises(RuntimeError, match="injected timed-phase failure") if fail_inside_timing else nullcontext() + ) + with expected_error: + with unobserved(adapter): + assert_original() + assert adapter.score_pairs.__func__ is CrossEncoderAdapter.score_pairs + assert adapter.set_active_lora.__func__ is CrossEncoderAdapter.set_active_lora + adapter.set_active_lora(None) + adapter.score_pairs(*pairs()) + assert_original() + if fail_inside_timing: + raise RuntimeError("injected timed-phase failure") + assert before == ( + len(adapter.batch_observations), + len(adapter.forward_observations), + len(adapter.selection_observations), + ) + assert model.__dict__["forward"] is adapter._observer_wrapper + adapter.score_pairs(*pairs()) + assert len(adapter.forward_observations) > before[1] + finally: + adapter.unload() + assert_original() + assert adapter._original_forward is None + assert adapter._forward_override is None + assert adapter._observer_wrapper is None + + +def observed_batch() -> dict: + return { + "item_ids": ["0", "1"], + "forwards": [{"input_shape": [2, 8], "logits": [[0.1], [0.2]], "enabled_adapters": []}], + } + + +@pytest.mark.parametrize("failure", ["empty", "rows", "logits", "nonfinite", "wrong_adapter", "enabled_base"]) +def test_missing_partial_or_wrong_tuner_observation_cannot_qualify(failure: str) -> None: + batch = copy.deepcopy(observed_batch()) + expected = None + if failure == "empty": + batch["forwards"] = [] + elif failure == "rows": + batch["item_ids"].append("2") + elif failure == "logits": + batch["forwards"][0]["logits"] = [[0.1]] + elif failure == "nonfinite": + batch["forwards"][0]["logits"][0][0] = float("nan") + elif failure == "wrong_adapter": + expected = "wanted" + batch["forwards"][0]["enabled_adapters"] = [{"module": "query", "adapters": ["other"]}] + else: + batch["forwards"][0]["enabled_adapters"] = [{"module": "query", "adapters": ["unexpected"]}] + with pytest.raises(AssertionError): + require_forward_selection(batch, expected) diff --git a/tools/benchmarks/cross_encoder_lora/tests/test_probe.py b/tools/benchmarks/cross_encoder_lora/tests/test_probe.py index 5fed4ea86..8864463da 100644 --- a/tools/benchmarks/cross_encoder_lora/tests/test_probe.py +++ b/tools/benchmarks/cross_encoder_lora/tests/test_probe.py @@ -17,6 +17,7 @@ from transformers import BertConfig, BertForSequenceClassification, BertTokenizerFast from tools.benchmarks.cross_encoder_lora import prepare as source_preparer +from tools.benchmarks.cross_encoder_lora.adapter import require_forward_coverage from tools.benchmarks.cross_encoder_lora.evidence import Evidence, atomic_json, vector_delta from tools.benchmarks.cross_encoder_lora.fixtures import prepare from tools.benchmarks.cross_encoder_lora.protocol import BASE_REVISION, ORDERS, compare, paired_gate, percentile @@ -179,6 +180,10 @@ def test_real_loader_sdk_lifecycle(tmp_path: Path, arm: str) -> None: assert Path(summary["environment"]["adapter_source"]).is_relative_to(source) events = [json.loads(line) for line in (output / "events.jsonl").read_text().splitlines()] assert {"shape_oracles", "performance", "finished"} <= {event["event"] for event in events} + batches = next(event["batches"] for event in events if event["event"] == "all_batches") + assert batches + for batch in batches: + require_forward_coverage(batch) if arm == "R": assert {"effects", "concurrent_batches", "resident_set", "restored_state"} <= { event["event"] for event in events diff --git a/tools/benchmarks/cross_encoder_lora/worker.py b/tools/benchmarks/cross_encoder_lora/worker.py index 33269264a..db24727d9 100644 --- a/tools/benchmarks/cross_encoder_lora/worker.py +++ b/tools/benchmarks/cross_encoder_lora/worker.py @@ -38,6 +38,7 @@ from sie_server.app.app_state_config import AppStateConfig from sie_server.types.inputs import Item +from tools.benchmarks.cross_encoder_lora.adapter import require_forward_selection from tools.benchmarks.cross_encoder_lora.evidence import Evidence, atomic_json, digest, fingerprint, vector_delta from tools.benchmarks.cross_encoder_lora.protocol import ( CONCURRENCIES, @@ -302,11 +303,10 @@ def validate_response(row: dict[str, Any], *, allow_loading: bool = False) -> No @contextmanager def unobserved(adapter: Any) -> Iterator[None]: - """Time production methods directly; no model hook, copying or observer lock.""" + """Time production methods with the original forward attribute lookup restored.""" score_pairs, select = adapter.score_pairs, adapter.set_active_lora before = (len(adapter.batch_observations), len(adapter.forward_observations), len(adapter.selection_observations)) - adapter._observer_handle.remove() - adapter._observer_handle = None + adapter.remove_forward_observer() adapter.score_pairs = CrossEncoderAdapter.score_pairs.__get__(adapter) adapter.set_active_lora = CrossEncoderAdapter.set_active_lora.__get__(adapter) try: @@ -314,7 +314,9 @@ def unobserved(adapter: Any) -> Iterator[None]: require( adapter.set_active_lora.__func__ is CrossEncoderAdapter.set_active_lora, "timed selection is instrumented" ) + adapter.assert_original_forward() yield + adapter.assert_original_forward() after = ( len(adapter.batch_observations), len(adapter.forward_observations), @@ -323,9 +325,7 @@ def unobserved(adapter: Any) -> Iterator[None]: require(before == after, "observer activity occurred during production timing") finally: adapter.score_pairs, adapter.set_active_lora = score_pairs, select - adapter._observer_handle = adapter._model.model.register_forward_hook( - adapter._observe_forward, with_kwargs=True - ) + adapter.install_forward_observer() def matching_vector(scores: list[float], oracles: dict[str, Any]) -> dict[str, Any] | None: @@ -677,12 +677,7 @@ def concurrent(index: int) -> dict[str, Any]: "concurrent selected scores differ from shape-matched oracle", ) expected_peft = PEFTLoRAMixin._peft_adapter_name(source) if source else None - for forward in batch["forwards"]: - enabled = forward["enabled_adapters"] - require(bool(enabled) == bool(alias), "actual adapter enable state differs at model forward") - require( - all(item["adapters"] == [expected_peft] for item in enabled), "wrong PEFT adapter at forward" - ) + require_forward_selection(batch, expected_peft) if len({item.split(":")[0] for item in batch["item_ids"]}) > 1: combined.add(alias) require(combined == {None, "a", "b"}, "did not observe cross-request batches for base, A and B") @@ -710,10 +705,7 @@ def concurrent(index: int) -> dict[str, Any]: "untimed SDK replay differs from preselection shape oracle", ) require(batch["selection"] is None, "untimed base replay selected a LoRA") - require( - all(not forward["enabled_adapters"] for forward in batch["forwards"]), - "untimed base replay left a PEFT adapter enabled", - ) + require_forward_selection(batch, None) if concurrency > 1: require( any(len({item.split(":")[0] for item in batch["item_ids"]}) > 1 for batch in batches), From f06f414a989eabb034145eaaeab8035f87273f6a Mon Sep 17 00:00:00 2001 From: svonava Date: Fri, 16 Oct 2026 18:33:55 -0700 Subject: [PATCH 07/16] fix(benchmarks): account for detached resident LoRA tensors --- .../cross_encoder_lora/tests/test_storage.py | 60 +++++++++++++++++++ tools/benchmarks/cross_encoder_lora/worker.py | 34 +++++++++-- 2 files changed, 90 insertions(+), 4 deletions(-) create mode 100644 tools/benchmarks/cross_encoder_lora/tests/test_storage.py diff --git a/tools/benchmarks/cross_encoder_lora/tests/test_storage.py b/tools/benchmarks/cross_encoder_lora/tests/test_storage.py new file mode 100644 index 000000000..976d953a7 --- /dev/null +++ b/tools/benchmarks/cross_encoder_lora/tests/test_storage.py @@ -0,0 +1,60 @@ +"""Real detached/attached PEFT storage accounting and inconsistent-state controls.""" + +from __future__ import annotations + +import pytest +import torch +from sie_server.adapters.cross_encoder import CrossEncoderAdapter + +from tools.benchmarks.cross_encoder_lora.fixtures import prepare +from tools.benchmarks.cross_encoder_lora.tests.test_probe import tiny_snapshot +from tools.benchmarks.cross_encoder_lora.worker import adapter_storage + + +@pytest.fixture +def resident(tmp_path): + snapshot = tiny_snapshot(tmp_path / "model") + fixtures = tmp_path / "fixtures" + prepare(fixtures, snapshot, smoke=True) + adapter = CrossEncoderAdapter(snapshot, lora_paths={name: str(fixtures / name) for name in ("a", "b")}) + adapter.load("cpu") + adapter.load_lora("a") + adapter.load_lora("b") + yield adapter + adapter.unload() + + +def test_storage_covers_detached_residents_without_double_counting_attached_layers(resident): + assert not any(".lora_" in name for name, _ in resident._model.model.named_parameters()) + detached = adapter_storage(resident) + assert detached == { + "resident_adapters": 2, + "storage_count": 16, + "tensor_storage_bytes": 4096, + "rounded_storage_bytes": 8192, + } + resident.set_active_lora("a") + assert adapter_storage(resident) == detached + resident.set_active_lora("b") + assert adapter_storage(resident) == detached + resident.set_active_lora(None) + assert adapter_storage(resident) == detached + resident.unload_lora("a") + assert adapter_storage(resident) == {name: value // 2 for name, value in detached.items()} + resident.unload_lora("b") + assert adapter_storage(resident) == dict.fromkeys(detached, 0) + + +@pytest.mark.parametrize("damage", ["source", "bytes", "orphan"]) +def test_storage_refuses_incomplete_resident_accounting(resident, damage): + manager = resident._lora + source = next(iter(manager._loaded)) + if damage == "source": + del manager._parameters[source] + elif damage == "bytes": + manager._loaded[source] += 1 + else: + layer = next(iter(manager._resident_layers.values())) + layer.lora_A["unaccounted"] = torch.nn.Linear(16, 4, bias=False) + with pytest.raises(AssertionError, match="resident"): + adapter_storage(resident) diff --git a/tools/benchmarks/cross_encoder_lora/worker.py b/tools/benchmarks/cross_encoder_lora/worker.py index db24727d9..625706641 100644 --- a/tools/benchmarks/cross_encoder_lora/worker.py +++ b/tools/benchmarks/cross_encoder_lora/worker.py @@ -168,12 +168,34 @@ def base_unchanged(before: dict[str, Any], after: dict[str, Any], *, restored: b def adapter_storage(adapter: Any) -> dict[str, int]: + """Inspect actual resident tensors outside timing, including detached layers.""" + parameters = {id(tensor): tensor for name, tensor in adapter._model.model.named_parameters() if ".lora_" in name} + manager = getattr(adapter, "_lora", None) + resident_count = 0 + if manager is not None: + require(set(manager._parameters) == set(manager._loaded), "resident parameter sources differ from admission") + declared = {} + for source, values in manager._parameters.items(): + require(bool(values), "resident adapter has no parameter evidence") + require( + sum(value.numel() * value.element_size() for value in values) == manager._loaded[source], + "resident parameter bytes differ from admission", + ) + declared.update({id(value): value for value in values}) + for layer in getattr(manager, "_resident_layers", {}).values(): + parameters.update( + {id(value): value for name, value in layer.named_parameters() if name.startswith("lora_")} + ) + require(set(parameters) == set(declared), "actual resident tensors differ from declared parameter ownership") + resident_count = len(manager._loaded) + else: + require(not parameters, "a model without a manager contains LoRA tensors") storages = {} - for name, tensor in adapter._model.model.named_parameters(): - if ".lora_" in name: - storage = tensor.untyped_storage() - storages[(str(tensor.device), storage.data_ptr())] = storage.nbytes() + for tensor in parameters.values(): + storage = tensor.untyped_storage() + storages[(str(tensor.device), storage.data_ptr())] = storage.nbytes() return { + "resident_adapters": resident_count, "storage_count": len(storages), "tensor_storage_bytes": sum(storages.values()), "rounded_storage_bytes": sum(math.ceil(size / 512) * 512 for size in storages.values()), @@ -687,6 +709,10 @@ def concurrent(index: int) -> dict[str, Any]: await call(None, "before-performance-base") require(set(registry.loaded_lora_names(MODEL)) == {"a", "b"}, "resident set before R measurement differs") summary["adapter_storage"] = adapter_storage(adapter) + require( + summary["adapter_storage"]["resident_adapters"] == RESIDENT_CAPACITY, + "storage evidence does not cover every resident adapter", + ) base_unchanged(original, await asyncio.to_thread(tensor_state, adapter)) summary["untimed_replays"] = {} From 482b599224920a97bb2fce4a740317efb89351ab Mon Sep 17 00:00:00 2001 From: svonava Date: Fri, 16 Oct 2026 18:54:02 -0700 Subject: [PATCH 08/16] test(benchmarks): add baseline stability diagnostic --- .../cross_encoder_lora/stability.py | 945 ++++++++++++++++++ .../tests/test_stability.py | 499 +++++++++ 2 files changed, 1444 insertions(+) create mode 100644 tools/benchmarks/cross_encoder_lora/stability.py create mode 100644 tools/benchmarks/cross_encoder_lora/tests/test_stability.py diff --git a/tools/benchmarks/cross_encoder_lora/stability.py b/tools/benchmarks/cross_encoder_lora/stability.py new file mode 100644 index 000000000..a4d5f05c5 --- /dev/null +++ b/tools/benchmarks/cross_encoder_lora/stability.py @@ -0,0 +1,945 @@ +"""Baseline-only stability diagnostic; never qualifies a candidate or provisions compute. + +Run with --inputs, --fixtures and --output. Six fresh baseline processes each +execute C1/C4/C16 with fixed S/L1/L2 windows: 30/200/200 sequential warmup +requests followed by the existing 200-request/12-second measurement. Later +window agreement is tested symmetrically; between-process dispersion remains +descriptive. The original timed methods and automatic GC behavior are reused. +""" + +# Only this checkout's Python worker runs; no shell or allocation. +# ruff: noqa: S603 + +from __future__ import annotations + +import argparse +import asyncio +import gc +import json +import math +import os +import resource +import signal +import socket +import statistics +import subprocess +import sys +import time +from http import HTTPStatus +from pathlib import Path +from typing import Any + +import torch +import uvicorn + +from tools.benchmarks.cross_encoder_lora import worker +from tools.benchmarks.cross_encoder_lora.adapter import require_forward_selection +from tools.benchmarks.cross_encoder_lora.evidence import Evidence, atomic_json, digest, fingerprint, vector_delta +from tools.benchmarks.cross_encoder_lora.protocol import ( + BASE_REVISION, + CPU_ORACLE, + MAX_BATCH_ITEMS, + MODEL, + ORACLE_ATOL, + RESTORE_ATOL, + SEQUENTIAL, + STEADY_SECONDS, + THREADS, + WEIGHTS_SHA256, + paired_gate, + percentile, + require, +) +from tools.benchmarks.cross_encoder_lora.runner import ( + KILL_GRACE_SECONDS, + TERMINATE_GRACE_SECONDS, + WAIT_GRACE_SECONDS, + extract, + harness_manifest, + source_environment, + stop, + terminate, +) + +PROCESSES = 6 +WINDOWS = (("S", 30), ("L1", 200), ("L2", 200)) +CONCURRENCIES = (1, 4, 16) +METRICS = ((1, "mean_ms"), (1, "p95_ms"), (4, "rps"), (4, "p95_ms"), (16, "rps"), (16, "p95_ms")) +RUNNER_LIMIT = 1200 +WORKER_LIMIT = 180 +GC_FIELDS = ("collections", "collected", "uncollectable") +RUSAGE_FIELDS = ("ru_utime", "ru_stime", "ru_minflt", "ru_majflt", "ru_nvcsw", "ru_nivcsw") +OPTIONAL_FILES = ("cpu.stat", "cpu.stat.local", "cpu.pressure", "cpu.max", "cpu.weight", "cpuset.cpus.effective") +COUNTER_FILES = ("cpu.stat", "cpu.stat.local") +CONTROL_FILES = ("cpu.max", "cpu.weight", "cpuset.cpus.effective") +GC_GENERATIONS = 3 +QUARTILES = 4 +PSI_FIELDS = ("avg10", "avg60", "avg300", "total") +MAX_PERCENT = 100 +MAX_CPU_WEIGHT = 10000 + + +class DiagnosticEvidence(Evidence): + def emit(self, event: str, **fields: Any) -> dict[str, Any]: + if fields.get("qualification"): + fields["hardware_preflight_only"] = True + fields["qualification"] = False + return super().emit(event, **fields) + + +def unified_cgroup(proc: Path = Path("/proc/self")) -> dict[str, Any]: + """Resolve the process's own namespace; never fall back to another cgroup.""" + try: + rows = (proc / "cgroup").read_text().splitlines() + membership = [row[3:] for row in rows if row.startswith("0::")] + require(len(membership) == 1, "unified cgroup membership is unavailable or ambiguous") + group = Path(membership[0]) + require(group.is_absolute() and ".." not in group.parts, "invalid cgroup membership") + matches = [] + for row in (proc / "mountinfo").read_text().splitlines(): + left, right = row.split(" - ", 1) + if right.split()[0] != "cgroup2": + continue + fields = left.split() + root = Path(fields[3].replace("\\040", " ").replace("\\134", "\\")) + mount = Path(fields[4].replace("\\040", " ").replace("\\134", "\\")) + if group.is_relative_to(root): + matches.append(mount / group.relative_to(root)) + require(len(matches) == 1, "cgroup mount cannot be resolved uniquely") + path = matches[0] + info = path.stat() + return {"path": str(path), "device": info.st_dev, "inode": info.st_ino, "error": None} + except (OSError, ValueError, IndexError, AssertionError) as error: + return {"path": None, "error": f"{type(error).__name__}: {error}"} + + +def flat_counters(text: str) -> dict[str, int]: + result = {} + for row in text.splitlines(): + key, value = row.split() + require(key not in result, "duplicate telemetry counter") + result[key] = int(value) + require(result[key] >= 0, "negative telemetry counter") + require(bool(result), "empty exposed telemetry counter file") + return result + + +def pressure(text: str) -> dict[str, dict[str, float | int]]: + result = {} + for row in text.splitlines(): + name, *parts = row.split() + require(name in {"some", "full"} and name not in result, "invalid PSI row") + values = dict(part.split("=", 1) for part in parts) + require(len(parts) == len(PSI_FIELDS) and set(values) == set(PSI_FIELDS), "invalid PSI fields") + parsed = {key: int(value) if key == "total" else float(value) for key, value in values.items()} + require(all(math.isfinite(value) and value >= 0 for value in parsed.values()), "invalid PSI value") + require(all(parsed[key] <= MAX_PERCENT for key in PSI_FIELDS[:-1]), "invalid PSI percentage") + result[name] = parsed + require("some" in result, "missing exposed PSI some row") + return result + + +def validate_core(core: dict[str, Any]) -> None: + require( + all(isinstance(core[key], int) and core[key] > 0 for key in ("wall_ns", "monotonic_ns")) + and core["gc_enabled"] is True, + "invalid required clock/GC state", + ) + require( + set(core["rusage"]) == set(RUSAGE_FIELDS) + and all(math.isfinite(value) and value >= 0 for value in core["rusage"].values()), + "invalid required rusage", + ) + for key in ("gc_thresholds", "gc_count"): + require( + len(core[key]) == GC_GENERATIONS and all(type(value) is int and value >= 0 for value in core[key]), + "invalid required GC counts/thresholds", + ) + require(core["gc_thresholds"][0] > 0, "automatic GC is disabled by its threshold") + require(len(core["gc_stats"]) == GC_GENERATIONS, "missing required GC generations") + require( + all( + set(row) == set(GC_FIELDS) and all(type(row[key]) is int and row[key] >= 0 for key in GC_FIELDS) + for row in core["gc_stats"] + ), + "invalid required GC statistics", + ) + + +def control_value(name: str, text: str) -> str: + if name == "cpu.max": + quota, period = text.split() + require((quota == "max" or int(quota) > 0) and int(period) > 0, "invalid exposed cpu.max") + elif name == "cpu.weight": + require(1 <= int(text) <= MAX_CPU_WEIGHT, "invalid exposed cpu.weight") + else: + for item in text.split(","): + bounds = [int(value) for value in item.split("-")] + require(len(bounds) in {1, 2} and bounds[0] >= 0 and bounds[-1] >= bounds[0], "invalid exposed cpuset") + return text + + +def snapshot() -> dict[str, Any]: + """Only called outside performance(), including all filesystem reads.""" + started = time.monotonic_ns() + usage = resource.getrusage(resource.RUSAGE_SELF) + core = { + "wall_ns": time.time_ns(), + "monotonic_ns": started, + "rusage": {key: getattr(usage, key) for key in RUSAGE_FIELDS}, + "gc_enabled": gc.isenabled(), + "gc_thresholds": list(gc.get_threshold()), + "gc_count": list(gc.get_count()), + "gc_stats": gc.get_stats(), + } + validate_core(core) + group = unified_cgroup() + files: dict[str, Any] = {} + errors = {} + if group["path"] is not None: + for name in OPTIONAL_FILES: + try: + text = (Path(group["path"]) / name).read_text().strip() + except OSError as error: + files[name] = None + errors[name] = f"{type(error).__name__}: {error}" + continue + # Missing optional files are unavailability, but malformed exposed + # counters must not be mistaken for trustworthy zero deltas. + files[name] = ( + flat_counters(text) + if name in COUNTER_FILES + else pressure(text) + if name == "cpu.pressure" + else control_value(name, text) + ) + else: + files = dict.fromkeys(OPTIONAL_FILES) + errors["cgroup"] = group["error"] + affinity = None + try: + if hasattr(os, "sched_getaffinity"): + affinity = sorted(os.sched_getaffinity(0)) + require(bool(affinity), "empty exposed affinity") + else: + errors["affinity"] = "sched_getaffinity unavailable on this platform" + except OSError as error: + errors["affinity"] = f"{type(error).__name__}: {error}" + return { + "qualification": False, + "core": core, + "cgroup": group, + "cgroup_files": files, + "optional_errors": errors, + "affinity": affinity, + "snapshot_read_ns": time.monotonic_ns() - started, + } + + +def counter_delta(before: dict[str, Any], after: dict[str, Any]) -> dict[str, Any]: + require(set(before) == set(after), "counter fields changed within a window") + result = {key: after[key] - value for key, value in before.items()} + require(all(math.isfinite(value) and value >= 0 for value in result.values()), "telemetry counter reset") + return result + + +def telemetry_delta(before: dict[str, Any], after: dict[str, Any]) -> dict[str, Any]: + first, last = before["core"], after["core"] + validate_core(first) + validate_core(last) + require( + last["wall_ns"] > first["wall_ns"] and last["monotonic_ns"] > first["monotonic_ns"], + "required clock did not advance", + ) + require( + first["gc_enabled"] and last["gc_enabled"] and first["gc_thresholds"] == last["gc_thresholds"], + "GC configuration changed", + ) + group_before, group_after = before["cgroup"], after["cgroup"] + require( + all(group_before.get(key) == group_after.get(key) for key in ("path", "device", "inode")), + "cgroup path or identity changed within a window", + ) + require(before["affinity"] == after["affinity"], "process affinity changed within a window") + for name in CONTROL_FILES: + require( + before["cgroup_files"].get(name) == after["cgroup_files"].get(name), + "cgroup quota/weight/cpuset changed within a window", + ) + optional = {} + for name in COUNTER_FILES: + a, b = before["cgroup_files"].get(name), after["cgroup_files"].get(name) + optional[name] = counter_delta(a, b) if a is not None and b is not None else None + a, b = before["cgroup_files"].get("cpu.pressure"), after["cgroup_files"].get("cpu.pressure") + optional["cpu.pressure"] = ( + counter_delta({k: v["total"] for k, v in a.items()}, {k: v["total"] for k, v in b.items()}) + if a is not None and b is not None + else None + ) + return { + "qualification": False, + "scope": "Whole performance call, including sequential warmup, measurement, drain, validation and raw emission", + "wall_s": (last["wall_ns"] - first["wall_ns"]) / 1e9, + "monotonic_s": (last["monotonic_ns"] - first["monotonic_ns"]) / 1e9, + "rusage": counter_delta(first["rusage"], last["rusage"]), + "gc_stats": [counter_delta(a, b) for a, b in zip(first["gc_stats"], last["gc_stats"], strict=True)], + "optional_cgroup": optional, + "cpu_pressure_counters_available": all(optional.get(name) is not None for name in ("cpu.stat", "cpu.pressure")), + "resource_telemetry_complete": before["affinity"] is not None + and after["affinity"] is not None + and all( + before["cgroup_files"].get(name) is not None and after["cgroup_files"].get(name) is not None + for name in OPTIONAL_FILES + ), + "optional_errors_before": before["optional_errors"], + "optional_errors_after": after["optional_errors"], + } + + +def metrics(phase: dict[str, Any]) -> dict[str, Any]: + samples = phase["samples"] + latencies = [row["latency_ms"] for row in samples] + require(bool(latencies) and all(math.isfinite(value) and value > 0 for value in latencies), "invalid raw latency") + elapsed = phase["elapsed_s"] + require(math.isfinite(elapsed) and elapsed > 0, "invalid phase elapsed") + result = { + "count": len(samples), + "elapsed_s": elapsed, + "mean_ms": statistics.mean(latencies), + "p95_ms": percentile(latencies, 0.95), + "rps": len(samples) / elapsed, + } + for key, value in result.items(): + require(math.isclose(value, phase[key], rel_tol=1e-12, abs_tol=0), "raw metric differs from phase aggregate") + return result + + +def chronological_quartiles(samples: list[dict[str, Any]]) -> list[dict[str, Any]]: + ordered = sorted(samples, key=lambda row: (row["start_ns"], row["label"])) + require(len(ordered) >= QUARTILES, "too few requests for fixed chronological quartiles") + result = [] + for index in range(QUARTILES): + rows = ordered[len(ordered) * index // QUARTILES : len(ordered) * (index + 1) // QUARTILES] + headers: dict[str, Any] = {} + keys = sorted({key for row in rows for key in row["responses"][0]["timing_headers"]}) + for key in keys: + values = [ + float(row["responses"][0]["timing_headers"][key]) + for row in rows + if key in row["responses"][0]["timing_headers"] + ] + require(all(math.isfinite(value) and value >= 0 for value in values), "invalid timing header") + headers[key] = { + "count": len(values), + "min": min(values), + "mean": statistics.mean(values), + "max": max(values), + "p95": sorted(values)[math.ceil(0.95 * len(values)) - 1], + } + latencies = [row["latency_ms"] for row in rows] + result.append( + { + "count": len(rows), + "start_ns": rows[0]["start_ns"], + "last_start_ns": rows[-1]["start_ns"], + "mean_ms": statistics.mean(latencies), + "p95_ms": percentile(latencies, 0.95), + "timing_headers": headers, + } + ) + return result + + +def verify_attempt(path: Path, record: dict[str, Any]) -> None: + """Reopen raw warmups/statuses as well as measured rows; never time this work.""" + events = [json.loads(line) for line in path.read_text().splitlines()] + require(len(events) == 1 and events[0]["event"] == "performance_attempt", "missing/extra performance attempt") + attempt = events[0] + require( + attempt["qualification"] is False and attempt["concurrency"] == record["concurrency"], + "attempt identity differs", + ) + require(len(attempt["warmups"]) == record["warmups"], "sequential warmup count differs") + measured = record["performance"] + require( + attempt["completed_samples"] + == [ + {key: value for key, value in row.items() if key != "diagnostic_matching_oracle"} + for row in measured["samples"] + ], + "attempt/raw measured rows differ", + ) + rows = [*attempt["warmups"], *attempt["completed_samples"]] + require( + len(attempt["wire_responses"]) == len(rows) + and all(row["status"] == HTTPStatus.OK for row in attempt["wire_responses"]), + "hidden wire retries or failure", + ) + for row in rows: + saved = row["scores"][:] + worker.validate_response(row) + require(saved == row["scores"] and row["selection"] is None, "baseline scores/selection differ") + require(row["end_ns"] >= row["start_ns"] > 0, "invalid request timestamps") + if record["concurrency"] == 1: + require(measured["count"] == (4 if record["smoke"] else SEQUENTIAL), "sequential request count differs") + else: + require(measured["elapsed_s"] >= (0.15 if record["smoke"] else STEADY_SECONDS), "closed-loop duration differs") + + +def measure_window( + url: str, + concurrency: int, + window: str, + warmups: int, + directory: Path, + base_scores: list[float], + oracles: dict[str, Any], + *, + smoke: bool, + continuity: dict[str, Any] | None = None, +) -> dict[str, Any]: + evidence = DiagnosticEvidence(directory) + record: dict[str, Any] = { + "qualification": False, + "completed": False, + "window": window, + "concurrency": concurrency, + "warmups": 2 if smoke else warmups, + "smoke": smoke, + } + original_warmups = worker.WARMUP + try: + record["gpu_before"] = worker.telemetry() + record["telemetry_before"] = snapshot() + if continuity is not None: + telemetry_delta(continuity, record["telemetry_before"]) + worker.WARMUP = warmups # ty: ignore[invalid-assignment] -- scoped diagnostic value, always restored + try: + measured = worker.performance(url, concurrency, evidence, smoke=smoke) + finally: + worker.WARMUP = original_warmups + record["telemetry_after"] = snapshot() + record["gpu_after"] = worker.telemetry() + record["telemetry_delta"] = telemetry_delta(record["telemetry_before"], record["telemetry_after"]) + record["performance"] = measured + record["attempt_sha256"] = digest(directory / "events.jsonl") + verify_attempt(directory / "events.jsonl", record) + for row in measured["samples"]: + if concurrency == 1: + require(vector_delta(row["scores"], base_scores) <= RESTORE_ATOL, "baseline C1 scores changed") + row["diagnostic_matching_oracle"] = worker.matching_vector(row["scores"], oracles) + record["metrics"] = metrics(measured) + record["chronological_quartiles"] = chronological_quartiles(measured["samples"]) + record["actual_timed_batch_shape"] = "unknown" + record["completed"] = True + if continuity is not None: + continuity.clear() + continuity.update(record["telemetry_after"]) + except BaseException as error: + record["error"] = worker.error_record(error) + raise + finally: + worker.WARMUP = original_warmups + atomic_json(directory / "window.json", record) + return { + "qualification": False, + "window": window, + "concurrency": concurrency, + "completed": True, + "metrics": record["metrics"], + "resource_telemetry_complete": record["telemetry_delta"]["resource_telemetry_complete"], + "record": str((directory / "window.json").relative_to(directory.parent.parent)), + "sha256": digest(directory / "window.json"), + } + + +async def serve_windows(args: argparse.Namespace, evidence: DiagnosticEvidence, summary: dict[str, Any]) -> None: + manifest = json.loads((args.fixtures / "manifest.json").read_text()) + require(bool(manifest["smoke"]) == args.smoke, "fixture mode mismatch") + snapshot_path = Path(manifest["snapshot"]) + for name, expected in manifest["model_files"].items(): + require(digest(snapshot_path / name) == expected, "model fixture changed") + if not args.smoke: + require(digest(snapshot_path / "model.safetensors") == WEIGHTS_SHA256, "checkpoint pin differs") + summary["fixtures"] = manifest + worker.configs(evidence.directory / "models", args.source, args.fixtures, "B", snapshot_path, smoke=args.smoke) + app = worker.AppFactory.create_app( + worker.AppStateConfig( + models_dir=evidence.directory / "models", + device="cpu" if args.smoke else "cuda:0", + remote_serving=False, + bind_host="127.0.0.1", + ) + ) + sock = socket.socket() + task = None + client = None + server = None + adapter = None + try: + sock.bind(("127.0.0.1", 0)) + url = f"http://127.0.0.1:{sock.getsockname()[1]}" + server = uvicorn.Server( + uvicorn.Config(app, log_level="warning", timeout_graceful_shutdown=20, access_log=False) + ) + task = asyncio.create_task(server.serve(sockets=[sock])) + client = worker.Client(url) + while not server.started: + if task.done(): + await task + raise RuntimeError("server exited before readiness") + await asyncio.sleep(0.05) + registry = app.state.registry + adapter = await registry.load_async(MODEL, "cpu" if args.smoke else "cuda:0") + model_worker = registry._loaded[MODEL].worker + require( + model_worker is not None and getattr(adapter, "_lora", None) is None, "baseline worker/manager mismatch" + ) + state = await asyncio.to_thread(worker.tensor_state, adapter) + expected_dtype = "torch.float32" if args.smoke else "torch.float16" + require({x["dtype"] for x in state["tensors"].values()} == {expected_dtype}, "baseline precision differs") + summary["actual_precision"] = expected_dtype + summary["original_tensor_state"] = state + summary["loaded_before_forward_memory"] = worker.memory() + bases = [await asyncio.to_thread(client.score, None, f"base-{i}") for i in range(3)] + baseline = bases[0]["scores"] + require(max(vector_delta(baseline, row["scores"]) for row in bases) <= RESTORE_ATOL, "unstable base scores") + if not args.smoke: + require(vector_delta(baseline, CPU_ORACLE) <= ORACLE_ATOL, "finite checkpoint oracle differs") + summary["base_scores"] = baseline + summary["shape_oracles"] = await worker.oracle(adapter, model_worker, None, smoke=args.smoke) + summary["initial_memory"] = worker.memory() + summary["untimed_replays"] = {} + adapter.observation_phase = "untimed-stability-setup" + for concurrency in CONCURRENCIES: + first = len(adapter.batch_observations) + rows = await asyncio.to_thread(worker.replay, url, concurrency) + batches = adapter.batch_observations[first:] + require(bool(batches), "missing actual untimed batch observations") + worker.validate_usage(rows, batches) + for batch in batches: + require_forward_selection(batch, None) + require( + vector_delta(batch["scores"], summary["shape_oracles"][worker.batch_key(batch)]["scores"]) + <= RESTORE_ATOL, + "untimed shape oracle differs", + ) + summary["untimed_replays"][str(concurrency)] = {"requests": rows, "batches": batches} + # These three setup memory observations mirror the shared B harness. + # There is no new forced collection between any diagnostic windows. + summary["resident_memory"] = worker.memory() + summary["windows"] = [] + continuity = dict(summary["required_telemetry_preflight"]) + with worker.unobserved(adapter): + for concurrency in CONCURRENCIES: + for window, warmups in WINDOWS: + directory = evidence.directory / f"C{concurrency}" / window + row = await asyncio.to_thread( + measure_window, + url, + concurrency, + window, + warmups, + directory, + baseline, + summary["shape_oracles"], + smoke=args.smoke, + continuity=continuity, + ) + summary["windows"].append(row) + atomic_json(evidence.directory / "process.json", summary) + # Only compact metadata survives here. Raw requests live on + # disk; ordinary reference release and automatic GC apply. + summary["restored_tensor_state"] = await asyncio.to_thread(worker.tensor_state, adapter) + worker.base_unchanged(state, summary["restored_tensor_state"], restored=True) + await registry.unload_async(MODEL) + summary["completed"] = True + finally: + try: + if client is not None: + client.close() + finally: + try: + if server is not None and task is not None: + server.should_exit = True + await asyncio.wait_for(asyncio.shield(task), timeout=30) + summary["shutdown_complete"] = True + finally: + sock.close() + + +async def child(args: argparse.Namespace) -> dict[str, Any]: + evidence = DiagnosticEvidence(args.output.resolve()) + summary: dict[str, Any] = { + "qualification": False, + "diagnostic_only": True, + "completed": False, + "shutdown_complete": False, + "process": args.process, + "pid": os.getpid(), + "source_commit": BASE_REVISION, + "smoke": args.smoke, + } + os.environ.update( + { + "SIE_TELEMETRY_DISABLED": "1", + "SIE_DISK_CACHE_ENABLED": "false", + "SIE_MAX_LORAS_PER_MODEL": "2", + "SIE_MAX_BATCH_REQUESTS": str(MAX_BATCH_ITEMS), + "TOKENIZERS_PARALLELISM": "false", + } + ) + torch.set_num_threads(THREADS) + torch.set_num_interop_threads(1) + try: + summary["required_telemetry_preflight"] = snapshot() + environment = worker.preflight(args, evidence) + environment["hardware_preflight_only"] = environment.pop("qualification") + environment["qualification"] = False + environment["source_imports"] = {} + for name, target in (("sdk", worker.SIEClient), ("server", worker.AppFactory)): + module_file = sys.modules[target.__module__].__file__ + assert module_file is not None, "source module has no file" + source_file = Path(module_file).resolve() # noqa: ASYNC240 -- preflight before model/server starts + require(source_file.is_relative_to(args.source.resolve()), "SDK/server imported outside baseline archive") + environment["source_imports"][name] = { + "relative_path": str(source_file.relative_to(args.source.resolve())), + "sha256": digest(source_file), + } + environment["source_imports"]["adapter"] = { + "relative_path": str(Path(environment["adapter_source"]).relative_to(args.source.resolve())), + "sha256": environment["adapter_sha256"], + } + summary["environment"] = environment + async with asyncio.timeout(args.trial_deadline_s): + await serve_windows(args, evidence, summary) + except BaseException as error: + summary["completed"] = False + summary["error"] = worker.error_record(error) + evidence.emit("failure", **summary["error"]) + raise + finally: + atomic_json(evidence.directory / "process.json", summary) + evidence.emit("finished", completed=summary["completed"], shutdown_complete=summary["shutdown_complete"]) + return summary + + +def reopen_window(directory: Path, record: dict[str, Any]) -> dict[str, Any]: + relative = f"C{record['concurrency']}/{record['window']}/window.json" + require(record["record"] == relative, "window path differs from identity") + path = directory / relative + require( + path.resolve().is_relative_to(directory.resolve()) and digest(path) == record["sha256"], + "window path/hash differs", + ) + window = json.loads(path.read_text()) + require(window["qualification"] is False and window["completed"] is True, "window is incomplete or qualified") + require( + (window["window"], window["concurrency"]) == (record["window"], record["concurrency"]), + "window identity differs", + ) + require( + window["warmups"] == (2 if window["smoke"] else dict(WINDOWS)[window["window"]]), "fixed warmup count differs" + ) + attempt = path.parent / "events.jsonl" + require(digest(attempt) == window["attempt_sha256"], "raw attempt hash differs") + verify_attempt(attempt, window) + recalculated = metrics(window["performance"]) + require(recalculated == window["metrics"] == record["metrics"], "window compact projection differs") + require( + telemetry_delta(window["telemetry_before"], window["telemetry_after"]) == window["telemetry_delta"], + "telemetry projection mismatch", + ) + require( + chronological_quartiles(window["performance"]["samples"]) == window["chronological_quartiles"], + "quartile projection mismatch", + ) + return window + + +def verify_process_sources(rows: list[dict[str, Any]], source: Path) -> None: + require(bool(rows), "missing process environments") + first = rows[0] + fields = ( + "versions", + "torchao", + "python", + "threads", + "device", + "torch_cuda", + "installed_distributions", + "installed_distributions_sha256", + "source_imports", + "hardware_preflight_only", + ) + for row in rows: + require(row["source_commit"] == BASE_REVISION and row["smoke"] == first["smoke"], "source/mode changed") + environment = row["environment"] + require(environment["qualification"] is False, "environment incorrectly marks qualification") + require( + environment["installed_distributions_sha256"] == fingerprint(environment["installed_distributions"]), + "installed distribution manifest hash differs", + ) + require(all(environment[key] == first["environment"][key] for key in fields), "process environments differ") + require(set(environment["source_imports"]) == {"adapter", "server", "sdk"}, "missing source import proof") + for imported in environment["source_imports"].values(): + path = source / imported["relative_path"] + require( + path.resolve().is_relative_to(source.resolve()) and digest(path) == imported["sha256"], + "imported source differs from immutable baseline", + ) + require( + row["actual_precision"] == first["actual_precision"] and row["fixtures"] == first["fixtures"], + "process model or precision differs", + ) + initial, reference = row["required_telemetry_preflight"], first["required_telemetry_preflight"] + require( + initial["core"]["gc_thresholds"] == reference["core"]["gc_thresholds"] + and initial["affinity"] == reference["affinity"], + "cross-process GC configuration/affinity differs", + ) + require( + all(initial["cgroup"].get(key) == reference["cgroup"].get(key) for key in ("path", "device", "inode")) + and all(initial["cgroup_files"].get(key) == reference["cgroup_files"].get(key) for key in CONTROL_FILES), + "cross-process cgroup identity/configuration differs", + ) + require( + vector_delta(row["base_scores"], first["base_scores"]) <= RESTORE_ATOL, "cross-process base scores differ" + ) + require(set(row["shape_oracles"]) == set(first["shape_oracles"]), "cross-process oracle workloads differ") + for key, value in row["shape_oracles"].items(): + require( + vector_delta(value["scores"], first["shape_oracles"][key]["scores"]) <= RESTORE_ATOL, + "cross-process shape oracle differs", + ) + + +def analyze(rows: list[dict[str, Any]]) -> dict[str, Any]: + require( + len(rows) == PROCESSES and [row["process"] for row in rows] == list(range(PROCESSES)), + "six process units required", + ) + require(len({row["pid"] for row in rows}) == PROCESSES, "fresh process identities required") + require( + all(row["qualification"] is False and row["completed"] and row["shutdown_complete"] for row in rows), + "incomplete diagnostic process", + ) + schedule = [(c, w) for c in CONCURRENCIES for w, _ in WINDOWS] + require( + all([(w["concurrency"], w["window"]) for w in row["windows"]] == schedule for row in rows), + "all54 fixed windows required", + ) + gates, dispersion = {}, {} + for concurrency, field in METRICS: + name = f"C{concurrency}_{field}" + values = { + window: [ + next(w for w in row["windows"] if w["concurrency"] == concurrency and w["window"] == window)["metrics"][ + field + ] + for row in rows + ] + for window, _ in WINDOWS + } + require( + all(math.isfinite(x) and x > 0 for series in values.values() for x in series), "invalid statistical operand" + ) + ratio = [a / b if field == "rps" else b / a for a, b in zip(values["L1"], values["L2"], strict=True)] + gates[name] = { + "L2_over_L1_degradation": paired_gate(ratio), + "L1_over_L2_degradation": paired_gate([1 / value for value in ratio]), + } + dispersion[name] = {} + for window, series in values.items(): + dispersion[name][window] = { + "values_by_process": series, + "min": min(series), + "median": statistics.median(series), + "max": max(series), + "max_over_min": max(series) / min(series), + "arithmetic_cv": statistics.stdev(series) / statistics.mean(series), + "log_sd": statistics.stdev([math.log(x) for x in series]), + } + passed = all(gate["passed"] for metric in gates.values() for gate in metric.values()) + return { + "qualification": False, + "statistical_stability_passed": passed, + "directional_gates": gates, + "between_process_dispersion": dispersion, + "resource_telemetry_complete": all(w["resource_telemetry_complete"] for row in rows for w in row["windows"]), + "interpretation": "Symmetric later-window agreement only; no causal warmup claim or fresh-process candidate qualification", + } + + +def admission(now: float, deadline: float, trial_limit: float) -> dict[str, float]: + reserve = trial_limit + WAIT_GRACE_SECONDS + TERMINATE_GRACE_SECONDS + KILL_GRACE_SECONDS + require(now + reserve < deadline, "insufficient time for complete diagnostic worker and cleanup") + return { + "monotonic_s": now, + "deadline_monotonic_s": deadline, + "reserve_s": reserve, + "margin_s": deadline - now - reserve, + } + + +def run(args: argparse.Namespace) -> dict[str, Any]: + started = time.monotonic() + args.output.mkdir(parents=True, exist_ok=False) + manifest = extract(args.inputs.resolve(), args.output / "sources") + summary: dict[str, Any] = { + "schema": 1, + "qualification": False, + "diagnostic_only": True, + "completed": False, + "statistical_stability_passed": False, + "smoke": args.smoke, + "source_manifest": manifest, + "executed_source": "B", + "harness_manifest": harness_manifest(), + "processes": [], + } + if args.environment_receipt is not None: + summary["environment_preparation"] = { + "sha256": digest(args.environment_receipt), + "receipt": json.loads(args.environment_receipt.read_text()), + } + deadline = started + args.deadline_s + summary["runner_clock"] = { + "monotonic_start_s": started, + "wall_start_ns": time.time_ns(), + "deadline_monotonic_s": deadline, + "budget_s": args.deadline_s, + } + summary["methods"] = { + "window_order": [{"name": name, "sequential_warmup_calls": count} for name, count in WINDOWS], + "phase_order": list(CONCURRENCIES), + "fresh_baseline_processes": PROCESSES, + "gc": "Original automatic GC; no extra per-window collections, disabling or callbacks", + "raw_retention": "Per-window raw disk records; previous payload references are released normally", + "telemetry_scope": "Whole performance call including warmup, drain, validation and raw emission; not timed-only", + "qualification": False, + } + try: + for index in range(1 if args.smoke else PROCESSES): + reserved = admission(time.monotonic(), deadline, args.trial_deadline_s) + directory = args.output / f"process-{index}" + source = (args.output / "sources/B").resolve() + command = [ + sys.executable, + "-m", + "tools.benchmarks.cross_encoder_lora.stability", + "--worker", + "--source", + str(source), + "--process", + str(index), + "--fixtures", + str(args.fixtures.resolve()), + "--output", + str(directory.resolve()), + "--trial-deadline-s", + str(args.trial_deadline_s), + ] + if args.smoke: + command.append("--smoke") + print(f"Starting baseline process {index + 1}", flush=True) + process = None + error = None + with (args.output / f"process-{index}.log").open("w") as log: + try: + process = subprocess.Popen( + command, env=source_environment(source), cwd=source, stdout=log, stderr=subprocess.STDOUT + ) + process.wait(timeout=args.trial_deadline_s + WAIT_GRACE_SECONDS) + except BaseException as caught: # noqa: BLE001 -- persist error after unconditional cleanup, then re-raise + error = caught + finally: + if process is not None: + stop(process) + terminal = directory / "process.json" + record = ( + json.loads(terminal.read_text()) + if terminal.exists() + else {"qualification": False, "completed": False, "shutdown_complete": False} + ) + row = { + "qualification": False, + "process": index, + "pid": record.get("pid"), + "completed": record["completed"] and error is None, + "shutdown_complete": record["shutdown_complete"], + "returncode": process.returncode if process is not None else None, + "admission": reserved, + "record": f"process-{index}/process.json", + "sha256": digest(terminal) if terminal.exists() else None, + } + if error is not None: + row["error"] = worker.error_record(error) + summary["processes"].append(row) + atomic_json(args.output / "summary.json", summary) + if error is not None: + raise error + require( + row["returncode"] == 0 and row["completed"] and row["shutdown_complete"], + "baseline process failed; evidence retained", + ) + print(f"Finished baseline process {index + 1}", flush=True) + complete = [] + for row in summary["processes"]: + path = args.output / row["record"] + require(digest(path) == row["sha256"], "process record changed") + record = json.loads(path.read_text()) + require(record["process"] == row["process"] and record["pid"] == row["pid"], "process identity mismatch") + previous = record["required_telemetry_preflight"] + for entry in record["windows"]: + reopened = reopen_window(path.parent, entry) + telemetry_delta(previous, reopened["telemetry_before"]) + previous = reopened["telemetry_after"] + entry["metrics"] = metrics(reopened["performance"]) + require( + entry["resource_telemetry_complete"] == reopened["telemetry_delta"]["resource_telemetry_complete"], + "telemetry projection mismatch", + ) + complete.append(record) + verify_process_sources(complete, (args.output / "sources/B").resolve()) + summary["completed"] = True + if not args.smoke: + summary["analysis"] = analyze(complete) + summary["statistical_stability_passed"] = summary["analysis"]["statistical_stability_passed"] + require(summary["statistical_stability_passed"], "fixed symmetric baseline stability gates did not pass") + except BaseException as error: + summary["error"] = worker.error_record(error) + raise + finally: + atomic_json(args.output / "summary.json", summary) + return summary + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--inputs", type=Path) + parser.add_argument("--fixtures", type=Path, required=True) + parser.add_argument("--output", type=Path, required=True) + parser.add_argument("--deadline-s", type=float, default=RUNNER_LIMIT) + parser.add_argument("--trial-deadline-s", type=float, default=WORKER_LIMIT) + parser.add_argument("--environment-receipt", type=Path) + parser.add_argument("--smoke", action="store_true") + parser.add_argument("--worker", action="store_true", help=argparse.SUPPRESS) + parser.add_argument("--source", type=Path, help=argparse.SUPPRESS) + parser.add_argument("--process", type=int, default=0, help=argparse.SUPPRESS) + args = parser.parse_args() + require( + 0 < args.trial_deadline_s <= WORKER_LIMIT and 0 < args.deadline_s <= RUNNER_LIMIT, + "diagnostic deadline outside fixed bounds", + ) + if args.worker: + require(args.source is not None, "worker requires exact source") + asyncio.run(child(args)) + else: + require(args.inputs is not None, "runner requires immutable inputs") + signal.signal(signal.SIGTERM, terminate) + run(args) + + +if __name__ == "__main__": + main() diff --git a/tools/benchmarks/cross_encoder_lora/tests/test_stability.py b/tools/benchmarks/cross_encoder_lora/tests/test_stability.py new file mode 100644 index 000000000..dc16e939b --- /dev/null +++ b/tools/benchmarks/cross_encoder_lora/tests/test_stability.py @@ -0,0 +1,499 @@ +"""Fixed baseline diagnostic controls; CPU smoke never qualifies GPU performance.""" + +# Owned Python children run the exact local diagnostic, without a shell. +# ruff: noqa: S603 + +from __future__ import annotations + +import ast +import copy +import json +import subprocess +import sys +from pathlib import Path +from typing import Any + +import pytest + +from tools.benchmarks.cross_encoder_lora import prepare as source_preparer +from tools.benchmarks.cross_encoder_lora import stability as probe +from tools.benchmarks.cross_encoder_lora import worker +from tools.benchmarks.cross_encoder_lora.evidence import digest, fingerprint +from tools.benchmarks.cross_encoder_lora.fixtures import prepare +from tools.benchmarks.cross_encoder_lora.protocol import BASE_REVISION, SEQUENTIAL +from tools.benchmarks.cross_encoder_lora.runner import ROOT, extract, source_environment +from tools.benchmarks.cross_encoder_lora.tests.test_probe import tiny_snapshot + + +def observed(tick: int = 1) -> dict[str, Any]: + return { + "qualification": False, + "core": { + "wall_ns": tick * 1000000000, + "monotonic_ns": tick * 1000000000, + "rusage": dict.fromkeys(probe.RUSAGE_FIELDS, tick), + "gc_enabled": True, + "gc_thresholds": [700, 10, 10], + "gc_count": [tick, 0, 0], + "gc_stats": [dict.fromkeys(probe.GC_FIELDS, tick) for _ in range(3)], + }, + "cgroup": {"path": None, "error": "unavailable"}, + "cgroup_files": dict.fromkeys(probe.OPTIONAL_FILES), + "optional_errors": {"cgroup": "unavailable"}, + "affinity": None, + "snapshot_read_ns": 1, + } + + +def process_rows() -> list[dict[str, Any]]: + return [ + { + "qualification": False, + "completed": True, + "shutdown_complete": True, + "process": index, + "pid": index + 100, + "windows": [ + { + "concurrency": c, + "window": name, + "resource_telemetry_complete": False, + "metrics": {"mean_ms": 10.0, "p95_ms": 12.0, "rps": 100.0}, + } + for c in probe.CONCURRENCIES + for name, _ in probe.WINDOWS + ], + } + for index in range(probe.PROCESSES) + ] + + +def test_symmetric_stability_and_between_process_dispersion() -> None: + records = process_rows() + result = probe.analyze(records) + assert result["statistical_stability_passed"] + assert not result["qualification"] + assert not result["resource_telemetry_complete"] + assert sum(len(value) for value in result["directional_gates"].values()) == 12 + for row in records: + for window in row["windows"]: + if window["window"] == "L2": + window["metrics"]["mean_ms"] *= 0.9 + result = probe.analyze(records) + gates = result["directional_gates"]["C1_mean_ms"] + assert gates["L2_over_L1_degradation"]["passed"] + assert not gates["L1_over_L2_degradation"]["passed"] + assert not result["statistical_stability_passed"] + assert result["between_process_dispersion"]["C1_mean_ms"]["L2"]["values_by_process"] == [9.0] * 6 + + +@pytest.mark.parametrize( + "failure", ["missing_process", "same_pid", "missing_window", "order", "cleanup", "negative", "uncertain"] +) +def test_all_fixed_process_units_and_positive_metrics_required(failure: str) -> None: + rows = process_rows() + if failure == "missing_process": + rows.pop() + elif failure == "same_pid": + rows[-1]["pid"] = rows[0]["pid"] + elif failure == "missing_window": + rows[-1]["windows"].pop() + elif failure == "order": + rows[-1]["windows"].reverse() + elif failure == "cleanup": + rows[-1]["shutdown_complete"] = False + elif failure == "negative": + for row in rows: + row["windows"][1]["metrics"]["mean_ms"] = -10 + row["windows"][2]["metrics"]["mean_ms"] = -10 + else: + for row, ratio in zip(rows, [0.94, 0.94, 0.94, 1.06, 1.06, 1.06], strict=True): + row["windows"][2]["metrics"]["mean_ms"] *= ratio + assert not probe.analyze(rows)["statistical_stability_passed"] + return + with pytest.raises(AssertionError): + probe.analyze(rows) + + +def test_optional_cgroup_unavailable_is_explicit_not_zero(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(probe, "unified_cgroup", lambda: {"path": None, "error": "not exposed"}) + first, last = probe.snapshot(), probe.snapshot() + delta = probe.telemetry_delta(first, last) + assert set(first["cgroup_files"]) == set(probe.OPTIONAL_FILES) + assert all(value is None for value in first["cgroup_files"].values()) + assert delta["optional_cgroup"]["cpu.stat"] is None + assert not delta["resource_telemetry_complete"] + assert delta["optional_errors_before"]["cgroup"] == "not exposed" + + +@pytest.mark.parametrize("failure", ["gc_missing", "gc_count", "rusage", "clock", "disabled"]) +def test_required_telemetry_validation(failure: str) -> None: + core = observed()["core"] + if failure == "gc_missing": + core["gc_stats"] = [] + elif failure == "gc_count": + core["gc_count"] = [-1, 0, 0] + elif failure == "rusage": + core["rusage"]["ru_utime"] = float("nan") + elif failure == "clock": + core["wall_ns"] = 0 + else: + core["gc_enabled"] = False + with pytest.raises(AssertionError): + probe.validate_core(core) + + +def test_own_cgroup_resolution_and_no_namespace_fallback(tmp_path: Path) -> None: + proc = tmp_path / "proc" + proc.mkdir() + mount = tmp_path / "mounted" + owned = mount / "child" + owned.mkdir(parents=True) + (proc / "cgroup").write_text("0::/namespace/child\n") + line = f"1 0 0:1 /namespace {mount} rw - cgroup2 cgroup rw\n" + (proc / "mountinfo").write_text(line) + assert probe.unified_cgroup(proc)["path"] == str(owned) + (proc / "mountinfo").write_text(line + line) + assert probe.unified_cgroup(proc)["path"] is None + (proc / "cgroup").write_text("0::/other/child\n") + (proc / "mountinfo").write_text(line) + assert probe.unified_cgroup(proc)["path"] is None + + +def test_exposed_counters_and_gauges_have_distinct_semantics() -> None: + first, last = observed(1), observed(2) + for record, ticks, avg in ((first, 100, 90), (last, 120, 1)): + record["cgroup"] = {"path": "/owned", "device": 1, "inode": 2, "error": None} + record["cgroup_files"]["cpu.stat"] = probe.flat_counters(f"usage_usec {ticks}\nnr_throttled 1\n") + record["cgroup_files"]["cpu.pressure"] = probe.pressure(f"some avg10={avg} avg60=0 avg300=0 total={ticks}\n") + last["core"]["gc_count"] = [0, 0, 0] # Automatic GC resets this gauge normally. + delta = probe.telemetry_delta(first, last) + assert delta["cpu_pressure_counters_available"] + assert not delta["resource_telemetry_complete"] # Other optional files remain unavailable. + assert delta["optional_cgroup"]["cpu.pressure"]["some"] == 20 + assert delta["optional_cgroup"]["cpu.stat"]["usage_usec"] == 20 + last["cgroup_files"]["cpu.pressure"]["some"]["total"] = 99 + with pytest.raises(AssertionError, match="reset"): + probe.telemetry_delta(first, last) + for text in ("usage_usec -1", "usage_usec 0\nusage_usec 1", ""): + with pytest.raises((AssertionError, ValueError)): + probe.flat_counters(text) + with pytest.raises(AssertionError, match="PSI"): + probe.pressure("some avg10=nan avg60=0 avg300=0 total=1") + + +@pytest.mark.parametrize( + "changed", ["path", "inode", "visibility", "affinity", "quota", "gc_thresholds", "rusage", "gc_stats"] +) +def test_telemetry_continuity_refuses_changes(changed: str) -> None: + first, last = observed(2), observed(3) + first["cgroup"] = {"path": "/owned", "device": 1, "inode": 2, "error": None} + last["cgroup"] = copy.deepcopy(first["cgroup"]) + if changed in {"path", "inode"}: + last["cgroup"][changed] = "changed" + elif changed == "visibility": + last["cgroup"] = {"path": None, "error": "lost"} + elif changed == "affinity": + last["affinity"] = [0] + elif changed == "quota": + last["cgroup_files"]["cpu.max"] = "800000 100000" + elif changed == "gc_thresholds": + last["core"]["gc_thresholds"][0] = 800 + elif changed == "rusage": + last["core"]["rusage"]["ru_nvcsw"] = 1 + else: + last["core"]["gc_stats"][0]["collections"] = 1 + with pytest.raises(AssertionError): + probe.telemetry_delta(first, last) + + +def response(label: str, index: int) -> dict[str, Any]: + scores = [4.0, 3.0, 2.0, 1.0] + return { + "label": label, + "selection": None, + "start_ns": index * 10000000 + 1, + "end_ns": (index + 1) * 10000000, + "latency_ms": 10.0, + "scores": scores, + "responses": [{"status": 200, "timing_headers": {"x-inference-time": "4.0"}}], + "result": { + "scores": [{"item_id": f"{label}:{i}", "rank": i, "score": score} for i, score in enumerate(scores)] + }, + } + + +def stub_performance(_url: str, concurrency: int, evidence: probe.DiagnosticEvidence, *, smoke: bool) -> dict[str, Any]: + warmups = [response(f"warm-{i}", i) for i in range(2 if smoke else worker.WARMUP)] + samples = [response(f"sample-{i}", i) for i in range(4 if smoke else SEQUENTIAL)] + elapsed = 0.2 if smoke else 12.0 + evidence.emit( + "performance_attempt", + concurrency=concurrency, + warmups=warmups, + completed_samples=samples, + wire_responses=[row["responses"][0] for row in [*warmups, *samples]], + ) + return { + "concurrency": concurrency, + "count": len(samples), + "elapsed_s": elapsed, + "mean_ms": 10.0, + "p95_ms": 10.0, + "rps": len(samples) / elapsed, + "samples": samples, + } + + +def install_stub(monkeypatch: pytest.MonkeyPatch) -> None: + counter = iter(range(1, 100)) + monkeypatch.setattr(probe, "snapshot", lambda: observed(next(counter))) + monkeypatch.setattr(worker, "telemetry", lambda: {"device": "test"}) + monkeypatch.setattr(worker, "performance", stub_performance) + + +def test_raw_retention_reopen_and_warmup_restoration(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + install_stub(monkeypatch) + original = worker.WARMUP + continuity = observed(0) + continuity["core"]["wall_ns"] = continuity["core"]["monotonic_ns"] = 1 + oracles = {"shape": {"copies": 1, "scores": [[4.0, 3.0, 2.0, 1.0][index] for index in worker.ORDER]}} + compact = probe.measure_window( + "unused", 1, "L1", 200, tmp_path / "C1/L1", [4.0, 3.0, 2.0, 1.0], oracles, smoke=False, continuity=continuity + ) + assert original == worker.WARMUP + raw = probe.reopen_window(tmp_path, compact) + assert len(raw["performance"]["samples"]) == 200 + assert len(raw["chronological_quartiles"]) == 4 + assert continuity == raw["telemetry_after"] + assert all(row["diagnostic_matching_oracle"]["workload_sha256"] == "shape" for row in raw["performance"]["samples"]) + assert "samples" not in json.dumps(compact) + tampered = copy.deepcopy(compact) + tampered["metrics"]["mean_ms"] = 9.0 + with pytest.raises(AssertionError, match="projection"): + probe.reopen_window(tmp_path, tampered) + tampered = {**compact, "record": "../outside.json"} + with pytest.raises(AssertionError, match="path"): + probe.reopen_window(tmp_path, tampered) + (tmp_path / "C1/L1/events.jsonl").write_text("{}\n") + with pytest.raises(AssertionError, match="attempt hash"): + probe.reopen_window(tmp_path, compact) + + +def test_failed_window_keeps_partial_raw_and_restores_warmup(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + install_stub(monkeypatch) + original = worker.WARMUP + + def failed(_url: str, _concurrency: int, evidence: probe.DiagnosticEvidence, *, smoke: bool) -> None: + assert worker.WARMUP == 200 + assert not smoke + evidence.emit("performance_attempt", completed_samples=[{"error": "retained"}]) + raise RuntimeError("request failed") + + monkeypatch.setattr(worker, "performance", failed) + with pytest.raises(RuntimeError, match="request failed"): + probe.measure_window("unused", 1, "L2", 200, tmp_path / "C1/L2", [4.0, 3.0, 2.0, 1.0], {}, smoke=False) + assert original == worker.WARMUP + terminal = json.loads((tmp_path / "C1/L2/window.json").read_text()) + assert not terminal["qualification"] + assert not terminal["completed"] + assert terminal["error"]["message"] == "request failed" + assert json.loads((tmp_path / "C1/L2/events.jsonl").read_text())["completed_samples"] == [{"error": "retained"}] + + +def test_full_cleanup_budget_is_strict() -> None: + assert probe.admission(1, 272, 180)["reserve_s"] == 270 + with pytest.raises(AssertionError, match="insufficient"): + probe.admission(1, 271, 180) + + +def test_negative_raw_latency_cannot_hide_in_positive_aggregates() -> None: + phase = { + "samples": [{"latency_ms": value} for value in [-1.0, 2.0, 3.0, 4.0]], + "elapsed_s": 1.0, + "count": 4, + "mean_ms": 2.0, + "p95_ms": 4.0, + "rps": 4.0, + } + with pytest.raises(AssertionError, match="raw latency"): + probe.metrics(phase) + + +def test_shared_timed_code_and_gc_are_unmodified() -> None: + baseline = subprocess.run( + ["git", "show", "f06f414a:tools/benchmarks/cross_encoder_lora/worker.py"], # noqa: S607 + cwd=ROOT, + capture_output=True, + text=True, + check=False, + ) + if baseline.returncode: + pytest.skip("timed-code invariant control requires its documented parent commit in git history") + names = {"performance", "Client", "unobserved"} + + def code(text: str) -> dict[str, str]: + return { + node.name: ast.dump(node) + for node in ast.parse(text).body + if isinstance(node, (ast.FunctionDef, ast.ClassDef)) and node.name in names + } + + assert code(Path(worker.__file__).read_text()) == code(baseline.stdout) + tree = ast.parse(Path(probe.__file__).read_text()) + assert not any( + isinstance(node, ast.Call) + and isinstance(node.func, ast.Attribute) + and isinstance(node.func.value, ast.Name) + and node.func.value.id == "gc" + and node.func.attr in {"collect", "disable", "set_threshold"} + for node in ast.walk(tree) + ) + + +def test_cross_process_source_and_environment_manifest(tmp_path: Path) -> None: + (tmp_path / "module.py").write_text("# immutable baseline fixture\n") + distribution = [["torch", "exact"]] + environment = { + "qualification": False, + "versions": {}, + "torchao": None, + "python": "fixed", + "threads": 4, + "device": "cpu", + "torch_cuda": None, + "hardware_preflight_only": False, + "installed_distributions": distribution, + "installed_distributions_sha256": fingerprint(distribution), + "source_imports": { + name: {"relative_path": "module.py", "sha256": digest(tmp_path / "module.py")} + for name in ("sdk", "server", "adapter") + }, + } + row: dict[str, Any] = { + "source_commit": BASE_REVISION, + "smoke": True, + "environment": environment, + "fixtures": {}, + "actual_precision": "float32", + "base_scores": [1.0], + "shape_oracles": {"a": {"scores": [1.0]}}, + "required_telemetry_preflight": observed(), + } + other = copy.deepcopy(row) + probe.verify_process_sources([row, other], tmp_path) + other["environment"]["installed_distributions"] = [["torch", "different"]] + other["environment"]["installed_distributions_sha256"] = fingerprint( + other["environment"]["installed_distributions"] + ) + with pytest.raises(AssertionError, match="environments differ"): + probe.verify_process_sources([row, other], tmp_path) + other = copy.deepcopy(row) + other["required_telemetry_preflight"]["affinity"] = [0] + with pytest.raises(AssertionError, match="affinity differs"): + probe.verify_process_sources([row, other], tmp_path) + (tmp_path / "module.py").write_text("# changed\n") + with pytest.raises(AssertionError, match="imported source differs"): + probe.verify_process_sources([row], tmp_path) + + +@pytest.fixture(scope="module") +def real_inputs(tmp_path_factory: pytest.TempPathFactory) -> tuple[Path, Path, Path]: + directory = tmp_path_factory.mktemp("stability-real") + available = subprocess.run(["git", "cat-file", "-e", BASE_REVISION], cwd=ROOT, capture_output=True, check=False) # noqa: S607 + if available.returncode: + pytest.skip("immutable B smoke requires the documented baseline in git history") + snapshot = tiny_snapshot(directory / "model") + fixtures, inputs = directory / "fixtures", directory / "inputs" + prepare(fixtures, snapshot, smoke=True) + source_preparer.prepare(ROOT, inputs, "HEAD") + return directory, fixtures, inputs + + +def test_real_immutable_baseline_nine_window_sdk_smoke(real_inputs: tuple[Path, Path, Path]) -> None: + directory, fixtures, inputs = real_inputs + output = directory / "diagnostic" + result = subprocess.run( + [ + sys.executable, + "-m", + "tools.benchmarks.cross_encoder_lora.stability", + "--inputs", + str(inputs), + "--fixtures", + str(fixtures), + "--output", + str(output), + "--smoke", + "--deadline-s", + "300", + "--trial-deadline-s", + "90", + ], + cwd=ROOT, + env=source_environment(ROOT), + capture_output=True, + text=True, + timeout=150, + check=False, + ) + summary = json.loads((output / "summary.json").read_text()) + assert result.returncode == 0, result.stdout + result.stderr + (output / "process-0.log").read_text() + assert summary["completed"] + assert not summary["qualification"] + assert not summary["statistical_stability_passed"] + process = json.loads((output / "process-0/process.json").read_text()) + assert process["shutdown_complete"] + assert not process["environment"]["qualification"] + assert len(process["windows"]) == 9 + assert process["source_commit"] == BASE_REVISION + for row in process["windows"]: + window = probe.reopen_window(output / "process-0", row) + assert window["warmups"] == 2 + assert not window["qualification"] + assert all(batch["forwards"] for replay in process["untimed_replays"].values() for batch in replay["batches"]) + probe.verify_process_sources([process], output / "sources/B") + + +def test_required_preflight_and_live_failure_cleanup(real_inputs: tuple[Path, Path, Path]) -> None: + directory, fixtures, inputs = real_inputs + sources = directory / "failure-sources" + extract(inputs, sources) + source = sources / "B" + for failure in ("preflight", "window"): + output = directory / f"failed-{failure}" + script = """ +import argparse, asyncio, json +from pathlib import Path +from tools.benchmarks.cross_encoder_lora import stability as s +def fail(*args, **kwargs): + raise RuntimeError('injected diagnostic failure') +args = argparse.Namespace(source=Path(SOURCE), fixtures=Path(FIXTURES), output=Path(OUTPUT), process=0, + smoke=True, trial_deadline_s=90) +if FAILURE == 'preflight': + s.snapshot = fail + s.serve_windows = lambda *args: (_ for _ in ()).throw(AssertionError('model work before telemetry')) +else: + s.measure_window = fail +asyncio.run(s.child(args)) +""" + bindings = f"SOURCE={str(source)!r}\nFIXTURES={str(fixtures)!r}\nOUTPUT={str(output)!r}\nFAILURE={failure!r}\n" + result = subprocess.run( + [sys.executable, "-c", bindings + script], + cwd=source, + env=source_environment(source), + capture_output=True, + text=True, + timeout=130, + check=False, + ) + assert result.returncode != 0 + record = json.loads((output / "process.json").read_text()) + assert not record["completed"] + assert not record["qualification"] + assert record["error"]["message"] == "injected diagnostic failure", result.stdout + result.stderr + assert record["shutdown_complete"] == (failure == "window") + if failure == "preflight": + assert "environment" not in record + else: + assert record["untimed_replays"]["16"]["batches"] From 84e2b18bade8aa504bb153f7a33e7c06aa115860 Mon Sep 17 00:00:00 2001 From: svonava Date: Fri, 16 Oct 2026 19:46:22 -0700 Subject: [PATCH 09/16] fix(benchmarks): retain lightweight response evidence safely --- .../tests/test_client_lifecycle.py | 371 ++++++++++++++++++ tools/benchmarks/cross_encoder_lora/worker.py | 44 ++- 2 files changed, 406 insertions(+), 9 deletions(-) create mode 100644 tools/benchmarks/cross_encoder_lora/tests/test_client_lifecycle.py diff --git a/tools/benchmarks/cross_encoder_lora/tests/test_client_lifecycle.py b/tools/benchmarks/cross_encoder_lora/tests/test_client_lifecycle.py new file mode 100644 index 000000000..93d899001 --- /dev/null +++ b/tools/benchmarks/cross_encoder_lora/tests/test_client_lifecycle.py @@ -0,0 +1,371 @@ +"""Wire evidence survives without the benchmark retaining successful responses.""" + +from __future__ import annotations + +import asyncio +import gc +import json +import weakref +from collections.abc import Callable, Iterator +from pathlib import Path +from typing import Any + +import httpx +import pytest +from sie_sdk._msgpack import packb, unpackb +from sie_sdk.client.errors import ServerError, SIEConnectionError + +from tools.benchmarks.cross_encoder_lora import worker +from tools.benchmarks.cross_encoder_lora.evidence import Evidence + +HTTP_CLIENT = httpx.Client +URL = "http://client-lifecycle.invalid" +TIMING = { + "x-queue-time": "1.25", + "x-tokenization-time": "0.02", + "x-inference-time": "3.75", + "x-postprocessing-time": "0.01", + "x-total-time": "5.03", +} + + +def success(request: httpx.Request) -> httpx.Response: + body = unpackb(request.content, numeric_arrays=False) + return httpx.Response( + 200, + content=packb( + { + "model": worker.MODEL, + "scores": [ + {"item_id": item["id"], "score": float(4 - index), "rank": index} + for index, item in enumerate(body["items"]) + ], + "usage": {"input_tokens": 17}, + } + ), + headers={"content-type": "application/msgpack", **TIMING}, + ) + + +def install_transport( + monkeypatch: pytest.MonkeyPatch, handler: Callable[[httpx.Request], httpx.Response] +) -> list[weakref.ReferenceType[httpx.Client]]: + clients = [] + + def factory(*args: Any, **kwargs: Any) -> httpx.Client: + client = HTTP_CLIENT(*args, **kwargs, transport=httpx.MockTransport(handler)) + clients.append(weakref.ref(client)) + return client + + monkeypatch.setattr(worker.httpx, "Client", factory) + return clients + + +class TrackedStream(httpx.SyncByteStream): + def __init__(self, body: bytes) -> None: + self.body = body + self.reads = 0 + + def __iter__(self) -> Iterator[bytes]: + self.reads += 1 + yield self.body + + +def test_success_hook_snapshots_only_primitive_status_headers_without_reading_body() -> None: + client = object.__new__(worker.Client) + client.responses = [] + stream = TrackedStream(b"body must remain unread by hook") + response = httpx.Response(200, headers={**TIMING, "unrelated": "ignored"}, stream=stream) + client.observe(response) + assert stream.reads == 0 + assert client.responses == [{"status": 200, "timing_headers": TIMING}] + assert json.loads(json.dumps(client.responses)) == client.responses + response.headers["x-total-time"] = "changed" + assert client.responses[0]["timing_headers"]["x-total-time"] == "5.03" + + +def test_actual_sdk_success_history_does_not_retain_response_or_closed_owner( + monkeypatch: pytest.MonkeyPatch, +) -> None: + responses = [] + + def handler(request: httpx.Request) -> httpx.Response: + response = success(request) + responses.append(weakref.ref(response)) + return response + + install_transport(monkeypatch, handler) + client = worker.Client(URL) + rows = [client.score(None, f"request-{i}", timed=True) for i in range(32)] + for row in rows: + worker.validate_response(row) + before = json.dumps(rows, sort_keys=True) + assert all(row["responses"] == [{"status": 200, "timing_headers": TIMING}] for row in rows) + # Ordinary HTTPX response-stream cycles remain. They can be collected while + # this live benchmark Client still owns all of its primitive wire history. + gc.collect() + assert len(responses) == 32 + assert all(ref() is None for ref in responses) + sdk, http = client.sdk, client._http + request_hooks = list(http.event_hooks["request"]) + + def unrelated(_: httpx.Response) -> None: + pass + + http.event_hooks["response"].append(unrelated) + owner = weakref.ref(client) + client.close() + assert http.is_closed + assert not sdk._finalizer.alive + assert http.event_hooks["response"] == [unrelated] + assert http.event_hooks["request"] == request_hooks + assert client.responses == [] + assert json.dumps(rows, sort_keys=True) == before + del client + # SDK/http are intentionally still live: the removed benchmark hook must + # no longer keep its owner alive. No library-owned cycle is modified. + assert owner() is None + assert sdk.jobs._c is sdk + + +@pytest.mark.parametrize("encoding", ["json", "msgpack", "malformed"]) +def test_error_body_and_retry_evidence_are_finalized_outside_hook(encoding: str) -> None: + body = {"error": {"code": "LORA_LOADING", "message": "adapter is loading"}} + content = json.dumps(body).encode() if encoding == "json" else packb(body) + if encoding == "malformed": + content = b"not JSON" + stream = TrackedStream(content) + response = httpx.Response( + 503, + headers={ + "content-type": "application/msgpack" if encoding == "msgpack" else "application/json", + "retry-after": "0.2", + **TIMING, + }, + stream=stream, + ) + client = object.__new__(worker.Client) + client.responses = [] + client.observe(response) + assert stream.reads == 0 + assert client.responses[0] is response + row = {"responses": client.responses.copy()} + worker.serialize_responses(row) + evidence = row["responses"][0] + assert stream.reads == 1 + assert evidence["status"] == 503 + assert evidence["timing_headers"] == TIMING + assert evidence["body_hex"] == content.hex() + assert evidence["retry_after"] == "0.2" + if encoding == "malformed": + assert evidence["body_decode_error"] == "JSONDecodeError" + assert evidence["code"] is None + else: + assert evidence["code"] == "LORA_LOADING" + json.dumps(row) + + +@pytest.mark.parametrize(("code", "cold"), [("LORA_LOADING", True), ("MODEL_LOADING", False)]) +def test_actual_sdk_attempts_preserve_cold_retry_and_reject_hidden_retry( + monkeypatch: pytest.MonkeyPatch, code: str, cold: bool +) -> None: + attempts = 0 + + def handler(request: httpx.Request) -> httpx.Response: + nonlocal attempts + attempts += 1 + if attempts == 1: + return httpx.Response( + 503, + content=packb({"error": {"code": code, "message": "loading"}}), + headers={"content-type": "application/msgpack", "retry-after": "0", **TIMING}, + ) + return success(request) + + install_transport(monkeypatch, handler) + client = worker.Client(URL) + try: + row = client.score("a" if cold else None, "retry", cold=cold, timed=True) + if cold: + worker.validate_response(row, allow_loading=True) + else: + with pytest.raises(AssertionError, match="hidden SDK retry"): + worker.validate_response(row) + assert attempts == 2 + assert [value["status"] for value in row["responses"]] == [503, 200] + assert row["responses"][0]["code"] == code + assert row["responses"][0]["retry_after"] == "0" + assert row["responses"][1]["timing_headers"] == TIMING + saved = json.dumps(row, sort_keys=True) + finally: + client.close() + assert json.dumps(row, sort_keys=True) == saved + + +def test_partial_performance_failure_preserves_unreturned_wire_attempt_before_close( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + attempts = 0 + closed = [] + original_close = worker.Client.close + + def handler(request: httpx.Request) -> httpx.Response: + nonlocal attempts + attempts += 1 + if attempts == 5: + return httpx.Response( + 500, + content=packb({"error": {"code": "INTERNAL_ERROR", "message": "injected failure"}}), + headers={"content-type": "application/msgpack", **TIMING}, + ) + return success(request) + + def close(client: worker.Client) -> None: + original_close(client) + closed.append(client._http.is_closed and not client.responses) + + install_transport(monkeypatch, handler) + monkeypatch.setattr(worker.Client, "close", close) + evidence = Evidence(tmp_path / "evidence") + with pytest.raises(ServerError): + worker.performance(URL, 1, evidence, smoke=True) + record = json.loads((evidence.directory / "events.jsonl").read_text()) + assert record["event"] == "performance_attempt" + assert len(record["warmups"]) == 2 + assert len(record["completed_samples"]) == 2 + assert [row["status"] for row in record["wire_responses"]] == [200, 200, 200, 200, 500] + failure = record["wire_responses"][-1] + assert failure["code"] == "INTERNAL_ERROR" + assert unpackb(bytes.fromhex(failure["body_hex"]), numeric_arrays=False)["error"]["message"] == "injected failure" + assert closed == [True] + + +def test_owned_hooks_and_history_clear_even_if_sdk_close_raises(monkeypatch: pytest.MonkeyPatch) -> None: + install_transport(monkeypatch, success) + client = worker.Client(URL) + row = client.score(None, "close-failure") + saved = json.dumps(row, sort_keys=True) + http = client._http + + def unrelated(_: httpx.Response) -> None: + pass + + def fail() -> None: + raise OSError("injected close failure") + + http.event_hooks["response"].append(unrelated) + monkeypatch.setattr(client.sdk, "close", fail) + try: + with pytest.raises(OSError, match="injected close failure"): + client.close() + assert http.event_hooks["response"] == [unrelated] + assert client.responses == [] + assert json.dumps(row, sort_keys=True) == saved + finally: + http.close() + + +def test_partially_read_error_stream_preserves_original_sdk_failure_and_all_attempts( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + attempts = 0 + closed = [] + original_close = worker.Client.close + + class BrokenStream(httpx.SyncByteStream): + def __iter__(self) -> Iterator[bytes]: + yield b'{"error":' + raise httpx.ReadError("injected partial error body") + + def handler(request: httpx.Request) -> httpx.Response: + nonlocal attempts + attempts += 1 + if attempts == 5: + return httpx.Response( + 500, + stream=BrokenStream(), + headers={"content-type": "application/json", "retry-after": "0.2", **TIMING}, + ) + return success(request) + + def close(client: worker.Client) -> None: + original_close(client) + closed.append(client._http.is_closed and not client.responses) + + install_transport(monkeypatch, handler) + monkeypatch.setattr(worker.Client, "close", close) + evidence = Evidence(tmp_path / "evidence") + with pytest.raises(SIEConnectionError, match="Connection lost mid-request") as failure: + worker.performance(URL, 1, evidence, smoke=True) + assert isinstance(failure.value.__cause__, httpx.ReadError) + record = json.loads((evidence.directory / "events.jsonl").read_text()) + assert len(record["warmups"]) == 2 + assert len(record["completed_samples"]) == 2 + assert [row["status"] for row in record["wire_responses"]] == [200, 200, 200, 200, 500] + final = record["wire_responses"][-1] + assert final["timing_headers"] == TIMING + assert final["content_type"] == "application/json" + assert final["retry_after"] == "0.2" + assert final["body_unavailable"] is True + assert final["body_read_error"] == "StreamConsumed" + assert final["code"] is None + assert "body_hex" not in final + assert closed == [True] + + +@pytest.mark.parametrize("body_failure", ["transport", "decoder"]) +def test_unavailable_error_body_retains_metadata_for_transport_and_decoder_failures(body_failure: str) -> None: + class BrokenStream(httpx.SyncByteStream): + def __iter__(self) -> Iterator[bytes]: + raise httpx.ReadError("injected stream failure") + yield b"" # pragma: no cover + + headers = {"content-type": "application/json", "retry-after": "0.2", **TIMING} + if body_failure == "decoder": + headers["content-encoding"] = "gzip" + stream = httpx.ByteStream(b"this is not gzip") + else: + stream = BrokenStream() + response = httpx.Response(500, headers=headers, stream=stream) + record = worker.describe_response(response) + assert record["status"] == 500 + assert record["timing_headers"] == TIMING + assert record["content_type"] == "application/json" + assert record["retry_after"] == "0.2" + assert record["body_unavailable"] is True + assert record["body_read_error"] == ("DecodingError" if body_failure == "decoder" else "ReadError") + assert "body_hex" not in record + + +@pytest.mark.parametrize("cancel_type", [KeyboardInterrupt, SystemExit, asyncio.CancelledError]) +def test_deferred_error_body_does_not_swallow_cancellation(cancel_type: type[BaseException]) -> None: + class CancelledStream(httpx.SyncByteStream): + def __iter__(self) -> Iterator[bytes]: + raise cancel_type("cancelled") + yield b"" # pragma: no cover + + response = httpx.Response(500, stream=CancelledStream()) + with pytest.raises(cancel_type): + worker.describe_response(response) + + +def test_performance_evidence_failure_still_closes_all_clients(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: + closed = [] + original_close = worker.Client.close + install_transport(monkeypatch, success) + + def close(client: worker.Client) -> None: + original_close(client) + closed.append(client._http.is_closed and not client.responses) + + def emit(*_: Any, **__: Any) -> None: + raise OSError("injected evidence failure") + + monkeypatch.setattr(worker.Client, "close", close) + evidence = Evidence(tmp_path / "evidence") + monkeypatch.setattr(evidence, "emit", emit) + gc_enabled = gc.isenabled() + with pytest.raises(OSError, match="injected evidence failure"): + worker.performance(URL, 4, evidence, smoke=True) + assert closed == [True] * 4 + assert gc.isenabled() == gc_enabled diff --git a/tools/benchmarks/cross_encoder_lora/worker.py b/tools/benchmarks/cross_encoder_lora/worker.py index 625706641..15c0ca746 100644 --- a/tools/benchmarks/cross_encoder_lora/worker.py +++ b/tools/benchmarks/cross_encoder_lora/worker.py @@ -68,6 +68,7 @@ MAX_LOADING_DELAY = 5 RESIDENT_CAPACITY = 2 REQUEST_ITEMS = len(DOCUMENTS) +RESPONSE_CAPTURE_MODE = "success-status-header-snapshots" def error_record(error: BaseException) -> dict[str, Any]: @@ -204,13 +205,15 @@ def adapter_storage(adapter: Any) -> dict[str, int]: class Client: def __init__(self, url: str) -> None: - self.responses: list[httpx.Response] = [] + self.responses: list[dict[str, Any] | httpx.Response] = [] http = httpx.Client(base_url=url, timeout=30, event_hooks={"response": [self.observe]}) + self._http = http self.sdk = SIEClient(url, timeout_s=30, http_client=http) def observe(self, response: httpx.Response) -> None: - # The only extra HTTP-hook work in timed phases. Decode after clocks stop. - self.responses.append(response) + # Keep successful wire evidence without retaining its Response/Request/ + # stream graph for the whole phase. Failure bodies are decoded later. + self.responses.append(describe_response(response) if response.status_code == HTTPStatus.OK else response) def score(self, selection: str | None, label: str, *, cold: bool = False, timed: bool = False) -> dict[str, Any]: started = time.perf_counter_ns() @@ -256,10 +259,25 @@ def score(self, selection: str | None, label: str, *, cold: bool = False, timed: return row def close(self) -> None: - self.sdk.close() + try: + self.sdk.close() + finally: + # Callers serialize their evidence before closing. Preserve unrelated + # hooks, and break only the observer/history this benchmark owns. + hooks = self._http.event_hooks["response"] + hooks[:] = [ + hook + for hook in hooks + if not ( + getattr(hook, "__self__", None) is self and getattr(hook, "__func__", None) is type(self).observe + ) + ] + self.responses.clear() -def describe_response(response: httpx.Response) -> dict[str, Any]: +def describe_response(response: httpx.Response | dict[str, Any]) -> dict[str, Any]: + if not isinstance(response, httpx.Response): + return response row: dict[str, Any] = { "status": response.status_code, "timing_headers": { @@ -275,7 +293,17 @@ def describe_response(response: httpx.Response) -> dict[str, Any]: }, } if response.status_code >= HTTPStatus.BAD_REQUEST: - response.read() + row["content_type"] = response.headers.get("content-type") + row["retry_after"] = response.headers.get("retry-after") + try: + response.read() + except (httpx.StreamError, httpx.HTTPError) as error: + # A partially read error stream can already have failed in the SDK. + # Preserve that original failure and its wire metadata; diagnostic + # re-reading must not replace it or discard earlier attempts. + row.update(body_unavailable=True, body_read_error=type(error).__name__, code=None) + return row + row["body_hex"] = response.content.hex() try: body = ( unpackb(response.content, numeric_arrays=False) @@ -286,9 +314,7 @@ def describe_response(response: httpx.Response) -> dict[str, Any]: row["body_decode_error"] = type(error).__name__ body = None error = body.get("error", body.get("detail", body)) if isinstance(body, dict) else None - row.update( - code=error.get("code") if isinstance(error, dict) else None, retry_after=response.headers.get("retry-after") - ) + row["code"] = error.get("code") if isinstance(error, dict) else None return row From 88c4053c3a9b1522bb817b0add2729a67d4c6c8d Mon Sep 17 00:00:00 2001 From: svonava Date: Fri, 16 Oct 2026 19:43:50 -0700 Subject: [PATCH 10/16] test(benchmarks): extend CrossEncoder qualification windows --- tools/benchmarks/cross_encoder_lora/README.md | 82 ++++++++++++++---- .../benchmarks/cross_encoder_lora/protocol.py | 12 +-- tools/benchmarks/cross_encoder_lora/runner.py | 41 ++++++--- .../cross_encoder_lora/stability.py | 59 ++++++++++--- .../tests/test_runner_budget.py | 67 ++++++++++++--- .../tests/test_stability.py | 86 ++++++++++++++++--- 6 files changed, 281 insertions(+), 66 deletions(-) diff --git a/tools/benchmarks/cross_encoder_lora/README.md b/tools/benchmarks/cross_encoder_lora/README.md index eb0b40f3a..c46a0537c 100644 --- a/tools/benchmarks/cross_encoder_lora/README.md +++ b/tools/benchmarks/cross_encoder_lora/README.md @@ -43,15 +43,21 @@ benchmark only inspects them and fails on a mismatch. ```console mise exec -- uv run --no-sync python -m tools.benchmarks.cross_encoder_lora.runner \ --inputs /tmp/ce-sources --fixtures /tmp/ce-fixtures \ - --output /tmp/ce-evidence --deadline-s 1740 --trial-deadline-s 120 + --output /tmp/ce-evidence --deadline-s 4200 --trial-deadline-s 270 ``` Use a fresh output directory. Set the outer compute-session deadline before launching, allowing bounded startup and cleanup. The runner owns and joins only its subprocesses; it cannot terminate an external GPU session. The operator must stop that exact session and record cleanup. A failed or incomplete trial -stops the experiment and retains its evidence. Never change thresholds or drop -failed trials to obtain a pass. +stops the experiment and retains its evidence. Each new trial requires strictly +more than its 270-second limit plus 90 seconds for wait, termination and kill +grace (45/35/10). The proposed outer limits are 4,500 seconds for the operator +and 4,800 seconds from provider start, with a 300-second startup cutoff and an +additional absolute ceiling of application creation plus 5,100 seconds. The +operator must reserve export time before starting the runner; these settings +do not themselves authorize or allocate compute. Never change thresholds or +drop failed trials to obtain a pass, or retry the same failed protocol unchanged. ## Frozen comparison @@ -61,11 +67,13 @@ N uses the candidate with no declared LoRA. R uses the candidate after A/B activity, with base selected and two adapters resident but disabled. Every arm uses the same password-reset query and four documents in `protocol.py`. -Each phase starts with 30 sequential warmup requests. C1 measures 200 sequential -requests; C4 and C16 each run a 12-second closed loop with one SDK client per +Each phase starts with 30 sequential warmup requests. C1 measures 2,000 sequential +requests; C4 and C16 each run a 60-second closed loop with one SDK client per lane. Throughput includes draining requests admitted before the deadline. Latencies use the nearest-rank p95. Neither warmup nor correctness calls enter -the primary statistics. +the primary statistics. Concurrent futures have a 120-second wait limit: the +60-second measurement, the existing 30-second request timeout, and 30 seconds +of scheduling margin. The worker and outer process limits still bound failures. All six metrics are required for **both** N/B and R/B: C1 mean and p95 latency, C4/C16 requests per second and p95 latency. The statistical unit is the six @@ -105,11 +113,13 @@ forward attribute lookup is restored: an original instance override is restored exactly, or the temporary instance attribute is deleted to restore descriptor lookup. Production `score_pairs` and `set_active_lora` are bound directly. The harness checks the original forward target and that no observer activity -occurred, including restoration after an exceptional exit. Response -validation and header decoding happen after the phase clock stops; exactly one -successful HTTP response is required per warmup/measured SDK call, excluding -hidden retries. The timed client retains timer, result-list and raw-response -list-append overhead; this is an SDK workload measurement. +occurred, including restoration after an exceptional exit. Successful response +status and timing headers are captured as small metadata snapshots in the HTTP +observer, without retaining the successful HTTPX response. Error responses stay +available for deferred body and retry evidence. Validation happens after the +phase clock stops; exactly one successful HTTP response is required per +warmup/measured SDK call, excluding hidden retries. The timed client retains +timer, result-list and snapshot overhead; this is an SDK workload measurement. Actual timed C4/C16 batch shapes are **unknown**. Their raw four-score vectors are compared diagnostically against complete preselection oracle vectors at @@ -132,11 +142,49 @@ live parameter allocation. The memory reference is captured after finite preselection base and shape-oracle warmup, before any LoRA attachment, to keep one-time CUDA library initialization separate from adapter storage. -These observation changes are an explicit **preallocation protocol amendment** -to the original plan of observing every timed forward. All trial orders, -warmup/request counts, phase durations, numerical tolerances and statistical -gates remain unchanged. The original timed batch-attribution claim is withdrawn; -the observed untimed replays supply that evidence instead. +Removing timed forward observation was an earlier **preallocation protocol +amendment**. It preserved that version's trial order, workload and gates, and +withdrew its timed batch-attribution claim. Observed untimed replays supply that +evidence instead. + +## Longer-window amendment + +The original qualification used 200 C1 requests and 12-second C4/C16 phases. +Its failed full-run evidence and the failed baseline-only stability diagnostic +remain separate, with their original source and artifact hashes. Those outcomes +are not reinterpreted as passes. The diagnostic's longer warmup did not establish +stability, so qualification retains 30 sequential warmup requests. + +The revised workload uses 2,000 C1 requests and 60-second concurrent phases to +observe longer intervals. The six balanced fresh-process triplets, checkpoint, +precision, dependencies, request payloads, numerical tolerances and all twelve +statistical gates stay fixed. Longer windows may reduce within-window sampling +variation; they do not guarantee comparability across fresh processes. There +are still only six independent paired units, not thousands of independent +latency samples for the t interval. + +A model-free lifetime probe separately established that the old observer held +successful responses for the whole phase and kept its client alive through a +bound callback. Metadata snapshots and removal of the owned hook/history on +close address that benchmark-owned retention. Normal HTTPX response-stream and +SDK namespace cycles remain, with automatic GC enabled. This finding does not +identify the cause of earlier GPU timing variation. Snapshot capture adds timed +metadata work; its CPU cost must be recorded, without subtracting it from the +measurements or claiming zero overhead. Client and server still share a process, +so the result includes their normal GC and GIL interactions. + +Freeze the reviewed source and operator bounds before one complete amended +18-process run. A separate baseline-readiness allocation is not required. Every +N/B and R/B gate and every correctness/lifecycle check must pass. If any fails, +retain all evidence and investigate before another method change; do not retry +unchanged, select favorable windows or pairs, waive N failures, or loosen gates. + +The optional `stability` diagnostic keeps its original 200-request/12-second +windows and 90-second future wait through scoped settings that are restored on +success and failure. It records the current observer mode and worker source +hash. A new invocation uses this checkout's snapshot observer and is **not** a +byte-identical replay of the earlier diagnostic. Exact historical replay uses +commit `482b599224920a97bb2fce4a740317efb89351ab` and its original protocol. ## Evidence and CPU checks @@ -145,6 +193,8 @@ gates and source hashes. Each entry links its complete `trial.json` by relative path and SHA-256; raw request arrays and oracle corpora stay in those per-trial files instead of being copied into the growing summary. Final cross-process checks reopen the complete records and verify their hashes and identities. +The runner records its monotonic start/deadline and every admission timestamp, +full cleanup reserve and remaining margin, including a refused admission. The summary also hashes the complete benchmark file manifest. Each trial captures the full installed-distribution manifest; its digest must remain identical across arms. An optional `--environment-receipt` attaches a separate diff --git a/tools/benchmarks/cross_encoder_lora/protocol.py b/tools/benchmarks/cross_encoder_lora/protocol.py index 6a40f19bc..b1cd3b7f7 100644 --- a/tools/benchmarks/cross_encoder_lora/protocol.py +++ b/tools/benchmarks/cross_encoder_lora/protocol.py @@ -1,4 +1,4 @@ -"""Preregistered workload and paired non-inferiority gates (2026-10-16).""" +"""Pinned workload and paired gates; README records the longer-window amendment.""" from __future__ import annotations @@ -25,13 +25,15 @@ EFFECT_DRIFT_MULTIPLIER = 5 ORDERS = ("BNR", "NRB", "RBN", "BRN", "RNB", "NBR") WARMUP = 30 -SEQUENTIAL = 200 -STEADY_SECONDS = 12 +SEQUENTIAL = 2000 +STEADY_SECONDS = 60 CONCURRENCIES = (4, 16) THREADS = 4 MAX_BATCH_ITEMS = 64 -MAX_TRIAL_SECONDS = 300 -MAX_EXPERIMENT_SECONDS = 3600 +MAX_TRIAL_SECONDS = 270 +MAX_EXPERIMENT_SECONDS = 4200 +# Measured concurrent phase + request timeout + scheduling margin. +PERFORMANCE_FUTURE_TIMEOUT_S = 120 EXPECTED_VERSIONS = { "torch": "2.9.1+cu129", "transformers": "4.57.6", diff --git a/tools/benchmarks/cross_encoder_lora/runner.py b/tools/benchmarks/cross_encoder_lora/runner.py index bc8c078bb..bac07e12b 100644 --- a/tools/benchmarks/cross_encoder_lora/runner.py +++ b/tools/benchmarks/cross_encoder_lora/runner.py @@ -193,9 +193,17 @@ def run(args: argparse.Namespace) -> dict[str, Any]: "smoke": args.smoke, "orders": ORDERS, "trials": [], + "admissions": [], } summary_path = args.output / "summary.json" - absolute_deadline = time.monotonic() + args.deadline_s + started = time.monotonic() + absolute_deadline = started + args.deadline_s + summary["runner_clock"] = { + "monotonic_start_s": started, + "wall_start_ns": time.time_ns(), + "deadline_monotonic_s": absolute_deadline, + "budget_s": args.deadline_s, + } if args.environment_receipt is not None: summary["environment_preparation"] = { "sha256": digest(args.environment_receipt), @@ -204,15 +212,26 @@ def run(args: argparse.Namespace) -> dict[str, Any]: try: schedule = [(0, "R")] if args.smoke else [(pair, arm) for pair, order in enumerate(ORDERS) for arm in order] for pair, arm in schedule: - require( - time.monotonic() - + args.trial_deadline_s - + WAIT_GRACE_SECONDS - + TERMINATE_GRACE_SECONDS - + KILL_GRACE_SECONDS - < absolute_deadline, - "insufficient time for a complete bounded next trial and cleanup", + admitted_at = time.monotonic() + cleanup_deadline = ( + admitted_at + args.trial_deadline_s + WAIT_GRACE_SECONDS + TERMINATE_GRACE_SECONDS + KILL_GRACE_SECONDS + ) + admitted = cleanup_deadline < absolute_deadline + summary["admissions"].append( + { + "pair": pair, + "arm": arm, + "monotonic_s": admitted_at, + "deadline_monotonic_s": absolute_deadline, + "trial_limit_s": args.trial_deadline_s, + "reserve_s": ( + args.trial_deadline_s + WAIT_GRACE_SECONDS + TERMINATE_GRACE_SECONDS + KILL_GRACE_SECONDS + ), + "margin_s": absolute_deadline - cleanup_deadline, + "admitted": admitted, + } ) + require(admitted, "insufficient time for a complete bounded next trial and cleanup") directory = args.output / f"trial-{pair}-{arm}" source = (args.output / "sources" / ("B" if arm == "B" else "candidate")).resolve() command = [ @@ -313,8 +332,8 @@ def main() -> None: help="Optional operator-owned JSON receipt of environment preparation (no secrets)", ) parser.add_argument("--output", type=Path, required=True) - parser.add_argument("--deadline-s", type=float, default=2400) - parser.add_argument("--trial-deadline-s", type=float, default=120) + parser.add_argument("--deadline-s", type=float, default=MAX_EXPERIMENT_SECONDS) + parser.add_argument("--trial-deadline-s", type=float, default=MAX_TRIAL_SECONDS) parser.add_argument("--smoke", action="store_true") args = parser.parse_args() require( diff --git a/tools/benchmarks/cross_encoder_lora/stability.py b/tools/benchmarks/cross_encoder_lora/stability.py index a4d5f05c5..dc4bc1779 100644 --- a/tools/benchmarks/cross_encoder_lora/stability.py +++ b/tools/benchmarks/cross_encoder_lora/stability.py @@ -2,9 +2,10 @@ Run with --inputs, --fixtures and --output. Six fresh baseline processes each execute C1/C4/C16 with fixed S/L1/L2 windows: 30/200/200 sequential warmup -requests followed by the existing 200-request/12-second measurement. Later +requests followed by a fixed 200-request/12-second measurement. Later window agreement is tested symmetrically; between-process dispersion remains -descriptive. The original timed methods and automatic GC behavior are reused. +descriptive. The current checkout's client observer and automatic GC are used; +exact historical replay requires the recorded original source commit. """ # Only this checkout's Python worker runs; no shell or allocation. @@ -25,6 +26,8 @@ import subprocess import sys import time +from collections.abc import Iterator +from contextlib import contextmanager from http import HTTPStatus from pathlib import Path from typing import Any @@ -42,8 +45,6 @@ MODEL, ORACLE_ATOL, RESTORE_ATOL, - SEQUENTIAL, - STEADY_SECONDS, THREADS, WEIGHTS_SHA256, paired_gate, @@ -64,6 +65,11 @@ PROCESSES = 6 WINDOWS = (("S", 30), ("L1", 200), ("L2", 200)) CONCURRENCIES = (1, 4, 16) +# Keep this diagnostic bounded independently of later qualification amendments. +SEQUENTIAL = 200 +STEADY_SECONDS = 12 +PERFORMANCE_FUTURE_TIMEOUT_S = 90 +HISTORICAL_SOURCE_REVISION = "482b599224920a97bb2fce4a740317efb89351ab" METRICS = ((1, "mean_ms"), (1, "p95_ms"), (4, "rps"), (4, "p95_ms"), (16, "rps"), (16, "p95_ms")) RUNNER_LIMIT = 1200 WORKER_LIMIT = 180 @@ -87,6 +93,41 @@ def emit(self, event: str, **fields: Any) -> dict[str, Any]: return super().emit(event, **fields) +def measurement_method() -> dict[str, Any]: + return { + "sequential_measured_requests": SEQUENTIAL, + "concurrent_measured_seconds": STEADY_SECONDS, + "future_wait_seconds": PERFORMANCE_FUTURE_TIMEOUT_S, + "response_capture": getattr(worker, "RESPONSE_CAPTURE_MODE", "response-reference-retention"), + "worker_source_sha256": digest(Path(worker.__file__)), + "exact_historical_replay_commit": HISTORICAL_SOURCE_REVISION, + "historical_byte_identity_claimed": False, + } + + +@contextmanager +def diagnostic_settings(warmups: int) -> Iterator[None]: + """Pin the old workload without leaking settings into qualification calls.""" + values = { + "WARMUP": warmups, + "SEQUENTIAL": SEQUENTIAL, + "STEADY_SECONDS": STEADY_SECONDS, + "PERFORMANCE_FUTURE_TIMEOUT_S": PERFORMANCE_FUTURE_TIMEOUT_S, + } + missing = object() + original = {name: getattr(worker, name, missing) for name in values} + try: + for name, value in values.items(): + setattr(worker, name, value) + yield + finally: + for name, value in original.items(): + if value is missing: + delattr(worker, name) + else: + setattr(worker, name, value) + + def unified_cgroup(proc: Path = Path("/proc/self")) -> dict[str, Any]: """Resolve the process's own namespace; never fall back to another cgroup.""" try: @@ -406,18 +447,15 @@ def measure_window( "concurrency": concurrency, "warmups": 2 if smoke else warmups, "smoke": smoke, + "measurement_method": measurement_method(), } - original_warmups = worker.WARMUP try: record["gpu_before"] = worker.telemetry() record["telemetry_before"] = snapshot() if continuity is not None: telemetry_delta(continuity, record["telemetry_before"]) - worker.WARMUP = warmups # ty: ignore[invalid-assignment] -- scoped diagnostic value, always restored - try: + with diagnostic_settings(warmups): measured = worker.performance(url, concurrency, evidence, smoke=smoke) - finally: - worker.WARMUP = original_warmups record["telemetry_after"] = snapshot() record["gpu_after"] = worker.telemetry() record["telemetry_delta"] = telemetry_delta(record["telemetry_before"], record["telemetry_after"]) @@ -439,7 +477,6 @@ def measure_window( record["error"] = worker.error_record(error) raise finally: - worker.WARMUP = original_warmups atomic_json(directory / "window.json", record) return { "qualification": False, @@ -579,6 +616,7 @@ async def child(args: argparse.Namespace) -> dict[str, Any]: "pid": os.getpid(), "source_commit": BASE_REVISION, "smoke": args.smoke, + "measurement_method": measurement_method(), } os.environ.update( { @@ -811,6 +849,7 @@ def run(args: argparse.Namespace) -> dict[str, Any]: "budget_s": args.deadline_s, } summary["methods"] = { + **measurement_method(), "window_order": [{"name": name, "sequential_warmup_calls": count} for name, count in WINDOWS], "phase_order": list(CONCURRENCIES), "fresh_baseline_processes": PROCESSES, diff --git a/tools/benchmarks/cross_encoder_lora/tests/test_runner_budget.py b/tools/benchmarks/cross_encoder_lora/tests/test_runner_budget.py index f755de8e1..f0ec56ca1 100644 --- a/tools/benchmarks/cross_encoder_lora/tests/test_runner_budget.py +++ b/tools/benchmarks/cross_encoder_lora/tests/test_runner_budget.py @@ -6,6 +6,7 @@ import json import signal import subprocess +import sys from pathlib import Path from typing import Any @@ -14,7 +15,7 @@ from tools.benchmarks.cross_encoder_lora import runner -def arguments(tmp_path: Path, budget: float) -> argparse.Namespace: +def arguments(tmp_path: Path, budget: float, trial_limit: float = 120) -> argparse.Namespace: return argparse.Namespace( output=tmp_path / "evidence", inputs=tmp_path / "inputs", @@ -23,7 +24,7 @@ def arguments(tmp_path: Path, budget: float) -> argparse.Namespace: smoke=True, environment_receipt=None, deadline_s=budget, - trial_deadline_s=120, + trial_deadline_s=trial_limit, ) @@ -32,11 +33,13 @@ def setup_sources(monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setattr(runner, "harness_manifest", dict) -@pytest.mark.parametrize(("budget", "admitted"), [(209.999, False), (210.0, False), (210.001, True)]) +@pytest.mark.parametrize("trial_limit", [120, 270]) +@pytest.mark.parametrize(("margin", "admitted"), [(-0.001, False), (0.0, False), (0.001, True)]) def test_trial_admission_reserves_full_cleanup( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, - budget: float, + trial_limit: float, + margin: float, admitted: bool, ) -> None: setup_sources(monkeypatch) @@ -64,10 +67,10 @@ def start(command: list[str], **_kwargs: Any) -> CompletedProcess: return CompletedProcess() monkeypatch.setattr(runner.subprocess, "Popen", start) - args = arguments(tmp_path, budget) + args = arguments(tmp_path, trial_limit + 90 + margin, trial_limit) if admitted: assert runner.run(args)["passed"] - assert waits == [165] + assert waits == [trial_limit + 45] else: with pytest.raises(AssertionError, match="insufficient time"): runner.run(args) @@ -75,11 +78,25 @@ def start(command: list[str], **_kwargs: Any) -> CompletedProcess: assert not terminal["passed"] assert terminal["trials"] == [] assert bool(started) is admitted - - + terminal = json.loads((args.output / "summary.json").read_text()) + assert terminal["runner_clock"]["monotonic_start_s"] == 100 + assert terminal["runner_clock"]["budget_s"] == args.deadline_s + assert terminal["runner_clock"]["deadline_monotonic_s"] == 100 + args.deadline_s + assert terminal["runner_clock"]["wall_start_ns"] > 0 + assert len(terminal["admissions"]) == 1 + admission = terminal["admissions"][0] + assert (admission["pair"], admission["arm"], admission["monotonic_s"]) == (0, "R", 100) + assert admission["trial_limit_s"] == trial_limit + assert admission["reserve_s"] == trial_limit + 90 + assert admission["margin_s"] == pytest.approx(margin, abs=1e-9) + assert admission["admitted"] is admitted + + +@pytest.mark.parametrize("trial_limit", [120, 270]) def test_full_timeout_cleanup_finishes_before_budget_and_retains_partial_trial( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, + trial_limit: float, ) -> None: setup_sources(monkeypatch) clock = [100.0] @@ -123,15 +140,43 @@ def start(command: list[str], **_kwargs: Any) -> HungProcess: return HungProcess() monkeypatch.setattr(runner.subprocess, "Popen", start) - args = arguments(tmp_path, 211) + args = arguments(tmp_path, trial_limit + 91, trial_limit) with pytest.raises(subprocess.TimeoutExpired): runner.run(args) - assert actions == [("wait", 165), ("signal", signal.SIGTERM), ("wait", 35), ("kill", None), ("wait", 10)] - assert clock[0] == 310 + assert actions == [ + ("wait", trial_limit + 45), + ("signal", signal.SIGTERM), + ("wait", 35), + ("kill", None), + ("wait", 10), + ] + assert clock[0] == 100 + trial_limit + 90 assert clock[0] < 100 + args.deadline_s terminal = json.loads((args.output / "summary.json").read_text()) assert not terminal["passed"] + assert terminal["admissions"][0]["margin_s"] == 1 + assert terminal["admissions"][0]["admitted"] trial = terminal["trials"][0] assert trial["performance"]["1"]["count"] == 7 assert trial["process_returncode"] == -9 assert trial["runner_error"]["type"] == "TimeoutExpired" + + +def test_default_cli_budget_matches_longer_qualification(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(sys, "argv", ["runner", "--inputs", "i", "--fixtures", "f", "--output", "o"]) + monkeypatch.setattr(runner.signal, "signal", lambda *_: None) + captured = [] + monkeypatch.setattr(runner, "run", captured.append) + runner.main() + assert len(captured) == 1 + assert (captured[0].deadline_s, captured[0].trial_deadline_s) == (4200, 270) + + +@pytest.mark.parametrize(("flag", "value"), [("--deadline-s", "4200.001"), ("--trial-deadline-s", "270.001")]) +def test_cli_rejects_limits_above_frozen_bounds(monkeypatch: pytest.MonkeyPatch, flag: str, value: str) -> None: + monkeypatch.setattr(sys, "argv", ["runner", "--inputs", "i", "--fixtures", "f", "--output", "o", flag, value]) + started = [] + monkeypatch.setattr(runner, "run", started.append) + with pytest.raises(AssertionError, match="deadline outside bounded protocol"): + runner.main() + assert not started diff --git a/tools/benchmarks/cross_encoder_lora/tests/test_stability.py b/tools/benchmarks/cross_encoder_lora/tests/test_stability.py index dc16e939b..c7230fc93 100644 --- a/tools/benchmarks/cross_encoder_lora/tests/test_stability.py +++ b/tools/benchmarks/cross_encoder_lora/tests/test_stability.py @@ -20,7 +20,7 @@ from tools.benchmarks.cross_encoder_lora import worker from tools.benchmarks.cross_encoder_lora.evidence import digest, fingerprint from tools.benchmarks.cross_encoder_lora.fixtures import prepare -from tools.benchmarks.cross_encoder_lora.protocol import BASE_REVISION, SEQUENTIAL +from tools.benchmarks.cross_encoder_lora.protocol import BASE_REVISION from tools.benchmarks.cross_encoder_lora.runner import ROOT, extract, source_environment from tools.benchmarks.cross_encoder_lora.tests.test_probe import tiny_snapshot @@ -225,8 +225,8 @@ def response(label: str, index: int) -> dict[str, Any]: def stub_performance(_url: str, concurrency: int, evidence: probe.DiagnosticEvidence, *, smoke: bool) -> dict[str, Any]: warmups = [response(f"warm-{i}", i) for i in range(2 if smoke else worker.WARMUP)] - samples = [response(f"sample-{i}", i) for i in range(4 if smoke else SEQUENTIAL)] - elapsed = 0.2 if smoke else 12.0 + samples = [response(f"sample-{i}", i) for i in range(4 if smoke else worker.SEQUENTIAL)] + elapsed = 0.2 if smoke else float(worker.STEADY_SECONDS) evidence.emit( "performance_attempt", concurrency=concurrency, @@ -252,18 +252,28 @@ def install_stub(monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setattr(worker, "performance", stub_performance) -def test_raw_retention_reopen_and_warmup_restoration(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: +def current_workload() -> tuple[Any, ...]: + return tuple( + getattr(worker, name, None) + for name in ("WARMUP", "SEQUENTIAL", "STEADY_SECONDS", "PERFORMANCE_FUTURE_TIMEOUT_S") + ) + + +def test_raw_retention_reopen_and_workload_restoration(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: install_stub(monkeypatch) - original = worker.WARMUP + monkeypatch.setattr(worker, "PERFORMANCE_FUTURE_TIMEOUT_S", 120, raising=False) + original = current_workload() continuity = observed(0) continuity["core"]["wall_ns"] = continuity["core"]["monotonic_ns"] = 1 oracles = {"shape": {"copies": 1, "scores": [[4.0, 3.0, 2.0, 1.0][index] for index in worker.ORDER]}} compact = probe.measure_window( "unused", 1, "L1", 200, tmp_path / "C1/L1", [4.0, 3.0, 2.0, 1.0], oracles, smoke=False, continuity=continuity ) - assert original == worker.WARMUP + assert original == current_workload() raw = probe.reopen_window(tmp_path, compact) assert len(raw["performance"]["samples"]) == 200 + assert raw["measurement_method"]["concurrent_measured_seconds"] == 12 + assert raw["measurement_method"]["future_wait_seconds"] == 90 assert len(raw["chronological_quartiles"]) == 4 assert continuity == raw["telemetry_after"] assert all(row["diagnostic_matching_oracle"]["workload_sha256"] == "shape" for row in raw["performance"]["samples"]) @@ -280,12 +290,13 @@ def test_raw_retention_reopen_and_warmup_restoration(tmp_path: Path, monkeypatch probe.reopen_window(tmp_path, compact) -def test_failed_window_keeps_partial_raw_and_restores_warmup(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: +def test_failed_window_keeps_partial_raw_and_restores_workload(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: install_stub(monkeypatch) - original = worker.WARMUP + monkeypatch.setattr(worker, "PERFORMANCE_FUTURE_TIMEOUT_S", 120, raising=False) + original = current_workload() def failed(_url: str, _concurrency: int, evidence: probe.DiagnosticEvidence, *, smoke: bool) -> None: - assert worker.WARMUP == 200 + assert current_workload() == (200, 200, 12, 90) assert not smoke evidence.emit("performance_attempt", completed_samples=[{"error": "retained"}]) raise RuntimeError("request failed") @@ -293,7 +304,7 @@ def failed(_url: str, _concurrency: int, evidence: probe.DiagnosticEvidence, *, monkeypatch.setattr(worker, "performance", failed) with pytest.raises(RuntimeError, match="request failed"): probe.measure_window("unused", 1, "L2", 200, tmp_path / "C1/L2", [4.0, 3.0, 2.0, 1.0], {}, smoke=False) - assert original == worker.WARMUP + assert original == current_workload() terminal = json.loads((tmp_path / "C1/L2/window.json").read_text()) assert not terminal["qualification"] assert not terminal["completed"] @@ -301,6 +312,22 @@ def failed(_url: str, _concurrency: int, evidence: probe.DiagnosticEvidence, *, assert json.loads((tmp_path / "C1/L2/events.jsonl").read_text())["completed_samples"] == [{"error": "retained"}] +def test_diagnostic_settings_restore_absent_attribute(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.delattr(worker, "PERFORMANCE_FUTURE_TIMEOUT_S", raising=False) + with probe.diagnostic_settings(30): + assert current_workload() == (30, 200, 12, 90) + assert not hasattr(worker, "PERFORMANCE_FUTURE_TIMEOUT_S") + + +def test_diagnostic_records_current_observer_and_exact_historical_source(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(worker, "RESPONSE_CAPTURE_MODE", "success-status-header-snapshots", raising=False) + method = probe.measurement_method() + assert method["response_capture"] == "success-status-header-snapshots" + assert method["worker_source_sha256"] == digest(Path(worker.__file__)) + assert method["exact_historical_replay_commit"] == "482b599224920a97bb2fce4a740317efb89351ab" + assert not method["historical_byte_identity_claimed"] + + def test_full_cleanup_budget_is_strict() -> None: assert probe.admission(1, 272, 180)["reserve_s"] == 270 with pytest.raises(AssertionError, match="insufficient"): @@ -320,7 +347,7 @@ def test_negative_raw_latency_cannot_hide_in_positive_aggregates() -> None: probe.metrics(phase) -def test_shared_timed_code_and_gc_are_unmodified() -> None: +def test_serving_observer_timed_loop_and_gc_are_unmodified() -> None: baseline = subprocess.run( ["git", "show", "f06f414a:tools/benchmarks/cross_encoder_lora/worker.py"], # noqa: S607 cwd=ROOT, @@ -330,11 +357,44 @@ def test_shared_timed_code_and_gc_are_unmodified() -> None: ) if baseline.returncode: pytest.skip("timed-code invariant control requires its documented parent commit in git history") - names = {"performance", "Client", "unobserved"} + # The Client capture lifecycle is intentionally amended. Preserve the + # production observation boundary and actual timed loop, allowing only + # the named wait limit that gained scheduling headroom for the longer run. + names = {"performance", "unobserved"} + + class TimedCore(ast.NodeTransformer): + def visit_FunctionDef(self, node: ast.FunctionDef) -> ast.FunctionDef: + if node.name == "performance": + timed = next(value for value in node.body if isinstance(value, ast.Try)) + end = next( + index + for index, value in enumerate(timed.body) + if isinstance(value, ast.Assign) + and any(isinstance(target, ast.Name) and target.id == "elapsed" for target in value.targets) + ) + timed.body = timed.body[: end + 1] + timed.finalbody = [] + self.generic_visit(node) + return node + + def visit_Call(self, node: ast.Call) -> ast.Call: + if ( + isinstance(node.func, ast.Attribute) + and node.func.attr == "result" + and isinstance(node.func.value, ast.Name) + and node.func.value.id == "future" + ): + timeout = next(value for value in node.keywords if value.arg == "timeout") + assert (isinstance(timeout.value, ast.Constant) and timeout.value.value == 90) or ( + isinstance(timeout.value, ast.Name) and timeout.value.id == "PERFORMANCE_FUTURE_TIMEOUT_S" + ) + timeout.value = ast.Constant(value=90) + self.generic_visit(node) + return node def code(text: str) -> dict[str, str]: return { - node.name: ast.dump(node) + node.name: ast.dump(TimedCore().visit(node)) for node in ast.parse(text).body if isinstance(node, (ast.FunctionDef, ast.ClassDef)) and node.name in names } From a2932d81ef0894d3270a26350a2cd0703fa66a82 Mon Sep 17 00:00:00 2001 From: svonava Date: Fri, 16 Oct 2026 19:49:13 -0700 Subject: [PATCH 11/16] fix(benchmarks): bind worker waits to qualification limits --- tools/benchmarks/cross_encoder_lora/worker.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/tools/benchmarks/cross_encoder_lora/worker.py b/tools/benchmarks/cross_encoder_lora/worker.py index 15c0ca746..110fd52d3 100644 --- a/tools/benchmarks/cross_encoder_lora/worker.py +++ b/tools/benchmarks/cross_encoder_lora/worker.py @@ -51,6 +51,7 @@ MAX_TRIAL_SECONDS, MODEL, ORACLE_ATOL, + PERFORMANCE_FUTURE_TIMEOUT_S, QUERY, RESTORE_ATOL, SEQUENTIAL, @@ -421,7 +422,7 @@ def lane(index: int) -> None: clock["deadline"] = clock["start"] + (0.15 if smoke else STEADY_SECONDS) barrier.wait(timeout=30) for future in futures: - future.result(timeout=90) + future.result(timeout=PERFORMANCE_FUTURE_TIMEOUT_S) elapsed = time.perf_counter() - clock["start"] for row in [*warmups, *samples]: validate_response(row) @@ -906,10 +907,13 @@ def main() -> None: parser.add_argument("--output", type=Path, required=True) parser.add_argument("--arm", choices=("B", "N", "R"), required=True) parser.add_argument("--pair", type=int, default=0) - parser.add_argument("--deadline-s", type=float, default=180) + parser.add_argument("--deadline-s", type=float, default=MAX_TRIAL_SECONDS) parser.add_argument("--smoke", action="store_true") args = parser.parse_args() - require(0 < args.deadline_s <= MAX_TRIAL_SECONDS, "worker deadline must be in (0, 300]") + require( + 0 < args.deadline_s <= MAX_TRIAL_SECONDS, + f"worker deadline must be in (0, {MAX_TRIAL_SECONDS}]", + ) asyncio.run(run(args)) From 250bd9a91c8f465aafe69de94aee98aad6ac3298 Mon Sep 17 00:00:00 2001 From: svonava Date: Sat, 17 Oct 2026 09:08:22 -0700 Subject: [PATCH 12/16] test: isolate cross-encoder benchmark load generation --- tools/benchmarks/cross_encoder_lora/README.md | 69 ++- .../cross_encoder_lora/calibrate_gc.py | 195 +++++++++ tools/benchmarks/cross_encoder_lora/client.py | 299 +++++++++++++ .../cross_encoder_lora/isolation.py | 316 ++++++++++++++ .../benchmarks/cross_encoder_lora/loadgen.py | 217 ++++++++++ .../cross_encoder_lora/process_metrics.py | 406 ++++++++++++++++++ .../cross_encoder_lora/stability.py | 227 +--------- .../tests/test_client_lifecycle.py | 46 +- .../tests/test_isolation.py | 352 +++++++++++++++ .../cross_encoder_lora/tests/test_probe.py | 3 +- .../tests/test_process_metrics.py | 210 +++++++++ .../tests/test_stability.py | 77 ++-- tools/benchmarks/cross_encoder_lora/worker.py | 290 +++---------- 13 files changed, 2194 insertions(+), 513 deletions(-) create mode 100644 tools/benchmarks/cross_encoder_lora/calibrate_gc.py create mode 100644 tools/benchmarks/cross_encoder_lora/client.py create mode 100644 tools/benchmarks/cross_encoder_lora/isolation.py create mode 100644 tools/benchmarks/cross_encoder_lora/loadgen.py create mode 100644 tools/benchmarks/cross_encoder_lora/process_metrics.py create mode 100644 tools/benchmarks/cross_encoder_lora/tests/test_isolation.py create mode 100644 tools/benchmarks/cross_encoder_lora/tests/test_process_metrics.py diff --git a/tools/benchmarks/cross_encoder_lora/README.md b/tools/benchmarks/cross_encoder_lora/README.md index c46a0537c..87392d700 100644 --- a/tools/benchmarks/cross_encoder_lora/README.md +++ b/tools/benchmarks/cross_encoder_lora/README.md @@ -170,8 +170,9 @@ close address that benchmark-owned retention. Normal HTTPX response-stream and SDK namespace cycles remain, with automatic GC enabled. This finding does not identify the cause of earlier GPU timing variation. Snapshot capture adds timed metadata work; its CPU cost must be recorded, without subtracting it from the -measurements or claiming zero overhead. Client and server still share a process, -so the result includes their normal GC and GIL interactions. +measurements or claiming zero overhead. That version shared one client/server process and included their normal GC and +GIL interactions; the subsequent isolation amendment below preserves its failed +result rather than reclassifying it. Freeze the reviewed source and operator bounds before one complete amended 18-process run. A separate baseline-readiness allocation is not required. Every @@ -186,6 +187,68 @@ hash. A new invocation uses this checkout's snapshot observer and is **not** a byte-identical replay of the earlier diagnostic. Exact historical replay uses commit `482b599224920a97bb2fce4a740317efb89351ab` and its original protocol. +## SDK process isolation amendment + +The complete longer-window run at `a2932d81ef0894d3270a26350a2cd0703fa66a82` +failed five of twelve upper-confidence gates. Its point estimates did not +establish a serving slowdown, but they also did not satisfy the fixed precision +requirement. No source defect or physical cause was established. The original +records and criteria remain unchanged. + +Qualification now starts one fresh Python **exec** load generator per server +trial and reuses it across C1/C4/C16. It imports only the selected archive's SDK +and benchmark client/telemetry helpers, never Torch, PEFT or the serving model. +The server retains its original process, CPU allocation, affinity and model +state. The SDK, HTTP, serving work, normal GC and every fixed workload/gate +remain included. Separation removes direct client/server interpreter and GIL +coupling; it does not remove shared CPU competition or guarantee precise results. +The historical `stability` command retains its in-process diagnostic method. + +Client startup, warmup, boundary handshakes, process shutdown and evidence serialization +are outside the request timers. The same lane loop releases a barrier after +recording its start and drains every admitted request before the end timestamp. +The parent waits asynchronously so the server event loop continues. A single +stdin reader monitors parent lifetime; unexpected EOF ends the owned child. +Normal shutdown wakes and joins that reader, writes final terminal evidence, +then reports completion; the parent reaps the child before closing stdin. No new process group or detached session is +created. Cancellation, malformed protocol and child errors fail the trial and +trigger bounded termination/reaping. SIGKILL or immediate parent-death exit can +lose the interrupted phase; prior completed atomic phase files remain evidence. + +Each PID records preallocated numeric GC events and self-duration. The combined +server-plus-client estimated callback cost must be at most **0.1%** of each +measured interval, including a fixed 10-microsecond allowance per intersecting +callback. A bounded local calibration must be retained before hardware use; +it checks the allowance rather than changing it after outcomes. This estimate +cannot prove zero scheduling or GC perturbation. Overflow, missing event pairs, +clock contradictions, callback-list changes or the cost limit fail qualification. +Run the public local calibration once from the frozen checkout and retain its +full JSON, including failures and outliers: + +```console +mise exec -- uv run --no-sync python -m tools.benchmarks.cross_encoder_lora.calibrate_gc --output /tmp/ce-gc-calibration.json +``` + +It estimates local mean cost, not worst-case event cost or another host's cost. +Short CPU smoke applies integrity checks but does not qualify performance or +apply a meaningful wall-time cost budget. No automatic collection is disabled, +and no forced collection is added during timing. + +CPU/context-switch/cgroup snapshots bracket the actual shared monotonic interval; +they include boundary work and are not exact request-level or GPU-only timers. +Shared cgroup observations are labeled and must never be summed across PIDs. +Missing optional counters remain unavailable. Existing server correctness +histories are retained and counted; the pre-timing memory checkpoint keeps its +existing explicit GC and CUDA synchronization. No speculative history cleanup +or heap normalization is introduced. The child retains its request histories +until all phases end. The server ingests them only after the child is reaped. + +Phase payloads, observed IPC clocks, PID/source identities, ordered terminal +records, lengths and hashes must agree. Raw requests, wire attempts and all +existing numerical checks remain available. Child observations and phase files +live under each trial's `loadgen/` directory, with a separate process receipt and +server boundary snapshots. Keep those files in the full durable archive. + ## Evidence and CPU checks `summary.json` schema 2 retains trial outcomes, aggregate metrics, statistical @@ -214,7 +277,7 @@ Store the evidence directory on durable storage and verify the full archive hash after retrieval; stdout is unsuitable for transferring large artifacts. ```console -mise run test -- tools/benchmarks/cross_encoder_lora/tests/test_probe.py +mise run test -- tools/benchmarks/cross_encoder_lora/tests mise run typecheck -- tools/benchmarks/cross_encoder_lora ``` diff --git a/tools/benchmarks/cross_encoder_lora/calibrate_gc.py b/tools/benchmarks/cross_encoder_lora/calibrate_gc.py new file mode 100644 index 000000000..77770cc49 --- /dev/null +++ b/tools/benchmarks/cross_encoder_lora/calibrate_gc.py @@ -0,0 +1,195 @@ +"""Bounded model-free calibration of the fixed GC observer allowance, not serving performance.""" + +from __future__ import annotations + +import argparse +import gc +import math +import platform +import statistics +import sys +import time +from array import array +from pathlib import Path +from typing import Any + +from tools.benchmarks.cross_encoder_lora.evidence import atomic_json, digest +from tools.benchmarks.cross_encoder_lora.process_metrics import CALLBACK_TAIL_ALLOWANCE_NS, GCRecorder +from tools.benchmarks.cross_encoder_lora.protocol import percentile, require + +BLOCKS = 6 +COLLECTIONS = 10_000 +DIRECT_PAIRS = 500 +T95_DF5 = 2.0150483733330233 + + +def bound(values: list[float]) -> dict[str, Any]: + require( + len(values) == BLOCKS and all(math.isfinite(value) for value in values), "six finite calibration units required" + ) + mean = statistics.mean(values) + upper = mean + T95_DF5 * statistics.stdev(values) / math.sqrt(BLOCKS) + return { + "block_mean_residual_ns": values, + "mean_ns": mean, + "one_sided_95_upper_ns": upper, + "allowance_ns": CALLBACK_TAIL_ALLOWANCE_NS, + "passed": max(values) <= CALLBACK_TAIL_ALLOWANCE_NS and upper <= CALLBACK_TAIL_ALLOWANCE_NS, + } + + +def collect_arm(observed: bool, count: int, record: dict[str, Any] | None = None) -> dict[str, Any]: + if record is None: + record = {} + record.update(observed=observed, passed=False, completed=0) + durations = array("Q", [0]) * count + collected = array("Q", [0]) * count + recorder = GCRecorder(capacity=2 * count + 64) + thresholds, prior = gc.get_threshold(), tuple(gc.callbacks) + require(gc.isenabled(), "normal GC must remain enabled") + if observed: + recorder.start() + try: + for index in range(count): + started = time.perf_counter_ns() + collected[index] = gc.collect(0) + durations[index] = time.perf_counter_ns() - started + record["completed"] = index + 1 + finally: + try: + if observed: + recorder.close() + finally: + rows = [list(recorder.buffer[i * 6 : (i + 1) * 6]) for i in range(recorder.count)] + record.update( + duration_ns=list(durations), + collected=list(collected), + records=rows, + recorder_errors=recorder.lifecycle_errors, + invalid=recorder.invalid, + overflow=recorder.overflow, + ) + require(gc.isenabled() and gc.get_threshold() == thresholds, "GC configuration changed") + require( + len(gc.callbacks) == len(prior) and all(a is b for a, b in zip(gc.callbacks, prior, strict=True)), + "GC callback identities changed", + ) + if observed: + require( + not recorder.lifecycle_errors and not recorder.invalid and not recorder.overflow, + "invalid calibration observer", + ) + require(len(rows) == 2 * count, "unexpected extra/missing calibration GC callbacks") + require( + all(row[1] == index % 2 and row[2] == 0 for index, row in enumerate(rows)), "calibration GC pairing differs" + ) + record["passed"] = True + return record + + +def direct(generation: int, pairs: int) -> dict[str, Any]: + # This is a diagnostic direct call, not an installed GC callback. It covers + # all field paths while the real-dispatch pairs above exercise CPython GC. + recorder = GCRecorder(capacity=2 * pairs) + durations = array("Q", [0]) * (2 * pairs) + info = {"generation": generation, "collected": 101, "uncollectable": 3} + callback = recorder._record + for index in range(2 * pairs): + phase = "start" if index % 2 == 0 else "stop" + started = time.perf_counter_ns() + callback(phase, info) + durations[index] = time.perf_counter_ns() - started + self_ns = [recorder.buffer[index * 6 + 5] for index in range(2 * pairs)] + residual: list[float] = [elapsed - cost for elapsed, cost in zip(durations, self_ns, strict=True)] + return { + "generation": generation, + "info": info, + "external_ns": list(durations), + "self_ns": self_ns, + "residual_ns": residual, + "mean_residual_ns": statistics.mean(residual), + "p99_residual_ns": percentile(residual, 0.99), + "maximum_residual_ns": max(residual), + } + + +def run(result: dict[str, Any]) -> None: + prior, thresholds = tuple(gc.callbacks), gc.get_threshold() + require(gc.isenabled(), "normal GC must remain enabled") + blocks: list[dict[str, Any]] = [] + result["blocks"] = blocks + for index in range(BLOCKS): + order = (False, True) if index % 2 == 0 else (True, False) + block: dict[str, Any] = {"block": index, "order": list(order), "arms": {}, "direct": []} + blocks.append(block) + arms = block["arms"] + for observed in order: + arms[str(observed)] = {} + collect_arm(observed, COLLECTIONS, arms[str(observed)]) + baseline, observed = arms["False"], arms["True"] + self_mean = statistics.mean(row[5] for row in observed["records"]) + # Pair means are the independent units. The baseline cost, including + # the identical loop/timer/storage, is subtracted only in this diagnostic. + # Signed residuals are retained; none are clipped or selected away. + residual = (statistics.mean(observed["duration_ns"]) - statistics.mean(baseline["duration_ns"])) / 2 - self_mean + unsubtracted = [ + (elapsed - observed["records"][2 * j][5] - observed["records"][2 * j + 1][5]) / 2 + for j, elapsed in enumerate(observed["duration_ns"]) + ] + block.update( + { + "residual_ns_per_callback": residual, + "unsubtracted_residual_p99_ns": percentile(unsubtracted, 0.99), + "unsubtracted_residual_maximum_ns": max(unsubtracted), + "direct": [direct(generation, DIRECT_PAIRS) for generation in range(3)], + } + ) + gates = {"actual_gc_dispatch": bound([row["residual_ns_per_callback"] for row in blocks])} + for generation in range(3): + gates[f"direct_gen{generation}"] = bound([row["direct"][generation]["mean_residual_ns"] for row in blocks]) + require(gc.isenabled() and gc.get_threshold() == thresholds, "GC configuration changed") + require( + len(gc.callbacks) == len(prior) and all(a is b for a, b in zip(gc.callbacks, prior, strict=True)), + "GC callback identities changed", + ) + result.update( + { + "qualification": False, + "passed": all(value["passed"] for value in gates.values()), + "python": sys.version, + "platform": platform.platform(), + "gc_thresholds": list(thresholds), + "source_sha256": digest(Path(__file__)), + "observer_sha256": digest(Path(__file__).with_name("process_metrics.py")), + "blocks": blocks, + "gates": gates, + "interpretation": "Local mean-overhead estimator calibration only. The fixed10us allowance is not a worst-case event bound or a guarantee for another host. All samples/outliers retained; no benchmark subtraction or threshold adjustment.", + "runtime_budget": "Combined server+client estimated observer cost<=0.001 in every real measured phase, unchanged.", + } + ) + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--output", type=Path, required=True) + args = parser.parse_args() + require(not args.output.exists(), "calibration output must be new") + result: dict[str, Any] = { + "qualification": False, + "passed": False, + "source_sha256": digest(Path(__file__)), + "observer_sha256": digest(Path(__file__).with_name("process_metrics.py")), + } + try: + run(result) + except BaseException as error: + result["error"] = {"type": type(error).__name__, "message": str(error)} + raise + finally: + atomic_json(args.output, result) + print({"passed": result["passed"], "qualification": False, "output_sha256": digest(args.output)}, flush=True) + require(result["passed"], "fixed GC allowance calibration failed; retain evidence and investigate") + + +if __name__ == "__main__": + main() diff --git a/tools/benchmarks/cross_encoder_lora/client.py b/tools/benchmarks/cross_encoder_lora/client.py new file mode 100644 index 000000000..5891a723a --- /dev/null +++ b/tools/benchmarks/cross_encoder_lora/client.py @@ -0,0 +1,299 @@ +"""SDK-only request capture and load loop, shared by local and isolated drivers.""" + +from __future__ import annotations + +import math +import statistics +import threading +import time +from collections.abc import Callable +from concurrent.futures import ThreadPoolExecutor +from http import HTTPStatus +from typing import Any + +import httpx +from sie_sdk import SIEClient +from sie_sdk._msgpack import unpackb +from sie_sdk.client.errors import ServerError + +from tools.benchmarks.cross_encoder_lora.evidence import Evidence +from tools.benchmarks.cross_encoder_lora.protocol import ( + DOCUMENTS, + MODEL, + PERFORMANCE_FUTURE_TIMEOUT_S, + QUERY, + SEQUENTIAL, + STEADY_SECONDS, + WARMUP, + percentile, + require, +) + +MAX_LOADING_RETRIES = 10 +MAX_LOADING_DELAY = 5 +REQUEST_ITEMS = len(DOCUMENTS) +RESPONSE_CAPTURE_MODE = "success-status-header-snapshots" + + +class Client: + def __init__(self, url: str) -> None: + self.responses: list[dict[str, Any] | httpx.Response] = [] + http = httpx.Client(base_url=url, timeout=30, event_hooks={"response": [self.observe]}) + self._http = http + self.sdk = SIEClient(url, timeout_s=30, http_client=http) + + def observe(self, response: httpx.Response) -> None: + # Keep successful wire evidence without retaining its Response/Request/ + # stream graph for the whole phase. Failure bodies are decoded later. + self.responses.append(describe_response(response) if response.status_code == HTTPStatus.OK else response) + + def score(self, selection: str | None, label: str, *, cold: bool = False, timed: bool = False) -> dict[str, Any]: + started = time.perf_counter_ns() + wall_started = time.time_ns() + first = len(self.responses) + # score() currently does not retry LORA_LOADING. These are explicitly + # fresh SDK calls for this pre-execution status, never transport retries. + for attempt in range(MAX_LOADING_RETRIES + 1): + try: + result = self.sdk.score( + MODEL, + {"text": QUERY["text"]}, + [{"id": f"{label}:{index}", "text": text} for index, text in enumerate(DOCUMENTS)], + options={"lora": selection}, + wait_for_capacity=False, + max_oom_retries=0, + provision_timeout_s=30, + ) + break + except ServerError as error: + if ( + not cold + or error.status_code != HTTPStatus.SERVICE_UNAVAILABLE + or error.code != "LORA_LOADING" + or attempt == MAX_LOADING_RETRIES + ): + raise + delay = error.retry_after if error.retry_after is not None else 0.2 + require(0 <= delay <= MAX_LOADING_DELAY, "unexpected loading retry delay") + time.sleep(delay) + elapsed = (time.perf_counter_ns() - started) / 1e6 + row = { + "label": label, + "selection": selection, + "latency_ms": elapsed, + "start_ns": wall_started, + "end_ns": time.time_ns(), + "result": result, + "responses": self.responses[first:], + } + if not timed: + validate_response(row, allow_loading=cold) + return row + + def close(self) -> None: + try: + self.sdk.close() + finally: + # Callers serialize their evidence before closing. Preserve unrelated + # hooks, and break only the observer/history this benchmark owns. + hooks = self._http.event_hooks["response"] + hooks[:] = [ + hook + for hook in hooks + if not ( + getattr(hook, "__self__", None) is self and getattr(hook, "__func__", None) is type(self).observe + ) + ] + self.responses.clear() + + +def describe_response(response: httpx.Response | dict[str, Any]) -> dict[str, Any]: + if not isinstance(response, httpx.Response): + return response + row: dict[str, Any] = { + "status": response.status_code, + "timing_headers": { + name: response.headers[name] + for name in ( + "x-queue-time", + "x-tokenization-time", + "x-inference-time", + "x-postprocessing-time", + "x-total-time", + ) + if name in response.headers + }, + } + if response.status_code >= HTTPStatus.BAD_REQUEST: + row["content_type"] = response.headers.get("content-type") + row["retry_after"] = response.headers.get("retry-after") + try: + response.read() + except (httpx.StreamError, httpx.HTTPError) as error: + # A partially read error stream can already have failed in the SDK. + # Preserve that original failure and its wire metadata; diagnostic + # re-reading must not replace it or discard earlier attempts. + row.update(body_unavailable=True, body_read_error=type(error).__name__, code=None) + return row + row["body_hex"] = response.content.hex() + try: + body = ( + unpackb(response.content, numeric_arrays=False) + if "msgpack" in response.headers.get("content-type", "") + else response.json() + ) + except ValueError as error: + row["body_decode_error"] = type(error).__name__ + body = None + error = body.get("error", body.get("detail", body)) if isinstance(body, dict) else None + row["code"] = error.get("code") if isinstance(error, dict) else None + return row + + +def serialize_responses(row: dict[str, Any]) -> None: + row["responses"] = [ + describe_response(value) if isinstance(value, httpx.Response) else value for value in row["responses"] + ] + + +def validate_response(row: dict[str, Any], *, allow_loading: bool = False) -> None: + serialize_responses(row) + statuses = row["responses"] + require(statuses and statuses[-1]["status"] == HTTPStatus.OK, "missing successful HTTP response") + if allow_loading: + require( + all( + value["status"] == HTTPStatus.SERVICE_UNAVAILABLE and value.get("code") == "LORA_LOADING" + for value in statuses[:-1] + ), + "unexpected retry during cold adapter request", + ) + else: + require(len(statuses) == 1, "hidden SDK retry disqualifies this request") + scores = row["result"]["scores"] + expected_ids = [f"{row['label']}:{i}" for i in range(REQUEST_ITEMS)] + by_id = {item["item_id"]: item["score"] for item in scores} + require(len(scores) == REQUEST_ITEMS and set(by_id) == set(expected_ids), "score result lost or changed item IDs") + require([item["rank"] for item in scores] == list(range(REQUEST_ITEMS)), "score ranks changed") + values = [by_id[key] for key in expected_ids] + row["scores"] = values + require(all(math.isfinite(value) for value in values), "nonfinite SDK score") + require([item["score"] for item in scores] == sorted(values, reverse=True), "score ranks are unordered") + + +def performance( + url: str, + concurrency: int, + evidence: Evidence, + *, + smoke: bool, + warmup: int = WARMUP, + sequential: int = SEQUENTIAL, + steady_seconds: float = STEADY_SECONDS, + future_timeout: float = PERFORMANCE_FUTURE_TIMEOUT_S, + before_measure: Callable[[], None] | None = None, + after_measure: Callable[[dict[str, float]], None] | None = None, +) -> dict[str, Any]: + clients: list[Client] = [] + samples: list[dict[str, Any]] = [] + warmups: list[dict[str, Any]] = [] + lock = threading.Lock() + clock: dict[str, float] = {} + measurement_notified = False + primary_error: BaseException | None = None + + def notify_measurement_end() -> None: + nonlocal measurement_notified + if "start" in clock and not measurement_notified: + clock.setdefault("end", time.perf_counter()) + measurement_notified = True + if after_measure is not None: + after_measure(dict(clock)) + + try: + for _ in range(concurrency): + clients.append(Client(url)) + for i in range(2 if smoke else warmup): + warmups.append(clients[0].score(None, f"warm-C{concurrency}-{i}", timed=True)) + barrier = threading.Barrier(concurrency + 1) + + def lane(index: int) -> None: + barrier.wait(timeout=30) + counter = 0 + while ( + counter < (4 if smoke else sequential) if concurrency == 1 else time.perf_counter() < clock["deadline"] + ): + row = clients[index].score(None, f"C{concurrency}-lane{index}-{counter}", timed=True) + with lock: + samples.append(row) + counter += 1 + + with ThreadPoolExecutor(max_workers=concurrency) as executor: + futures = [executor.submit(lane, i) for i in range(concurrency)] + try: + if before_measure is not None: + before_measure() + except BaseException: + barrier.abort() + raise + clock["start"] = time.perf_counter() + clock["deadline"] = clock["start"] + (0.15 if smoke else steady_seconds) + barrier.wait(timeout=30) + for future in futures: + future.result(timeout=future_timeout) + clock["end"] = time.perf_counter() + elapsed = clock["end"] - clock["start"] + notify_measurement_end() + for row in [*warmups, *samples]: + validate_response(row) + latencies = [row["latency_ms"] for row in samples] + return { + "concurrency": concurrency, + "count": len(samples), + "elapsed_s": elapsed, + "mean_ms": statistics.mean(latencies), + "p95_ms": percentile(latencies, 0.95), + "rps": len(samples) / elapsed, + "samples": samples, + "measurement_clock": dict(clock), + } + except BaseException as error: + primary_error = error + raise + finally: + # Finalize each owned responsibility independently. A broken telemetry + # handshake must not replace an SDK failure or erase its wire attempts. + finalization_errors: list[BaseException] = [] + + def finalize(action: Callable[[], None]) -> None: + try: + action() + except BaseException as error: # noqa: BLE001 -- finalize all owned resources, then re-raise + finalization_errors.append(error) + + finalize(notify_measurement_end) + for row in [*warmups, *samples]: + finalize(lambda row=row: serialize_responses(row)) + finalize( + lambda: evidence.emit( + "performance_attempt", + concurrency=concurrency, + completed_samples=samples, + warmups=warmups, + wire_responses=[describe_response(response) for client in clients for response in client.responses], + primary_error=None + if primary_error is None + else {"type": type(primary_error).__name__, "message": str(primary_error)}, + finalization_errors=[ + {"type": type(error).__name__, "message": str(error)} for error in finalization_errors + ], + ) + ) + for client in clients: + finalize(client.close) + if finalization_errors: + if primary_error is not None: + for error in finalization_errors: + primary_error.add_note(f"Secondary benchmark finalization failure: {type(error).__name__}: {error}") + else: + raise finalization_errors[0] diff --git a/tools/benchmarks/cross_encoder_lora/isolation.py b/tools/benchmarks/cross_encoder_lora/isolation.py new file mode 100644 index 000000000..3524b7270 --- /dev/null +++ b/tools/benchmarks/cross_encoder_lora/isolation.py @@ -0,0 +1,316 @@ +"""Async parent ownership of one exec-only SDK child; no compute provisioning.""" + +from __future__ import annotations + +import asyncio +import json +import math +import os +import sys +import time +import uuid +from pathlib import Path +from types import TracebackType +from typing import Any, Self + +from tools.benchmarks.cross_encoder_lora.evidence import Evidence, atomic_json, digest +from tools.benchmarks.cross_encoder_lora.loadgen import CONTROL_LIMIT, CONTROL_TIMEOUT_S, PHASES +from tools.benchmarks.cross_encoder_lora.process_metrics import MAX_CALLBACK_FRACTION, GCRecorder, bracket, snapshot +from tools.benchmarks.cross_encoder_lora.protocol import PERFORMANCE_FUTURE_TIMEOUT_S, require + + +class IsolatedLoadgen: + """Leave uvicorn running while awaiting a bounded, same-group SDK child.""" + + def __init__(self, url: str, source: Path, evidence: Evidence, *, smoke: bool) -> None: + self.url, self.source, self.evidence, self.smoke = url, source.resolve(), evidence, smoke + self.token = uuid.uuid4().hex + self.process: asyncio.subprocess.Process | None = None + self.log: Any = None + self.records: list[dict[str, Any]] = [] + self.observations: dict[int, dict[str, Any]] = {} + self.receipt: dict[str, Any] = {"passed": False, "reaped": False, "phases": self.records} + + async def _send(self, command: str, concurrency: int | None = None) -> None: + assert self.process is not None + assert self.process.stdin is not None + value = {"token": self.token, "command": command, "concurrency": concurrency} + self.process.stdin.write((json.dumps(value) + "\n").encode()) + await asyncio.wait_for(self.process.stdin.drain(), timeout=5) + + async def _read( + self, event: str, concurrency: int | None = None, *, wait_s: float = CONTROL_TIMEOUT_S + ) -> dict[str, Any]: + assert self.process is not None + assert self.process.stdout is not None + line = await asyncio.wait_for(self.process.stdout.readline(), timeout=wait_s) + require( + bool(line) and len(line) <= CONTROL_LIMIT and line.endswith(b"\n"), "missing or oversized loadgen message" + ) + value = json.loads(line) + require(isinstance(value, dict), "invalid loadgen control object") + require(value.get("token") == self.token and value.get("event") == event, "unexpected loadgen event") + require(value.get("concurrency") == concurrency, "loadgen phase identity differs") + return value + + async def __aenter__(self) -> Self: + command = [ + sys.executable, + "-m", + "tools.benchmarks.cross_encoder_lora.loadgen", + "--source", + str(self.source), + "--output", + str(self.evidence.directory / "loadgen"), + "--url", + self.url, + "--token", + self.token, + ] + if self.smoke: + command.append("--smoke") + self.log = (self.evidence.directory / "loadgen.log").open("wb") + try: + self.process = await asyncio.create_subprocess_exec( + *command, + stdin=asyncio.subprocess.PIPE, + stdout=asyncio.subprocess.PIPE, + stderr=self.log, + close_fds=True, + limit=CONTROL_LIMIT, + ) + hello = await self._read("hello") + identity = hello["identity"] + require(identity["pid"] == self.process.pid and identity["pid"] != os.getpid(), "loadgen PID differs") + require( + identity["ppid"] == os.getpid() and identity["process_group"] == os.getpgrp(), + "loadgen ownership/group differs", + ) + require( + identity["source"] == str(self.source) and identity["forbidden_imports_present"] is False, + "loadgen source/import preflight failed", + ) + sdk_path = Path(identity["sdk_path"]) + require( + sdk_path.is_relative_to(self.source / "packages/sie_sdk/src"), "loadgen SDK source escaped its archive" + ) + require(digest(sdk_path) == identity["sdk_sha256"], "loadgen SDK implementation hash differs") + clock = time.get_clock_info("perf_counter") + require( + identity["clock"]["implementation"] == clock.implementation, + "client/server monotonic clock implementations differ", + ) + self.receipt["identity"] = identity + self.evidence.emit("loadgen_started", **identity) + return self + except BaseException as error: + try: + await self._cleanup(False) + except BaseException as cleanup_error: # noqa: BLE001 -- retain primary startup failure + error.add_note(f"Loadgen startup cleanup failed: {type(cleanup_error).__name__}: {cleanup_error}") + raise + + async def phase(self, concurrency: int) -> dict[str, Any]: + require( + len(self.records) < len(PHASES) and concurrency == PHASES[len(self.records)], + "duplicate or reordered measured phase", + ) + monitor = GCRecorder() + observation: dict[str, Any] = {"server_pid": os.getpid(), "client_identity": self.receipt["identity"]} + try: + await self._send("phase", concurrency) + await self._read("ready", concurrency) + monitor.start() + observation["before"] = snapshot() + await self._send("go", concurrency) + measured = await self._read("measured", concurrency, wait_s=PERFORMANCE_FUTURE_TIMEOUT_S) + clock = measured["clock"] + require( + set(clock) == {"start", "deadline", "end"} + and all(type(v) in (int, float) and math.isfinite(v) for v in clock.values()), + "invalid loadgen measurement clock", + ) + require( + 0 < clock["start"] < clock["end"] and clock["deadline"] > clock["start"], "loadgen clock order differs" + ) + observation["clock"] = clock + observation["after"] = snapshot() + monitor.close() + observation["gc"] = monitor.report(clock["start"], clock["end"]) + observation["resources"] = bracket(observation["before"], observation["after"], clock) + await self._send("observed", concurrency) + complete = await self._read("phase_complete", concurrency) + record = complete["record"] + require( + type(record["concurrency"]) is int + and record["concurrency"] == concurrency + and record["path"] == f"phase-C{concurrency}.json", + "loadgen record identity differs", + ) + path = self.evidence.directory / "loadgen" / record["path"] + require( + path.is_file() and path.stat().st_size == record["bytes"] and digest(path) == record["sha256"], + "loadgen phase record is incomplete or corrupt", + ) + self.records.append(record) + self.observations[concurrency] = observation + self.evidence.emit( + "loadgen_phase_complete", concurrency=concurrency, record=record, server_observation=observation + ) + return record + finally: + monitor.close() + primary_error = sys.exception() + try: + atomic_json(self.evidence.directory / f"server-C{concurrency}-observations.json", observation) + except BaseException as write_error: + if primary_error is None: + raise + primary_error.add_note( + f"Secondary server observation write failure: {type(write_error).__name__}: {write_error}" + ) + + async def _cleanup(self, successful: bool) -> None: + try: + if self.process is not None: + try: + if successful: + require(len(self.records) == len(PHASES), "loadgen did not complete every phase") + await self._send("finish") + finished = await self._read("finished") + await asyncio.wait_for(self.process.wait(), timeout=10) + require(self.process.returncode == 0, "loadgen failed after final evidence") + terminal = self.evidence.directory / "loadgen/terminal.json" + require(digest(terminal) == finished["terminal_sha256"], "loadgen terminal record differs") + self._validate_terminal(json.loads(terminal.read_text())) + self.receipt["terminal_sha256"] = finished["terminal_sha256"] + self.receipt["passed"] = True + finally: + if self.process.returncode is None: + try: + self.process.terminate() + except ProcessLookupError: + self.receipt["termination_raced_exit"] = True + try: + await asyncio.wait_for(self.process.wait(), timeout=3) + except TimeoutError: + try: + self.process.kill() + except ProcessLookupError: + self.receipt["kill_raced_exit"] = True + await asyncio.wait_for(self.process.wait(), timeout=2) + self.receipt["returncode"] = self.process.returncode + self.receipt["reaped"] = self.process.returncode is not None + if self.process.stdin is not None: + self.process.stdin.close() + else: + self.receipt["reaped"] = True + finally: + if self.log is not None: + self.log.close() + atomic_json(self.evidence.directory / "loadgen-process.json", self.receipt) + + async def __aexit__( + self, error_type: type[BaseException] | None, error: BaseException | None, traceback: TracebackType | None + ) -> None: + task = asyncio.create_task(self._cleanup(error_type is None)) + try: + await asyncio.shield(task) + except asyncio.CancelledError as cancelled: + # Cancellation may arrive while the successful finish handshake is + # waiting for a silent child. Cancel that wait, join its bounded + # teardown, then explicitly reap even if it never started running. + task.cancel() + try: + await asyncio.shield(task) + except asyncio.CancelledError: + pass + except BaseException as cleanup_error: # noqa: BLE001 -- cancellation remains primary + self.receipt["cancelled_cleanup_error"] = { + "type": type(cleanup_error).__name__, + "message": str(cleanup_error), + } + try: + await asyncio.shield(asyncio.create_task(self._cleanup(False))) + except BaseException as cleanup_error: # noqa: BLE001 -- preserve cancellation after bounded reaping + cancelled.add_note( + f"Secondary cancelled cleanup failure: {type(cleanup_error).__name__}: {cleanup_error}" + ) + raise + except BaseException as cleanup_error: + if error is None: + raise + error.add_note(f"Secondary loadgen cleanup failure: {type(cleanup_error).__name__}: {cleanup_error}") + self.receipt["cleanup_error"] = {"type": type(cleanup_error).__name__, "message": str(cleanup_error)} + try: + atomic_json(self.evidence.directory / "loadgen-process.json", self.receipt) + except BaseException as write_error: # noqa: BLE001 -- original phase failure remains primary + error.add_note(f"Secondary cleanup receipt write failure: {type(write_error).__name__}: {write_error}") + + def _validate_terminal(self, terminal: dict[str, Any]) -> None: + require(terminal.get("passed") is True and "error" not in terminal, "loadgen terminal did not pass") + require(terminal.get("identity") == self.receipt["identity"], "loadgen terminal identity differs") + require(terminal.get("smoke") is self.smoke, "loadgen terminal mode differs") + require(terminal.get("phases") == self.records, "loadgen terminal phase records differ") + require(tuple(record["concurrency"] for record in self.records) == PHASES, "loadgen terminal schedule differs") + require( + type(terminal.get("finished_ns")) is int and terminal["finished_ns"] > 0, + "loadgen terminal clock is missing", + ) + + def _validate_phase(self, record: dict[str, Any], value: dict[str, Any]) -> dict[str, Any]: + concurrency = record["concurrency"] + server = self.observations[concurrency] + identity = self.receipt["identity"] + require( + type(value.get("concurrency")) is int and value["concurrency"] == concurrency, + "phase payload concurrency differs", + ) + require( + value.get("identity") == identity and value.get("smoke") is self.smoke, "phase payload provenance differs" + ) + require(value.get("measurement_clock") == server["clock"], "phase payload differs from observed IPC clock") + client = value["client_observation"] + require(client.get("clock") == server["clock"], "client observation clock differs") + require( + all(client[key]["core"]["pid"] == identity["pid"] for key in ("before", "after")), + "phase observation PID differs", + ) + require(client["gc"]["pid"] == identity["pid"], "client GC observation PID differs") + require(server["gc"]["pid"] == os.getpid(), "server GC observation PID differs") + for observation in (client, server): + gc_report = observation["gc"] + require( + gc_report["window_start_ns"] == round(server["clock"]["start"] * 1e9) + and gc_report["window_end_ns"] == round(server["clock"]["end"] * 1e9), + "GC observation window differs", + ) + require(gc_report["integrity_passed"] is True, "GC observer integrity gate failed") + cost = gc_report["estimated_observer_fraction"] + require(type(cost) in (int, float) and math.isfinite(cost) and cost >= 0, "invalid observer cost") + combined = server["gc"]["estimated_observer_fraction"] + client["gc"]["estimated_observer_fraction"] + # Tiny CPU smoke has no performance qualification. It still must prove + # evidence integrity; the unchanged prospective cost gate is mandatory + # for every real measurement window on the qualifying allocation. + if not self.smoke: + require(combined <= MAX_CALLBACK_FRACTION, "combined observer cost gate failed") + value["server_observation"] = server + value["combined_estimated_observer_fraction"] = combined + value["observer_cost_gate_applied"] = not self.smoke + return value + + def results(self) -> dict[str, dict[str, Any]]: + require( + self.receipt["passed"] and self.receipt["reaped"], + "loadgen must finish and be reaped before ingesting raw rows", + ) + values = {} + for record in self.records: + path = self.evidence.directory / "loadgen" / record["path"] + require( + path.stat().st_size == record["bytes"] and digest(path) == record["sha256"], + "completed loadgen evidence changed", + ) + value = self._validate_phase(record, json.loads(path.read_text())) + values[str(record["concurrency"])] = value + return values diff --git a/tools/benchmarks/cross_encoder_lora/loadgen.py b/tools/benchmarks/cross_encoder_lora/loadgen.py new file mode 100644 index 000000000..eb1fdae7a --- /dev/null +++ b/tools/benchmarks/cross_encoder_lora/loadgen.py @@ -0,0 +1,217 @@ +"""One SDK-only exec child, reused across a trial's three measured phases.""" + +from __future__ import annotations + +import argparse +import inspect +import json +import os +import queue +import select +import signal +import sys +import threading +import time +from pathlib import Path +from typing import Any + +from sie_sdk import SIEClient + +from tools.benchmarks.cross_encoder_lora.client import performance +from tools.benchmarks.cross_encoder_lora.evidence import Evidence, atomic_json, digest +from tools.benchmarks.cross_encoder_lora.process_metrics import GCRecorder, bracket, snapshot +from tools.benchmarks.cross_encoder_lora.protocol import require + +CONTROL_LIMIT = 65_536 +CONTROL_TIMEOUT_S = 30 +PHASES = (1, 4, 16) +FORBIDDEN_IMPORTS = ("torch", "transformers", "peft", "sie_server") + + +def require_sdk_only() -> None: + require( + not any(name.split(".", 1)[0] in FORBIDDEN_IMPORTS for name in sys.modules), + "load generator imported serving or CUDA model libraries", + ) + + +class Control: + """One stdin reader; loss of its sole parent writer ends this owned child.""" + + def __init__(self, token: str) -> None: + self.token = token + self.messages: queue.Queue[dict[str, Any]] = queue.Queue(maxsize=8) + self.normal_exit = False + self.stop_read, self.stop_write = os.pipe() + self.reader = threading.Thread(target=self._read, name="loadgen-parent-pipe", daemon=True) + self.reader.start() + + def _read(self) -> None: + pending = bytearray() + try: + while True: + readable, _, _ = select.select([sys.stdin.fileno(), self.stop_read], [], []) + if self.stop_read in readable: + return + data = os.read(sys.stdin.fileno(), 4096) + if not data: + if not self.normal_exit: + # A hard-killed parent cannot finalize current samples. + # Previously completed atomic files remain the evidence. + os.kill(os.getpid(), signal.SIGTERM) + return + pending.extend(data) + while b"\n" in pending: + end = pending.index(b"\n") + 1 + line = bytes(pending[:end]) + del pending[:end] + try: + require(len(line) <= CONTROL_LIMIT, "invalid control line length") + value = json.loads(line) + require(isinstance(value, dict), "invalid control object") + except (ValueError, AssertionError) as error: + value = {"reader_error": type(error).__name__} + # Malformed input does not disable the sole EOF monitor. + # A flooded queue fails closed without blocking this reader. + self.messages.put_nowait(value) + require(len(pending) <= CONTROL_LIMIT, "unterminated control line is too large") + except BaseException: # noqa: BLE001 -- losing the lifetime monitor must terminate this owned child + if not self.normal_exit: + os.kill(os.getpid(), signal.SIGTERM) + + def close(self) -> None: + # Raw-fd reads avoid a daemon holding Python's buffered stdin lock at + # interpreter shutdown. Wake and join it explicitly before normal exit. + self.normal_exit = True + os.write(self.stop_write, b"x") + self.reader.join(timeout=2) + require(not self.reader.is_alive(), "parent-pipe monitor failed to stop") + os.close(self.stop_read) + os.close(self.stop_write) + + def receive(self, command: str, concurrency: int | None = None) -> dict[str, Any]: + value = self.messages.get(timeout=CONTROL_TIMEOUT_S) + require("reader_error" not in value, "control reader failed") + require(value.get("token") == self.token and value.get("command") == command, "unexpected parent command") + require(value.get("concurrency") == concurrency, "parent phase identity differs") + return value + + def send(self, event: str, concurrency: int | None = None, **fields: Any) -> None: + value = {"token": self.token, "event": event, "concurrency": concurrency, **fields} + line = json.dumps(value, allow_nan=False) + require(len(line.encode()) < CONTROL_LIMIT, "control output exceeds bounded protocol") + print(line, flush=True) + + +def identity(source: Path) -> dict[str, Any]: + require_sdk_only() + implementation = Path(inspect.getfile(SIEClient)).resolve() + expected = source.resolve() / "packages/sie_sdk/src" + require(implementation.is_relative_to(expected), "load generator SDK did not come from the selected source archive") + clock = time.get_clock_info("perf_counter") + require(clock.monotonic and not clock.adjustable, "load generator requires a monotonic performance clock") + return { + "pid": os.getpid(), + "ppid": os.getppid(), + "process_group": os.getpgrp(), + "sdk_path": str(implementation), + "sdk_sha256": digest(implementation), + "python": sys.executable, + "source": str(source.resolve()), + "clock": {name: getattr(clock, name) for name in ("implementation", "resolution", "monotonic", "adjustable")}, + "forbidden_imports_present": False, + } + + +def run(args: argparse.Namespace) -> None: + evidence = Evidence(args.output.resolve()) + control = Control(args.token) + terminal: dict[str, Any] = {"passed": False, "phases": [], "identity": identity(args.source), "smoke": args.smoke} + retained_results = [] + control.send("hello", identity=terminal["identity"]) + try: + for concurrency in PHASES: + control.receive("phase", concurrency) + require_sdk_only() + monitor = GCRecorder() + monitor.start() + observations: dict[str, Any] = {} + + def before_measure() -> None: + observations["before"] = snapshot() + control.send("ready", concurrency) + control.receive("go", concurrency) + + def after_measure(clock: dict[str, float]) -> None: + observations["clock"] = clock + observations["after"] = snapshot() + monitor.close() + control.send("measured", concurrency, clock=clock) + control.receive("observed", concurrency) + + try: + result = performance( + args.url, + concurrency, + evidence, + smoke=args.smoke, + before_measure=before_measure, + after_measure=after_measure, + ) + finally: + monitor.close() + if "clock" in observations: + observations["gc"] = monitor.report(observations["clock"]["start"], observations["clock"]["end"]) + atomic_json(evidence.directory / f"phase-C{concurrency}-observations.json", observations) + require(observations["gc"]["integrity_passed"], "client GC observation integrity failed") + if not args.smoke: + require( + observations["gc"]["cost_budget_passed"], "client GC observation exceeded its prospective cost gate" + ) + observations["resources"] = bracket( + observations["before"], observations["after"], result["measurement_clock"] + ) + result["client_observation"] = observations + result["identity"] = terminal["identity"] + result["smoke"] = args.smoke + require_sdk_only() + path = evidence.directory / f"phase-C{concurrency}.json" + atomic_json(path, result) + record = { + "concurrency": concurrency, + "path": path.name, + "sha256": digest(path), + "bytes": path.stat().st_size, + } + terminal["phases"].append(record) + atomic_json(evidence.directory / "terminal.json", terminal) + # Preserve the historical request-history lifetime, but only in the + # load generator's heap. The serving process never ingests these + # rows until every timed phase is complete. + retained_results.append(result) + control.send("phase_complete", concurrency, record=record) + control.receive("finish") + terminal["passed"] = True + except BaseException as error: + terminal["error"] = {"type": type(error).__name__, "message": str(error)} + raise + finally: + control.close() + terminal["finished_ns"] = time.time_ns() + atomic_json(evidence.directory / "terminal.json", terminal) + if terminal["passed"]: + control.send("finished", terminal_sha256=digest(evidence.directory / "terminal.json")) + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--source", type=Path, required=True) + parser.add_argument("--output", type=Path, required=True) + parser.add_argument("--url", required=True) + parser.add_argument("--token", required=True) + parser.add_argument("--smoke", action="store_true") + run(parser.parse_args()) + + +if __name__ == "__main__": + main() diff --git a/tools/benchmarks/cross_encoder_lora/process_metrics.py b/tools/benchmarks/cross_encoder_lora/process_metrics.py new file mode 100644 index 000000000..2cde82443 --- /dev/null +++ b/tools/benchmarks/cross_encoder_lora/process_metrics.py @@ -0,0 +1,406 @@ +"""Process-local observations; filesystem reads occur only outside timed work.""" + +from __future__ import annotations + +import gc +import math +import os +import resource +import time +from array import array +from pathlib import Path +from typing import Any + +from tools.benchmarks.cross_encoder_lora.protocol import require + +GC_FIELDS = ("collections", "collected", "uncollectable") + +RUSAGE_FIELDS = ("ru_utime", "ru_stime", "ru_minflt", "ru_majflt", "ru_nvcsw", "ru_nivcsw") + +OPTIONAL_FILES = ("cpu.stat", "cpu.stat.local", "cpu.pressure", "cpu.max", "cpu.weight", "cpuset.cpus.effective") + +COUNTER_FILES = ("cpu.stat", "cpu.stat.local") + +CONTROL_FILES = ("cpu.max", "cpu.weight", "cpuset.cpus.effective") + +GC_GENERATIONS = 3 + +PSI_FIELDS = ("avg10", "avg60", "avg300", "total") + +MAX_PERCENT = 100 + +MAX_CPU_WEIGHT = 10000 + + +def unified_cgroup(proc: Path = Path("/proc/self")) -> dict[str, Any]: + """Resolve the process's own namespace; never fall back to another cgroup.""" + try: + rows = (proc / "cgroup").read_text().splitlines() + membership = [row[3:] for row in rows if row.startswith("0::")] + require(len(membership) == 1, "unified cgroup membership is unavailable or ambiguous") + group = Path(membership[0]) + require(group.is_absolute() and ".." not in group.parts, "invalid cgroup membership") + matches = [] + for row in (proc / "mountinfo").read_text().splitlines(): + left, right = row.split(" - ", 1) + if right.split()[0] != "cgroup2": + continue + fields = left.split() + root = Path(fields[3].replace("\\040", " ").replace("\\134", "\\")) + mount = Path(fields[4].replace("\\040", " ").replace("\\134", "\\")) + if group.is_relative_to(root): + matches.append(mount / group.relative_to(root)) + require(len(matches) == 1, "cgroup mount cannot be resolved uniquely") + path = matches[0] + info = path.stat() + return {"path": str(path), "device": info.st_dev, "inode": info.st_ino, "error": None} + except (OSError, ValueError, IndexError, AssertionError) as error: + return {"path": None, "error": f"{type(error).__name__}: {error}"} + + +def flat_counters(text: str) -> dict[str, int]: + result = {} + for row in text.splitlines(): + key, value = row.split() + require(key not in result, "duplicate telemetry counter") + result[key] = int(value) + require(result[key] >= 0, "negative telemetry counter") + require(bool(result), "empty exposed telemetry counter file") + return result + + +def pressure(text: str) -> dict[str, dict[str, float | int]]: + result = {} + for row in text.splitlines(): + name, *parts = row.split() + require(name in {"some", "full"} and name not in result, "invalid PSI row") + values = dict(part.split("=", 1) for part in parts) + require(len(parts) == len(PSI_FIELDS) and set(values) == set(PSI_FIELDS), "invalid PSI fields") + parsed = {key: int(value) if key == "total" else float(value) for key, value in values.items()} + require(all(math.isfinite(value) and value >= 0 for value in parsed.values()), "invalid PSI value") + require(all(parsed[key] <= MAX_PERCENT for key in PSI_FIELDS[:-1]), "invalid PSI percentage") + result[name] = parsed + require("some" in result, "missing exposed PSI some row") + return result + + +def validate_core(core: dict[str, Any]) -> None: + require( + all(isinstance(core[key], int) and core[key] > 0 for key in ("wall_ns", "monotonic_ns")) + and core["gc_enabled"] is True, + "invalid required clock/GC state", + ) + require( + set(core["rusage"]) == set(RUSAGE_FIELDS) + and all(math.isfinite(value) and value >= 0 for value in core["rusage"].values()), + "invalid required rusage", + ) + for key in ("gc_thresholds", "gc_count"): + require( + len(core[key]) == GC_GENERATIONS and all(type(value) is int and value >= 0 for value in core[key]), + "invalid required GC counts/thresholds", + ) + require(core["gc_thresholds"][0] > 0, "automatic GC is disabled by its threshold") + require(len(core["gc_stats"]) == GC_GENERATIONS, "missing required GC generations") + require( + all( + set(row) == set(GC_FIELDS) and all(type(row[key]) is int and row[key] >= 0 for key in GC_FIELDS) + for row in core["gc_stats"] + ), + "invalid required GC statistics", + ) + + +def control_value(name: str, text: str) -> str: + if name == "cpu.max": + quota, period = text.split() + require((quota == "max" or int(quota) > 0) and int(period) > 0, "invalid exposed cpu.max") + elif name == "cpu.weight": + require(1 <= int(text) <= MAX_CPU_WEIGHT, "invalid exposed cpu.weight") + else: + for item in text.split(","): + bounds = [int(value) for value in item.split("-")] + require(len(bounds) in {1, 2} and bounds[0] >= 0 and bounds[-1] >= bounds[0], "invalid exposed cpuset") + return text + + +def snapshot() -> dict[str, Any]: + """Only called outside performance(), including all filesystem reads.""" + started = time.monotonic_ns() + usage = resource.getrusage(resource.RUSAGE_SELF) + core = { + "wall_ns": time.time_ns(), + "monotonic_ns": started, + "perf_counter_ns": time.perf_counter_ns(), + "pid": os.getpid(), + "ppid": os.getppid(), + "rusage": {key: getattr(usage, key) for key in RUSAGE_FIELDS}, + "gc_enabled": gc.isenabled(), + "gc_thresholds": list(gc.get_threshold()), + "gc_count": list(gc.get_count()), + "gc_stats": gc.get_stats(), + } + validate_core(core) + group = unified_cgroup() + files: dict[str, Any] = {} + errors = {} + if group["path"] is not None: + for name in OPTIONAL_FILES: + try: + text = (Path(group["path"]) / name).read_text().strip() + except OSError as error: + files[name] = None + errors[name] = f"{type(error).__name__}: {error}" + continue + # Missing optional files are unavailability, but malformed exposed + # counters must not be mistaken for trustworthy zero deltas. + files[name] = ( + flat_counters(text) + if name in COUNTER_FILES + else pressure(text) + if name == "cpu.pressure" + else control_value(name, text) + ) + else: + files = dict.fromkeys(OPTIONAL_FILES) + errors["cgroup"] = group["error"] + affinity = None + try: + if hasattr(os, "sched_getaffinity"): + affinity = sorted(os.sched_getaffinity(0)) + require(bool(affinity), "empty exposed affinity") + else: + errors["affinity"] = "sched_getaffinity unavailable on this platform" + except OSError as error: + errors["affinity"] = f"{type(error).__name__}: {error}" + return { + "qualification": False, + "core": core, + "cgroup": group, + "cgroup_files": files, + "optional_errors": errors, + "affinity": affinity, + "snapshot_read_ns": time.monotonic_ns() - started, + } + + +def counter_delta(before: dict[str, Any], after: dict[str, Any]) -> dict[str, Any]: + require(set(before) == set(after), "counter fields changed within a window") + result = {key: after[key] - value for key, value in before.items()} + require(all(math.isfinite(value) and value >= 0 for value in result.values()), "telemetry counter reset") + return result + + +def telemetry_delta(before: dict[str, Any], after: dict[str, Any]) -> dict[str, Any]: + first, last = before["core"], after["core"] + validate_core(first) + validate_core(last) + require( + last["wall_ns"] > first["wall_ns"] and last["monotonic_ns"] > first["monotonic_ns"], + "required clock did not advance", + ) + require( + first["gc_enabled"] and last["gc_enabled"] and first["gc_thresholds"] == last["gc_thresholds"], + "GC configuration changed", + ) + group_before, group_after = before["cgroup"], after["cgroup"] + require( + all(group_before.get(key) == group_after.get(key) for key in ("path", "device", "inode")), + "cgroup path or identity changed within a window", + ) + require(before["affinity"] == after["affinity"], "process affinity changed within a window") + for name in CONTROL_FILES: + require( + before["cgroup_files"].get(name) == after["cgroup_files"].get(name), + "cgroup quota/weight/cpuset changed within a window", + ) + optional = {} + for name in COUNTER_FILES: + a, b = before["cgroup_files"].get(name), after["cgroup_files"].get(name) + optional[name] = counter_delta(a, b) if a is not None and b is not None else None + a, b = before["cgroup_files"].get("cpu.pressure"), after["cgroup_files"].get("cpu.pressure") + optional["cpu.pressure"] = ( + counter_delta({k: v["total"] for k, v in a.items()}, {k: v["total"] for k, v in b.items()}) + if a is not None and b is not None + else None + ) + return { + "qualification": False, + "scope": "Whole performance call, including sequential warmup, measurement, drain, validation and raw emission", + "wall_s": (last["wall_ns"] - first["wall_ns"]) / 1e9, + "monotonic_s": (last["monotonic_ns"] - first["monotonic_ns"]) / 1e9, + "rusage": counter_delta(first["rusage"], last["rusage"]), + "gc_stats": [counter_delta(a, b) for a, b in zip(first["gc_stats"], last["gc_stats"], strict=True)], + "optional_cgroup": optional, + "cpu_pressure_counters_available": all(optional.get(name) is not None for name in ("cpu.stat", "cpu.pressure")), + "resource_telemetry_complete": before["affinity"] is not None + and after["affinity"] is not None + and all( + before["cgroup_files"].get(name) is not None and after["cgroup_files"].get(name) is not None + for name in OPTIONAL_FILES + ), + "optional_errors_before": before["optional_errors"], + "optional_errors_after": after["optional_errors"], + } + + +# Prospective observation limits, independent of the performance gates. +MAX_CALLBACK_FRACTION = 0.001 +CALLBACK_TAIL_ALLOWANCE_NS = 10_000 +GC_RECORD_CAPACITY = 65_536 +GC_RECORD_WIDTH = 6 + + +class GCRecorder: + """Bounded numeric records, with no I/O, locks or growing collections in callbacks. + + Python still creates transient scalar objects. The reported cost includes + measured callback self time plus a fixed per-call tail allowance; it is an + estimate with a conservative allowance, not proof of zero perturbation. + """ + + def __init__(self, capacity: int = GC_RECORD_CAPACITY) -> None: + require(type(capacity) is int and capacity > 0, "invalid GC recorder capacity") + self.capacity = capacity + self.buffer = array("Q", [0]) * (capacity * GC_RECORD_WIDTH) + self.count = 0 + self.overflow = False + self.invalid = False + self._hook: Any = None + self._prior: tuple[Any, ...] = () + self._thresholds: tuple[int, ...] = () + self._closed = False + self.lifecycle_errors: list[str] = [] + + def _record(self, phase: str, info: dict[str, Any]) -> None: + entered = time.perf_counter_ns() + generation = info.get("generation") + if phase not in ("start", "stop") or type(generation) is not int or not 0 <= generation < GC_GENERATIONS: + self.invalid = True + return + if self.count >= self.capacity: + self.overflow = True + return + collected, uncollectable = info.get("collected", 0), info.get("uncollectable", 0) + if type(collected) is not int or type(uncollectable) is not int or collected < 0 or uncollectable < 0: + self.invalid = True + return + offset = self.count * GC_RECORD_WIDTH + self.buffer[offset] = entered + self.buffer[offset + 1] = int(phase == "stop") + self.buffer[offset + 2] = generation + self.buffer[offset + 3] = collected + self.buffer[offset + 4] = uncollectable + self.count += 1 + self.buffer[offset + 5] = time.perf_counter_ns() - entered + + def start(self) -> None: + require(self._hook is None and not self._closed, "GC recorder cannot be restarted") + require(gc.isenabled() and gc.get_threshold()[0] > 0, "ordinary automatic GC must remain enabled") + self._prior = tuple(gc.callbacks) + self._thresholds = gc.get_threshold() + self._hook = self._record + gc.callbacks.append(self._hook) + + def close(self) -> None: + if self._closed: + return + hook, self._hook = self._hook, None + if hook is None: + self.lifecycle_errors.append("GC recorder was never installed") + else: + matches = [index for index, item in enumerate(gc.callbacks) if item is hook] + if len(matches) != 1: + self.lifecycle_errors.append("owned GC callback identity changed") + for index in reversed(matches): + gc.callbacks.pop(index) + if len(gc.callbacks) != len(self._prior) or any( + current is not prior for current, prior in zip(gc.callbacks, self._prior, strict=False) + ): + self.lifecycle_errors.append("pre-existing GC callbacks changed") + if not gc.isenabled() or gc.get_threshold() != self._thresholds: + self.lifecycle_errors.append("GC enable state or thresholds changed") + self._closed = True + + def report(self, start_s: float, end_s: float) -> dict[str, Any]: + require(self._closed, "GC recorder must close before materializing evidence") + require(math.isfinite(start_s) and math.isfinite(end_s) and 0 < start_s < end_s, "invalid GC measurement clock") + begin, end = round(start_s * 1e9), round(end_s * 1e9) + rows = [ + list(self.buffer[index * GC_RECORD_WIDTH : (index + 1) * GC_RECORD_WIDTH]) for index in range(self.count) + ] + pending: dict[int, int] = {} + intervals = [] + pair_errors = [] + for stamp, stopped, generation, collected, uncollectable, _cost in rows: + if not stopped: + if generation in pending: + pair_errors.append("duplicate GC start") + pending[generation] = stamp + elif generation not in pending: + pair_errors.append("GC stop has no observed start") + else: + first = pending.pop(generation) + if stamp < first: + pair_errors.append("GC event clock reversed") + intervals.append( + { + "generation": generation, + "start_ns": first, + "end_ns": stamp, + "collected": collected, + "uncollectable": uncollectable, + "overlap_ns": max(0, min(end, stamp) - max(begin, first)), + } + ) + if pending: + pair_errors.append("GC start has no observed stop") + relevant = [row for row in rows if row[0] <= end and row[0] + row[5] + CALLBACK_TAIL_ALLOWANCE_NS >= begin] + self_ns = sum(row[5] for row in relevant) + allowance_ns = len(relevant) * CALLBACK_TAIL_ALLOWANCE_NS + estimated_fraction = (self_ns + allowance_ns) / (end - begin) + errors = [*self.lifecycle_errors, *pair_errors] + if self.overflow: + errors.append("GC recorder overflow") + if self.invalid: + errors.append("invalid GC callback event") + integrity_passed = not errors + cost_passed = estimated_fraction <= MAX_CALLBACK_FRACTION + if not cost_passed: + errors.append("GC observer cost exceeds prospective window budget") + return { + "pid": os.getpid(), + "clock": "perf_counter_ns, verified against child/server handshake bounds", + "columns": ["timestamp_ns", "stop", "generation", "collected", "uncollectable", "callback_self_ns"], + "records": rows, + "intervals": intervals, + "window_start_ns": begin, + "window_end_ns": end, + "callback_count_intersecting_window": len(relevant), + "callback_count_entered_in_window": sum(begin <= row[0] <= end for row in rows), + "callback_self_ns": self_ns, + "per_event_tail_allowance_ns": CALLBACK_TAIL_ALLOWANCE_NS, + "estimated_observer_fraction": estimated_fraction, + "maximum_observer_fraction": MAX_CALLBACK_FRACTION, + "uncertainty": "Includes a fixed 10us/call tail allowance requiring a recorded preflight calibration; timing cost does not bound every scheduling/GC perturbation.", + "integrity_passed": integrity_passed, + "cost_budget_passed": cost_passed, + "passed": not errors, + "errors": errors, + } + + +def bracket(before: dict[str, Any], after: dict[str, Any], clock: dict[str, float]) -> dict[str, Any]: + """Attribute exact GC intervals separately from enclosing CPU/cgroup deltas.""" + require(before["core"]["pid"] == after["core"]["pid"], "process identity changed") + begin, end = clock["start"], clock["end"] + require( + before["core"]["perf_counter_ns"] / 1e9 <= begin < end <= after["core"]["perf_counter_ns"] / 1e9, + "process telemetry does not bracket the measured monotonic interval", + ) + delta = telemetry_delta(before, after) + delta["scope"] = ( + "Enclosing process interval around the load-generator timer; includes boundary handshake/observation overhead, not exact per-request CPU time" + ) + delta["cgroup_scope"] = "Shared container counters; never sum client and server observations" + return {"before": before, "after": after, "delta": delta} diff --git a/tools/benchmarks/cross_encoder_lora/stability.py b/tools/benchmarks/cross_encoder_lora/stability.py index dc4bc1779..479900b66 100644 --- a/tools/benchmarks/cross_encoder_lora/stability.py +++ b/tools/benchmarks/cross_encoder_lora/stability.py @@ -15,11 +15,9 @@ import argparse import asyncio -import gc import json import math import os -import resource import signal import socket import statistics @@ -34,10 +32,13 @@ import torch import uvicorn +from sie_sdk import SIEClient from tools.benchmarks.cross_encoder_lora import worker from tools.benchmarks.cross_encoder_lora.adapter import require_forward_selection +from tools.benchmarks.cross_encoder_lora.client import validate_response from tools.benchmarks.cross_encoder_lora.evidence import Evidence, atomic_json, digest, fingerprint, vector_delta +from tools.benchmarks.cross_encoder_lora.process_metrics import CONTROL_FILES, snapshot, telemetry_delta from tools.benchmarks.cross_encoder_lora.protocol import ( BASE_REVISION, CPU_ORACLE, @@ -73,16 +74,7 @@ METRICS = ((1, "mean_ms"), (1, "p95_ms"), (4, "rps"), (4, "p95_ms"), (16, "rps"), (16, "p95_ms")) RUNNER_LIMIT = 1200 WORKER_LIMIT = 180 -GC_FIELDS = ("collections", "collected", "uncollectable") -RUSAGE_FIELDS = ("ru_utime", "ru_stime", "ru_minflt", "ru_majflt", "ru_nvcsw", "ru_nivcsw") -OPTIONAL_FILES = ("cpu.stat", "cpu.stat.local", "cpu.pressure", "cpu.max", "cpu.weight", "cpuset.cpus.effective") -COUNTER_FILES = ("cpu.stat", "cpu.stat.local") -CONTROL_FILES = ("cpu.max", "cpu.weight", "cpuset.cpus.effective") -GC_GENERATIONS = 3 QUARTILES = 4 -PSI_FIELDS = ("avg10", "avg60", "avg300", "total") -MAX_PERCENT = 100 -MAX_CPU_WEIGHT = 10000 class DiagnosticEvidence(Evidence): @@ -128,215 +120,6 @@ def diagnostic_settings(warmups: int) -> Iterator[None]: setattr(worker, name, value) -def unified_cgroup(proc: Path = Path("/proc/self")) -> dict[str, Any]: - """Resolve the process's own namespace; never fall back to another cgroup.""" - try: - rows = (proc / "cgroup").read_text().splitlines() - membership = [row[3:] for row in rows if row.startswith("0::")] - require(len(membership) == 1, "unified cgroup membership is unavailable or ambiguous") - group = Path(membership[0]) - require(group.is_absolute() and ".." not in group.parts, "invalid cgroup membership") - matches = [] - for row in (proc / "mountinfo").read_text().splitlines(): - left, right = row.split(" - ", 1) - if right.split()[0] != "cgroup2": - continue - fields = left.split() - root = Path(fields[3].replace("\\040", " ").replace("\\134", "\\")) - mount = Path(fields[4].replace("\\040", " ").replace("\\134", "\\")) - if group.is_relative_to(root): - matches.append(mount / group.relative_to(root)) - require(len(matches) == 1, "cgroup mount cannot be resolved uniquely") - path = matches[0] - info = path.stat() - return {"path": str(path), "device": info.st_dev, "inode": info.st_ino, "error": None} - except (OSError, ValueError, IndexError, AssertionError) as error: - return {"path": None, "error": f"{type(error).__name__}: {error}"} - - -def flat_counters(text: str) -> dict[str, int]: - result = {} - for row in text.splitlines(): - key, value = row.split() - require(key not in result, "duplicate telemetry counter") - result[key] = int(value) - require(result[key] >= 0, "negative telemetry counter") - require(bool(result), "empty exposed telemetry counter file") - return result - - -def pressure(text: str) -> dict[str, dict[str, float | int]]: - result = {} - for row in text.splitlines(): - name, *parts = row.split() - require(name in {"some", "full"} and name not in result, "invalid PSI row") - values = dict(part.split("=", 1) for part in parts) - require(len(parts) == len(PSI_FIELDS) and set(values) == set(PSI_FIELDS), "invalid PSI fields") - parsed = {key: int(value) if key == "total" else float(value) for key, value in values.items()} - require(all(math.isfinite(value) and value >= 0 for value in parsed.values()), "invalid PSI value") - require(all(parsed[key] <= MAX_PERCENT for key in PSI_FIELDS[:-1]), "invalid PSI percentage") - result[name] = parsed - require("some" in result, "missing exposed PSI some row") - return result - - -def validate_core(core: dict[str, Any]) -> None: - require( - all(isinstance(core[key], int) and core[key] > 0 for key in ("wall_ns", "monotonic_ns")) - and core["gc_enabled"] is True, - "invalid required clock/GC state", - ) - require( - set(core["rusage"]) == set(RUSAGE_FIELDS) - and all(math.isfinite(value) and value >= 0 for value in core["rusage"].values()), - "invalid required rusage", - ) - for key in ("gc_thresholds", "gc_count"): - require( - len(core[key]) == GC_GENERATIONS and all(type(value) is int and value >= 0 for value in core[key]), - "invalid required GC counts/thresholds", - ) - require(core["gc_thresholds"][0] > 0, "automatic GC is disabled by its threshold") - require(len(core["gc_stats"]) == GC_GENERATIONS, "missing required GC generations") - require( - all( - set(row) == set(GC_FIELDS) and all(type(row[key]) is int and row[key] >= 0 for key in GC_FIELDS) - for row in core["gc_stats"] - ), - "invalid required GC statistics", - ) - - -def control_value(name: str, text: str) -> str: - if name == "cpu.max": - quota, period = text.split() - require((quota == "max" or int(quota) > 0) and int(period) > 0, "invalid exposed cpu.max") - elif name == "cpu.weight": - require(1 <= int(text) <= MAX_CPU_WEIGHT, "invalid exposed cpu.weight") - else: - for item in text.split(","): - bounds = [int(value) for value in item.split("-")] - require(len(bounds) in {1, 2} and bounds[0] >= 0 and bounds[-1] >= bounds[0], "invalid exposed cpuset") - return text - - -def snapshot() -> dict[str, Any]: - """Only called outside performance(), including all filesystem reads.""" - started = time.monotonic_ns() - usage = resource.getrusage(resource.RUSAGE_SELF) - core = { - "wall_ns": time.time_ns(), - "monotonic_ns": started, - "rusage": {key: getattr(usage, key) for key in RUSAGE_FIELDS}, - "gc_enabled": gc.isenabled(), - "gc_thresholds": list(gc.get_threshold()), - "gc_count": list(gc.get_count()), - "gc_stats": gc.get_stats(), - } - validate_core(core) - group = unified_cgroup() - files: dict[str, Any] = {} - errors = {} - if group["path"] is not None: - for name in OPTIONAL_FILES: - try: - text = (Path(group["path"]) / name).read_text().strip() - except OSError as error: - files[name] = None - errors[name] = f"{type(error).__name__}: {error}" - continue - # Missing optional files are unavailability, but malformed exposed - # counters must not be mistaken for trustworthy zero deltas. - files[name] = ( - flat_counters(text) - if name in COUNTER_FILES - else pressure(text) - if name == "cpu.pressure" - else control_value(name, text) - ) - else: - files = dict.fromkeys(OPTIONAL_FILES) - errors["cgroup"] = group["error"] - affinity = None - try: - if hasattr(os, "sched_getaffinity"): - affinity = sorted(os.sched_getaffinity(0)) - require(bool(affinity), "empty exposed affinity") - else: - errors["affinity"] = "sched_getaffinity unavailable on this platform" - except OSError as error: - errors["affinity"] = f"{type(error).__name__}: {error}" - return { - "qualification": False, - "core": core, - "cgroup": group, - "cgroup_files": files, - "optional_errors": errors, - "affinity": affinity, - "snapshot_read_ns": time.monotonic_ns() - started, - } - - -def counter_delta(before: dict[str, Any], after: dict[str, Any]) -> dict[str, Any]: - require(set(before) == set(after), "counter fields changed within a window") - result = {key: after[key] - value for key, value in before.items()} - require(all(math.isfinite(value) and value >= 0 for value in result.values()), "telemetry counter reset") - return result - - -def telemetry_delta(before: dict[str, Any], after: dict[str, Any]) -> dict[str, Any]: - first, last = before["core"], after["core"] - validate_core(first) - validate_core(last) - require( - last["wall_ns"] > first["wall_ns"] and last["monotonic_ns"] > first["monotonic_ns"], - "required clock did not advance", - ) - require( - first["gc_enabled"] and last["gc_enabled"] and first["gc_thresholds"] == last["gc_thresholds"], - "GC configuration changed", - ) - group_before, group_after = before["cgroup"], after["cgroup"] - require( - all(group_before.get(key) == group_after.get(key) for key in ("path", "device", "inode")), - "cgroup path or identity changed within a window", - ) - require(before["affinity"] == after["affinity"], "process affinity changed within a window") - for name in CONTROL_FILES: - require( - before["cgroup_files"].get(name) == after["cgroup_files"].get(name), - "cgroup quota/weight/cpuset changed within a window", - ) - optional = {} - for name in COUNTER_FILES: - a, b = before["cgroup_files"].get(name), after["cgroup_files"].get(name) - optional[name] = counter_delta(a, b) if a is not None and b is not None else None - a, b = before["cgroup_files"].get("cpu.pressure"), after["cgroup_files"].get("cpu.pressure") - optional["cpu.pressure"] = ( - counter_delta({k: v["total"] for k, v in a.items()}, {k: v["total"] for k, v in b.items()}) - if a is not None and b is not None - else None - ) - return { - "qualification": False, - "scope": "Whole performance call, including sequential warmup, measurement, drain, validation and raw emission", - "wall_s": (last["wall_ns"] - first["wall_ns"]) / 1e9, - "monotonic_s": (last["monotonic_ns"] - first["monotonic_ns"]) / 1e9, - "rusage": counter_delta(first["rusage"], last["rusage"]), - "gc_stats": [counter_delta(a, b) for a, b in zip(first["gc_stats"], last["gc_stats"], strict=True)], - "optional_cgroup": optional, - "cpu_pressure_counters_available": all(optional.get(name) is not None for name in ("cpu.stat", "cpu.pressure")), - "resource_telemetry_complete": before["affinity"] is not None - and after["affinity"] is not None - and all( - before["cgroup_files"].get(name) is not None and after["cgroup_files"].get(name) is not None - for name in OPTIONAL_FILES - ), - "optional_errors_before": before["optional_errors"], - "optional_errors_after": after["optional_errors"], - } - - def metrics(phase: dict[str, Any]) -> dict[str, Any]: samples = phase["samples"] latencies = [row["latency_ms"] for row in samples] @@ -418,7 +201,7 @@ def verify_attempt(path: Path, record: dict[str, Any]) -> None: ) for row in rows: saved = row["scores"][:] - worker.validate_response(row) + validate_response(row) require(saved == row["scores"] and row["selection"] is None, "baseline scores/selection differ") require(row["end_ns"] >= row["start_ns"] > 0, "invalid request timestamps") if record["concurrency"] == 1: @@ -635,7 +418,7 @@ async def child(args: argparse.Namespace) -> dict[str, Any]: environment["hardware_preflight_only"] = environment.pop("qualification") environment["qualification"] = False environment["source_imports"] = {} - for name, target in (("sdk", worker.SIEClient), ("server", worker.AppFactory)): + for name, target in (("sdk", SIEClient), ("server", worker.AppFactory)): module_file = sys.modules[target.__module__].__file__ assert module_file is not None, "source module has no file" source_file = Path(module_file).resolve() # noqa: ASYNC240 -- preflight before model/server starts diff --git a/tools/benchmarks/cross_encoder_lora/tests/test_client_lifecycle.py b/tools/benchmarks/cross_encoder_lora/tests/test_client_lifecycle.py index 93d899001..23e6c12c6 100644 --- a/tools/benchmarks/cross_encoder_lora/tests/test_client_lifecycle.py +++ b/tools/benchmarks/cross_encoder_lora/tests/test_client_lifecycle.py @@ -15,7 +15,7 @@ from sie_sdk._msgpack import packb, unpackb from sie_sdk.client.errors import ServerError, SIEConnectionError -from tools.benchmarks.cross_encoder_lora import worker +from tools.benchmarks.cross_encoder_lora import client as worker from tools.benchmarks.cross_encoder_lora.evidence import Evidence HTTP_CLIENT = httpx.Client @@ -369,3 +369,47 @@ def emit(*_: Any, **__: Any) -> None: worker.performance(URL, 4, evidence, smoke=True) assert closed == [True] * 4 assert gc.isenabled() == gc_enabled + + +def test_primary_sdk_error_survives_observer_failure_with_all_wire_attempts( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + attempts = 0 + + def handler(request: httpx.Request) -> httpx.Response: + nonlocal attempts + attempts += 1 + if attempts == 4: # two warmups, one completed timed request, one failure + return httpx.Response(500, json={"error": {"code": "INTERNAL_ERROR", "message": "primary SDK failure"}}) + return success(request) + + def after(_clock: dict[str, float]) -> None: + raise BrokenPipeError("secondary observer failed") + + install_transport(monkeypatch, handler) + evidence = Evidence(tmp_path / "out") + with pytest.raises(ServerError) as failure: + worker.performance(URL, 1, evidence, smoke=True, after_measure=after) + assert "secondary observer failed" in " ".join(failure.value.__notes__) + row = json.loads((evidence.directory / "events.jsonl").read_text()) + assert len(row["completed_samples"]) == 1 + assert [value["status"] for value in row["wire_responses"]] == [200, 200, 200, 500] + assert row["primary_error"]["type"] == "ServerError" + assert row["finalization_errors"] == [{"type": "BrokenPipeError", "message": "secondary observer failed"}] + + +def test_before_measure_failure_aborts_waiting_threads_and_preserves_warmups( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + install_transport(monkeypatch, success) + + def before() -> None: + raise OSError("boundary failed before timer") + + evidence = Evidence(tmp_path / "out") + with pytest.raises(OSError, match="boundary failed"): + worker.performance(URL, 4, evidence, smoke=True, before_measure=before) + row = json.loads((evidence.directory / "events.jsonl").read_text()) + assert len(row["warmups"]) == 2 + assert row["completed_samples"] == [] + assert len(row["wire_responses"]) == 2 diff --git a/tools/benchmarks/cross_encoder_lora/tests/test_isolation.py b/tools/benchmarks/cross_encoder_lora/tests/test_isolation.py new file mode 100644 index 000000000..05fb7311e --- /dev/null +++ b/tools/benchmarks/cross_encoder_lora/tests/test_isolation.py @@ -0,0 +1,352 @@ +"""Real SDK-only child protocol, identity, clock and owned-process controls.""" + +from __future__ import annotations + +import ast +import asyncio +import copy +import json +import os +import subprocess +import sys +import threading +from collections.abc import Iterator +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from pathlib import Path +from typing import Any, cast + +import pytest +from sie_sdk._msgpack import packb, unpackb + +from tools.benchmarks.cross_encoder_lora.evidence import Evidence, atomic_json, digest +from tools.benchmarks.cross_encoder_lora.isolation import IsolatedLoadgen +from tools.benchmarks.cross_encoder_lora.loadgen import PHASES +from tools.benchmarks.cross_encoder_lora.protocol import MODEL +from tools.benchmarks.cross_encoder_lora.runner import ROOT, source_environment + + +@pytest.fixture +def sdk_server(monkeypatch: pytest.MonkeyPatch) -> Iterator[str]: + class Handler(BaseHTTPRequestHandler): + protocol_version = "HTTP/1.1" + + def do_POST(self) -> None: + body = unpackb(self.rfile.read(int(self.headers["content-length"])), numeric_arrays=False) + content = packb( + { + "model": MODEL, + "scores": [ + {"item_id": item["id"], "score": float(4 - index), "rank": index} + for index, item in enumerate(body["items"]) + ], + "usage": {"input_tokens": 17}, + } + ) + self.send_response(200) + self.send_header("Content-Type", "application/msgpack") + self.send_header("Content-Length", str(len(content))) + self.send_header("X-Total-Time", "0.01") + self.end_headers() + self.wfile.write(content) + self.wfile.flush() + + def log_message(self, format: str, *args: Any) -> None: + pass + + monkeypatch.setenv("PYTHONPATH", source_environment(ROOT)["PYTHONPATH"]) + server = ThreadingHTTPServer(("127.0.0.1", 0), Handler) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + try: + yield f"http://127.0.0.1:{server.server_port}" + finally: + server.shutdown() + server.server_close() + thread.join(timeout=5) + assert not thread.is_alive() + + +@pytest.mark.asyncio +async def test_one_sdk_only_exec_pid_reused_across_all_phases_and_reaped(sdk_server: str, tmp_path: Path) -> None: + evidence = Evidence(tmp_path / "out") + async with IsolatedLoadgen(sdk_server, ROOT, evidence, smoke=True) as owner: + assert owner.process is not None + pid = owner.process.pid + assert pid != os.getpid() + for concurrency in PHASES: + await owner.phase(concurrency) + assert owner.process.pid == pid + with pytest.raises(AssertionError, match="reaped"): + owner.results() + assert owner.receipt["passed"] + assert owner.receipt["reaped"] + assert owner.receipt["returncode"] == 0 + identity = owner.receipt["identity"] + assert identity["forbidden_imports_present"] is False + assert identity["process_group"] == os.getpgrp() + assert identity["ppid"] == os.getpid() + assert Path(identity["sdk_path"]).is_relative_to(ROOT / "packages/sie_sdk/src") + results = owner.results() + assert list(results) == ["1", "4", "16"] + assert results["1"]["count"] == 4 + for phase in results.values(): + assert phase["identity"] == identity + assert phase["client_observation"]["gc"]["integrity_passed"] + assert phase["server_observation"]["gc"]["integrity_passed"] + assert phase["measurement_clock"] == phase["server_observation"]["clock"] + assert phase["observer_cost_gate_applied"] is False # CPU smoke never qualifies performance. + assert all(len(row["responses"]) == 1 for row in phase["samples"]) + terminal = json.loads((evidence.directory / "loadgen/terminal.json").read_text()) + assert terminal["phases"] == owner.records + with pytest.raises(ProcessLookupError): + os.kill(pid, 0) + + +def synthetic_owner(tmp_path: Path) -> tuple[IsolatedLoadgen, dict[str, Any], dict[str, Any]]: + owner = IsolatedLoadgen("http://unused.invalid", ROOT, Evidence(tmp_path / "out"), smoke=False) + identity = {"pid": 999999, "source": str(ROOT)} + owner.receipt["identity"] = identity + clock = {"start": 1.0, "deadline": 61.0, "end": 61.1} + + def observation(pid: int) -> dict[str, Any]: + return { + "before": {"core": {"pid": pid}}, + "after": {"core": {"pid": pid}}, + "clock": clock, + "gc": { + "pid": pid, + "integrity_passed": True, + "window_start_ns": 1_000_000_000, + "window_end_ns": 61_100_000_000, + "estimated_observer_fraction": 0.0001, + }, + } + + owner.observations[1] = observation(os.getpid()) + record = {"concurrency": 1} + value = { + "concurrency": 1, + "smoke": False, + "identity": identity, + "measurement_clock": clock, + "client_observation": observation(identity["pid"]), + } + return owner, record, value + + +@pytest.mark.parametrize( + "mutation", + [ + "swapped_phase", + "clock", + "observation_clock", + "identity", + "pid", + "window", + "cost_nan", + "combined_budget", + "integrity", + ], +) +def test_phase_payload_must_match_observed_ipc_provenance_and_combined_budget(tmp_path: Path, mutation: str) -> None: + owner, record, original = synthetic_owner(tmp_path) + value = copy.deepcopy(original) + assert owner._validate_phase(record, copy.deepcopy(original))["observer_cost_gate_applied"] + if mutation == "swapped_phase": + value["concurrency"] = 4 + elif mutation == "clock": + value["measurement_clock"]["end"] += 1 + elif mutation == "observation_clock": + value["client_observation"]["clock"] = {"start": 2.0, "deadline": 62.0, "end": 62.1} + elif mutation == "identity": + value["identity"]["source"] = "/wrong" + elif mutation == "pid": + value["client_observation"]["after"]["core"]["pid"] += 1 + elif mutation == "window": + value["client_observation"]["gc"]["window_start_ns"] += 1 + elif mutation == "cost_nan": + value["client_observation"]["gc"]["estimated_observer_fraction"] = float("nan") + elif mutation == "combined_budget": + owner.observations[1]["gc"]["estimated_observer_fraction"] = 0.0006 + value["client_observation"]["gc"]["estimated_observer_fraction"] = 0.0006 + else: + value["client_observation"]["gc"]["integrity_passed"] = False + with pytest.raises(AssertionError): + owner._validate_phase(record, value) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("mutation", ["failed", "identity", "schedule", "smoke"]) +async def test_valid_terminal_hash_cannot_mask_wrong_terminal_payload( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, mutation: str +) -> None: + owner, _, _ = synthetic_owner(tmp_path) + owner.records.extend({"concurrency": c} for c in PHASES) + terminal: dict[str, Any] = { + "passed": True, + "identity": owner.receipt["identity"], + "phases": owner.records, + "smoke": False, + "finished_ns": 1, + } + terminal = copy.deepcopy(terminal) + if mutation == "failed": + terminal["passed"] = False + elif mutation == "identity": + terminal["identity"]["pid"] += 1 + elif mutation == "schedule": + terminal["phases"].reverse() + else: + terminal["smoke"] = True + path = owner.evidence.directory / "loadgen/terminal.json" + path.parent.mkdir() + atomic_json(path, terminal) + + class Exited: + returncode = 0 + stdin = None + + async def wait(self) -> int: + return 0 + + async def send(*_args: Any) -> None: + pass + + async def read(*_args: Any, **_kwargs: Any) -> dict[str, Any]: + return {"terminal_sha256": digest(path)} + + owner.process = cast("asyncio.subprocess.Process", Exited()) + monkeypatch.setattr(owner, "_send", send) + monkeypatch.setattr(owner, "_read", read) + with pytest.raises(AssertionError, match="terminal"): + await owner._cleanup(True) + assert owner.receipt["passed"] is False + assert owner.receipt["reaped"] is True + + +@pytest.mark.asyncio +async def test_cancel_during_finish_handshake_reaps_silent_real_child(tmp_path: Path) -> None: + owner = IsolatedLoadgen("http://unused.invalid", ROOT, Evidence(tmp_path / "out"), smoke=True) + owner.process = await asyncio.create_subprocess_exec( + sys.executable, + "-c", + "import time; time.sleep(60)", + stdin=asyncio.subprocess.PIPE, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.DEVNULL, + close_fds=True, + ) + owner.records.extend({"concurrency": c} for c in PHASES) + task = asyncio.create_task(owner.__aexit__(None, None, None)) + await asyncio.sleep(0.05) + task.cancel() + with pytest.raises(asyncio.CancelledError): + await asyncio.wait_for(task, timeout=8) + assert owner.receipt["reaped"] + assert owner.process is not None + assert owner.process.returncode is not None + + +@pytest.mark.asyncio +async def test_child_crash_and_parent_phase_cancellation_reap(sdk_server: str, tmp_path: Path) -> None: + owner = IsolatedLoadgen(sdk_server, ROOT, Evidence(tmp_path / "out"), smoke=True) + with pytest.raises(AssertionError, match="missing"): + async with owner: + assert owner.process is not None + owner.process.kill() + await owner.process.wait() + await owner._read("ready", 1) + assert owner.receipt["reaped"] + assert owner.process is not None + assert owner.process.returncode is not None + + +@pytest.mark.asyncio +async def test_termination_exit_race_preserves_original_failure(tmp_path: Path) -> None: + owner = IsolatedLoadgen("http://unused.invalid", ROOT, Evidence(tmp_path / "out"), smoke=True) + + class RacedExit: + returncode = None + stdin = None + + def terminate(self) -> None: + raise ProcessLookupError("already exited") + + async def wait(self) -> int: + self.returncode = 1 + return 1 + + owner.process = cast("asyncio.subprocess.Process", RacedExit()) + original = ValueError("primary phase failure") + await owner.__aexit__(ValueError, original, None) + assert owner.receipt["termination_raced_exit"] + assert owner.receipt["reaped"] + assert owner.receipt["returncode"] == 1 + + +@pytest.mark.asyncio +async def test_cleanup_write_failure_never_masks_primary_phase_error( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + from tools.benchmarks.cross_encoder_lora import isolation + + owner = IsolatedLoadgen("http://unused.invalid", ROOT, Evidence(tmp_path / "out"), smoke=True) + + def fail(*_args: Any) -> None: + raise OSError("injected receipt write failure") + + monkeypatch.setattr(isolation, "atomic_json", fail) + original = ValueError("original phase error") + await owner.__aexit__(ValueError, original, None) + assert any("receipt write failure" in note for note in original.__notes__) + assert owner.receipt["reaped"] + + +def test_sdk_calls_lanes_and_measured_executor_core_match_recorded_harness() -> None: + from tools.benchmarks.cross_encoder_lora import client + + baseline = subprocess.run( + ["git", "show", "a2932d81:tools/benchmarks/cross_encoder_lora/worker.py"], # noqa: S607 + cwd=ROOT, + capture_output=True, + text=True, + check=False, + ) + if baseline.returncode: + pytest.skip("source invariant requires the recorded parent commit in git history") + old, new = ast.parse(baseline.stdout), ast.parse(Path(client.__file__).read_text()) + + def definition(tree: ast.AST, name: str) -> ast.AST: + return next( + node for node in ast.walk(tree) if isinstance(node, (ast.FunctionDef, ast.ClassDef)) and node.name == name + ) + + class Names(ast.NodeTransformer): + def visit_Name(self, node: ast.Name) -> ast.Name: + names = { + "sequential": "SEQUENTIAL", + "steady_seconds": "STEADY_SECONDS", + "future_timeout": "PERFORMANCE_FUTURE_TIMEOUT_S", + } + return ast.Name(id=names.get(node.id, node.id), ctx=node.ctx) + + assert ast.dump(definition(old, "Client")) == ast.dump(definition(new, "Client")) + assert ast.dump(definition(old, "lane")) == ast.dump(Names().visit(definition(new, "lane"))) + + def executor(tree: ast.AST) -> ast.With: + return next( + node + for node in ast.walk(definition(tree, "performance")) + if isinstance(node, ast.With) + and any( + isinstance(item.context_expr, ast.Call) + and getattr(item.context_expr.func, "id", None) == "ThreadPoolExecutor" + for item in node.items + ) + ) + + new_executor = executor(new) + # Only the pre-clock readiness hook is new. It aborts the barrier on failure. + assert isinstance(new_executor.body[1], ast.Try) + new_executor.body.pop(1) + assert ast.dump(executor(old)) == ast.dump(Names().visit(new_executor)) diff --git a/tools/benchmarks/cross_encoder_lora/tests/test_probe.py b/tools/benchmarks/cross_encoder_lora/tests/test_probe.py index 8864463da..339efbc0c 100644 --- a/tools/benchmarks/cross_encoder_lora/tests/test_probe.py +++ b/tools/benchmarks/cross_encoder_lora/tests/test_probe.py @@ -18,11 +18,12 @@ from tools.benchmarks.cross_encoder_lora import prepare as source_preparer from tools.benchmarks.cross_encoder_lora.adapter import require_forward_coverage +from tools.benchmarks.cross_encoder_lora.client import describe_response, validate_response from tools.benchmarks.cross_encoder_lora.evidence import Evidence, atomic_json, vector_delta from tools.benchmarks.cross_encoder_lora.fixtures import prepare from tools.benchmarks.cross_encoder_lora.protocol import BASE_REVISION, ORDERS, compare, paired_gate, percentile from tools.benchmarks.cross_encoder_lora.runner import ROOT, extract, source_environment -from tools.benchmarks.cross_encoder_lora.worker import ORDER, describe_response, matching_vector, validate_response +from tools.benchmarks.cross_encoder_lora.worker import ORDER, matching_vector def tiny_snapshot(directory: Path) -> Path: diff --git a/tools/benchmarks/cross_encoder_lora/tests/test_process_metrics.py b/tools/benchmarks/cross_encoder_lora/tests/test_process_metrics.py new file mode 100644 index 000000000..bbbf35f80 --- /dev/null +++ b/tools/benchmarks/cross_encoder_lora/tests/test_process_metrics.py @@ -0,0 +1,210 @@ +"""GC recording is bounded and diagnostic counters keep their actual scope.""" + +from __future__ import annotations + +import gc +import time +from array import array +from pathlib import Path +from typing import Any + +import pytest + +from tools.benchmarks.cross_encoder_lora import process_metrics as metrics + + +def report(rows: list[list[int]], start: float = 1.0, end: float = 2.0) -> dict[str, Any]: + recorder = metrics.GCRecorder(capacity=max(1, len(rows))) + recorder.start() + recorder.close() + recorder.count = len(rows) + recorder.buffer = array("Q", [value for row in rows for value in row]) + return recorder.report(start, end) + + +def test_actual_gc_records_preserve_callbacks_thresholds_and_pairing() -> None: + callbacks = tuple(gc.callbacks) + thresholds = gc.get_threshold() + recorder = metrics.GCRecorder() + before = metrics.snapshot() + recorder.start() + started = time.perf_counter() + cycle = [] + cycle.append(cycle) + del cycle + gc.collect() + ended = time.perf_counter() + recorder.close() + after = metrics.snapshot() + value = recorder.report(started, ended) + assert value["integrity_passed"] + assert value["records"] + assert any(interval["generation"] == 2 and interval["collected"] >= 1 for interval in value["intervals"]) + assert tuple(gc.callbacks) == callbacks + assert gc.get_threshold() == thresholds + assert gc.isenabled() + assert recorder._hook is None + bracket = metrics.bracket(before, after, {"start": started, "end": ended}) + assert "Enclosing process interval" in bracket["delta"]["scope"] + assert "never sum" in bracket["delta"]["cgroup_scope"] + + +def test_callback_straddling_start_is_charged_and_collection_overlap_reported() -> None: + value = report([[999_999_000, 0, 0, 0, 0, 500], [1_100_000_000, 1, 0, 4, 0, 500]]) + assert value["passed"] + assert value["callback_count_entered_in_window"] == 1 + assert value["callback_count_intersecting_window"] == 2 + assert value["callback_self_ns"] == 1000 + assert value["intervals"][0]["overlap_ns"] == 100_000_000 + assert value["estimated_observer_fraction"] == 0.000021 + + +@pytest.mark.parametrize( + "rows", + [ + [[1_100_000_000, 0, 0, 0, 0, 1]], + [[1_100_000_000, 1, 0, 0, 0, 1]], + [[1_200_000_000, 0, 0, 0, 0, 1], [1_100_000_000, 1, 0, 0, 0, 1]], + [[1_100_000_000, 0, 0, 0, 0, 1], [1_200_000_000, 0, 0, 0, 0, 1]], + ], +) +def test_incomplete_or_reversed_gc_events_fail_integrity(rows: list[list[int]]) -> None: + assert not report(rows)["integrity_passed"] + + +def test_overflow_and_invalid_events_fail_without_unbounded_growth() -> None: + recorder = metrics.GCRecorder(capacity=1) + recorder.start() + recorder._record("start", {"generation": 0}) + recorder._record("stop", {"generation": 0}) + recorder._record("bad", {"generation": 0}) + recorder.close() + value = recorder.report(1.0, time.perf_counter()) + assert len(recorder.buffer) == metrics.GC_RECORD_WIDTH + assert not value["integrity_passed"] + assert "GC recorder overflow" in value["errors"] + assert "invalid GC callback event" in value["errors"] + + +def test_prior_callbacks_are_preserved_and_foreign_change_rejects() -> None: + def prior(_phase: str, _info: dict[str, Any]) -> None: + pass + + def unexpected(_phase: str, _info: dict[str, Any]) -> None: + pass + + gc.callbacks.append(prior) + recorder = metrics.GCRecorder() + try: + recorder.start() + gc.callbacks.append(unexpected) + recorder.close() + assert prior in gc.callbacks + assert unexpected in gc.callbacks + assert "pre-existing GC callbacks changed" in recorder.report(1.0, 2.0)["errors"] + finally: + recorder.close() + gc.callbacks.remove(prior) + gc.callbacks.remove(unexpected) + + +def test_cost_limit_never_changes_performance_gates() -> None: + value = report([[1_100_000_000, 0, 0, 0, 0, 600_000], [1_200_000_000, 1, 0, 0, 0, 600_000]]) + assert value["integrity_passed"] + assert not value["cost_budget_passed"] + assert not value["passed"] + assert value["maximum_observer_fraction"] == 0.001 + + +@pytest.mark.parametrize(("started", "ended"), [(2.0, 1.0), (1.0, 1.0), (float("nan"), 2.0)]) +def test_invalid_measurement_clock_rejected(started: float, ended: float) -> None: + with pytest.raises(AssertionError, match="clock"): + report([], started, ended) + + +def test_callable_equality_cannot_hide_changed_callback_identity() -> None: + class Permissive: + def __call__(self, _phase: str, _info: dict[str, Any]) -> None: + pass + + def __eq__(self, _other: object) -> bool: + return True + + def __hash__(self) -> int: + return 0 + + original, replacement = Permissive(), Permissive() + gc.callbacks.append(original) + recorder = metrics.GCRecorder() + try: + recorder.start() + index = next(index for index, value in enumerate(gc.callbacks) if value is original) + gc.callbacks[index] = replacement + recorder.close() + assert "pre-existing GC callbacks changed" in recorder.report(1.0, 2.0)["errors"] + finally: + recorder.close() + gc.callbacks[:] = [value for value in gc.callbacks if value is not original and value is not replacement] + + +def test_calibration_uses_six_units_fixed_allowance_and_signed_residuals() -> None: + from tools.benchmarks.cross_encoder_lora.calibrate_gc import bound + + assert bound([-100.0] * 6)["mean_ns"] == -100.0 + assert bound([10_000.0] * 6)["passed"] + assert not bound([10_000.1] * 6)["passed"] + uncertain = bound([0, 0, 0, 10_000, 10_000, 10_000]) + assert uncertain["one_sided_95_upper_ns"] > uncertain["mean_ns"] + with pytest.raises(AssertionError, match="six"): + bound([1.0] * 5) + + +def test_calibration_preserves_gc_and_exercises_nonzero_fields() -> None: + from tools.benchmarks.cross_encoder_lora.calibrate_gc import collect_arm, direct + + prior = tuple(gc.callbacks) + thresholds = gc.get_threshold() + for observed in (False, True): + value = collect_arm(observed, 5) + assert len(value["duration_ns"]) == 5 + assert len(value["records"]) == (10 if observed else 0) + for generation in range(3): + value = direct(generation, 5) + assert value["info"]["collected"] > 0 + assert value["info"]["uncollectable"] > 0 + assert len(value["residual_ns"]) == 10 + assert tuple(gc.callbacks) == prior + assert gc.get_threshold() == thresholds + assert gc.isenabled() + + +def test_calibration_failure_writes_partial_raw_arms_before_reraising( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + import json + import sys + + from tools.benchmarks.cross_encoder_lora import calibrate_gc + + class ExtraStop(metrics.GCRecorder): + def _record(self, phase: str, info: dict[str, Any]) -> None: + super()._record(phase, info) + if phase == "stop": + super()._record(phase, info) + + monkeypatch.setattr(calibrate_gc, "GCRecorder", ExtraStop) + monkeypatch.setattr(calibrate_gc, "COLLECTIONS", 3) + output = tmp_path / "failed-calibration.json" + monkeypatch.setattr(sys, "argv", ["calibrate_gc", "--output", str(output)]) + with pytest.raises(AssertionError, match="extra/missing"): + calibrate_gc.main() + value = json.loads(output.read_text()) + assert value["passed"] is False + assert value["qualification"] is False + assert value["error"]["type"] == "AssertionError" + assert value["blocks"][0]["arms"]["False"]["passed"] + observed = value["blocks"][0]["arms"]["True"] + assert not observed["passed"] + assert observed["completed"] == 3 + assert len(observed["duration_ns"]) == 3 + assert len(observed["records"]) == 9 diff --git a/tools/benchmarks/cross_encoder_lora/tests/test_stability.py b/tools/benchmarks/cross_encoder_lora/tests/test_stability.py index c7230fc93..f0973a6e8 100644 --- a/tools/benchmarks/cross_encoder_lora/tests/test_stability.py +++ b/tools/benchmarks/cross_encoder_lora/tests/test_stability.py @@ -16,8 +16,8 @@ import pytest from tools.benchmarks.cross_encoder_lora import prepare as source_preparer +from tools.benchmarks.cross_encoder_lora import process_metrics, worker from tools.benchmarks.cross_encoder_lora import stability as probe -from tools.benchmarks.cross_encoder_lora import worker from tools.benchmarks.cross_encoder_lora.evidence import digest, fingerprint from tools.benchmarks.cross_encoder_lora.fixtures import prepare from tools.benchmarks.cross_encoder_lora.protocol import BASE_REVISION @@ -31,14 +31,14 @@ def observed(tick: int = 1) -> dict[str, Any]: "core": { "wall_ns": tick * 1000000000, "monotonic_ns": tick * 1000000000, - "rusage": dict.fromkeys(probe.RUSAGE_FIELDS, tick), + "rusage": dict.fromkeys(process_metrics.RUSAGE_FIELDS, tick), "gc_enabled": True, "gc_thresholds": [700, 10, 10], "gc_count": [tick, 0, 0], - "gc_stats": [dict.fromkeys(probe.GC_FIELDS, tick) for _ in range(3)], + "gc_stats": [dict.fromkeys(process_metrics.GC_FIELDS, tick) for _ in range(3)], }, "cgroup": {"path": None, "error": "unavailable"}, - "cgroup_files": dict.fromkeys(probe.OPTIONAL_FILES), + "cgroup_files": dict.fromkeys(process_metrics.OPTIONAL_FILES), "optional_errors": {"cgroup": "unavailable"}, "affinity": None, "snapshot_read_ns": 1, @@ -116,10 +116,10 @@ def test_all_fixed_process_units_and_positive_metrics_required(failure: str) -> def test_optional_cgroup_unavailable_is_explicit_not_zero(monkeypatch: pytest.MonkeyPatch) -> None: - monkeypatch.setattr(probe, "unified_cgroup", lambda: {"path": None, "error": "not exposed"}) + monkeypatch.setattr(process_metrics, "unified_cgroup", lambda: {"path": None, "error": "not exposed"}) first, last = probe.snapshot(), probe.snapshot() delta = probe.telemetry_delta(first, last) - assert set(first["cgroup_files"]) == set(probe.OPTIONAL_FILES) + assert set(first["cgroup_files"]) == set(process_metrics.OPTIONAL_FILES) assert all(value is None for value in first["cgroup_files"].values()) assert delta["optional_cgroup"]["cpu.stat"] is None assert not delta["resource_telemetry_complete"] @@ -140,7 +140,7 @@ def test_required_telemetry_validation(failure: str) -> None: else: core["gc_enabled"] = False with pytest.raises(AssertionError): - probe.validate_core(core) + process_metrics.validate_core(core) def test_own_cgroup_resolution_and_no_namespace_fallback(tmp_path: Path) -> None: @@ -152,20 +152,22 @@ def test_own_cgroup_resolution_and_no_namespace_fallback(tmp_path: Path) -> None (proc / "cgroup").write_text("0::/namespace/child\n") line = f"1 0 0:1 /namespace {mount} rw - cgroup2 cgroup rw\n" (proc / "mountinfo").write_text(line) - assert probe.unified_cgroup(proc)["path"] == str(owned) + assert process_metrics.unified_cgroup(proc)["path"] == str(owned) (proc / "mountinfo").write_text(line + line) - assert probe.unified_cgroup(proc)["path"] is None + assert process_metrics.unified_cgroup(proc)["path"] is None (proc / "cgroup").write_text("0::/other/child\n") (proc / "mountinfo").write_text(line) - assert probe.unified_cgroup(proc)["path"] is None + assert process_metrics.unified_cgroup(proc)["path"] is None def test_exposed_counters_and_gauges_have_distinct_semantics() -> None: first, last = observed(1), observed(2) for record, ticks, avg in ((first, 100, 90), (last, 120, 1)): record["cgroup"] = {"path": "/owned", "device": 1, "inode": 2, "error": None} - record["cgroup_files"]["cpu.stat"] = probe.flat_counters(f"usage_usec {ticks}\nnr_throttled 1\n") - record["cgroup_files"]["cpu.pressure"] = probe.pressure(f"some avg10={avg} avg60=0 avg300=0 total={ticks}\n") + record["cgroup_files"]["cpu.stat"] = process_metrics.flat_counters(f"usage_usec {ticks}\nnr_throttled 1\n") + record["cgroup_files"]["cpu.pressure"] = process_metrics.pressure( + f"some avg10={avg} avg60=0 avg300=0 total={ticks}\n" + ) last["core"]["gc_count"] = [0, 0, 0] # Automatic GC resets this gauge normally. delta = probe.telemetry_delta(first, last) assert delta["cpu_pressure_counters_available"] @@ -177,9 +179,9 @@ def test_exposed_counters_and_gauges_have_distinct_semantics() -> None: probe.telemetry_delta(first, last) for text in ("usage_usec -1", "usage_usec 0\nusage_usec 1", ""): with pytest.raises((AssertionError, ValueError)): - probe.flat_counters(text) + process_metrics.flat_counters(text) with pytest.raises(AssertionError, match="PSI"): - probe.pressure("some avg10=nan avg60=0 avg300=0 total=1") + process_metrics.pressure("some avg10=nan avg60=0 avg300=0 total=1") @pytest.mark.parametrize( @@ -347,59 +349,28 @@ def test_negative_raw_latency_cannot_hide_in_positive_aggregates() -> None: probe.metrics(phase) -def test_serving_observer_timed_loop_and_gc_are_unmodified() -> None: +def test_serving_observer_and_gc_are_unmodified() -> None: baseline = subprocess.run( - ["git", "show", "f06f414a:tools/benchmarks/cross_encoder_lora/worker.py"], # noqa: S607 + ["git", "show", "a2932d81:tools/benchmarks/cross_encoder_lora/worker.py"], # noqa: S607 cwd=ROOT, capture_output=True, text=True, check=False, ) if baseline.returncode: - pytest.skip("timed-code invariant control requires its documented parent commit in git history") - # The Client capture lifecycle is intentionally amended. Preserve the - # production observation boundary and actual timed loop, allowing only - # the named wait limit that gained scheduling headroom for the longer run. - names = {"performance", "unobserved"} - - class TimedCore(ast.NodeTransformer): - def visit_FunctionDef(self, node: ast.FunctionDef) -> ast.FunctionDef: - if node.name == "performance": - timed = next(value for value in node.body if isinstance(value, ast.Try)) - end = next( - index - for index, value in enumerate(timed.body) - if isinstance(value, ast.Assign) - and any(isinstance(target, ast.Name) and target.id == "elapsed" for target in value.targets) - ) - timed.body = timed.body[: end + 1] - timed.finalbody = [] - self.generic_visit(node) - return node - - def visit_Call(self, node: ast.Call) -> ast.Call: - if ( - isinstance(node.func, ast.Attribute) - and node.func.attr == "result" - and isinstance(node.func.value, ast.Name) - and node.func.value.id == "future" - ): - timeout = next(value for value in node.keywords if value.arg == "timeout") - assert (isinstance(timeout.value, ast.Constant) and timeout.value.value == 90) or ( - isinstance(timeout.value, ast.Name) and timeout.value.id == "PERFORMANCE_FUTURE_TIMEOUT_S" - ) - timeout.value = ast.Constant(value=90) - self.generic_visit(node) - return node + pytest.skip("invariant control requires its documented parent commit in git history") def code(text: str) -> dict[str, str]: return { - node.name: ast.dump(TimedCore().visit(node)) + node.name: ast.dump(node) for node in ast.parse(text).body - if isinstance(node, (ast.FunctionDef, ast.ClassDef)) and node.name in names + if isinstance(node, ast.FunctionDef) and node.name in {"unobserved", "memory"} } assert code(Path(worker.__file__).read_text()) == code(baseline.stdout) + # Historical diagnostic timing still uses the in-process wrapper and does + # not force GC. The new qualification load loop has its own extraction and + # clock-boundary tests; exact historical replay remains pinned to 482b. tree = ast.parse(Path(probe.__file__).read_text()) assert not any( isinstance(node, ast.Call) diff --git a/tools/benchmarks/cross_encoder_lora/worker.py b/tools/benchmarks/cross_encoder_lora/worker.py index 110fd52d3..01c56d044 100644 --- a/tools/benchmarks/cross_encoder_lora/worker.py +++ b/tools/benchmarks/cross_encoder_lora/worker.py @@ -12,7 +12,6 @@ import os import resource import socket -import statistics import subprocess import sys import threading @@ -20,26 +19,30 @@ import weakref from collections.abc import Iterator from concurrent.futures import ThreadPoolExecutor -from contextlib import ExitStack, contextmanager +from contextlib import contextmanager from http import HTTPStatus from pathlib import Path from typing import Any -import httpx import torch import uvicorn import yaml -from sie_sdk import SIEClient -from sie_sdk._msgpack import unpackb -from sie_sdk.client.errors import RequestError, ServerError +from sie_sdk.client.errors import RequestError from sie_server.adapters.cross_encoder import CrossEncoderAdapter from sie_server.adapters.peft_lora_mixin import PEFTLoRAMixin from sie_server.app.app_factory import AppFactory from sie_server.app.app_state_config import AppStateConfig from sie_server.types.inputs import Item +from tools.benchmarks.cross_encoder_lora import client as client_workload from tools.benchmarks.cross_encoder_lora.adapter import require_forward_selection +from tools.benchmarks.cross_encoder_lora.client import ( + REQUEST_ITEMS, + Client, + describe_response, +) from tools.benchmarks.cross_encoder_lora.evidence import Evidence, atomic_json, digest, fingerprint, vector_delta +from tools.benchmarks.cross_encoder_lora.isolation import IsolatedLoadgen from tools.benchmarks.cross_encoder_lora.protocol import ( CONCURRENCIES, CPU_ORACLE, @@ -59,17 +62,13 @@ THREADS, WARMUP, WEIGHTS_SHA256, - percentile, require, ) +RESPONSE_CAPTURE_MODE = client_workload.RESPONSE_CAPTURE_MODE HERE = Path(__file__).resolve().parent ORDER = sorted(range(len(DOCUMENTS)), key=lambda index: len(DOCUMENTS[index])) -MAX_LOADING_RETRIES = 10 -MAX_LOADING_DELAY = 5 RESIDENT_CAPACITY = 2 -REQUEST_ITEMS = len(DOCUMENTS) -RESPONSE_CAPTURE_MODE = "success-status-header-snapshots" def error_record(error: BaseException) -> dict[str, Any]: @@ -204,152 +203,6 @@ def adapter_storage(adapter: Any) -> dict[str, int]: } -class Client: - def __init__(self, url: str) -> None: - self.responses: list[dict[str, Any] | httpx.Response] = [] - http = httpx.Client(base_url=url, timeout=30, event_hooks={"response": [self.observe]}) - self._http = http - self.sdk = SIEClient(url, timeout_s=30, http_client=http) - - def observe(self, response: httpx.Response) -> None: - # Keep successful wire evidence without retaining its Response/Request/ - # stream graph for the whole phase. Failure bodies are decoded later. - self.responses.append(describe_response(response) if response.status_code == HTTPStatus.OK else response) - - def score(self, selection: str | None, label: str, *, cold: bool = False, timed: bool = False) -> dict[str, Any]: - started = time.perf_counter_ns() - wall_started = time.time_ns() - first = len(self.responses) - # score() currently does not retry LORA_LOADING. These are explicitly - # fresh SDK calls for this pre-execution status, never transport retries. - for attempt in range(MAX_LOADING_RETRIES + 1): - try: - result = self.sdk.score( - MODEL, - {"text": QUERY["text"]}, - [{"id": f"{label}:{index}", "text": text} for index, text in enumerate(DOCUMENTS)], - options={"lora": selection}, - wait_for_capacity=False, - max_oom_retries=0, - provision_timeout_s=30, - ) - break - except ServerError as error: - if ( - not cold - or error.status_code != HTTPStatus.SERVICE_UNAVAILABLE - or error.code != "LORA_LOADING" - or attempt == MAX_LOADING_RETRIES - ): - raise - delay = error.retry_after if error.retry_after is not None else 0.2 - require(0 <= delay <= MAX_LOADING_DELAY, "unexpected loading retry delay") - time.sleep(delay) - elapsed = (time.perf_counter_ns() - started) / 1e6 - row = { - "label": label, - "selection": selection, - "latency_ms": elapsed, - "start_ns": wall_started, - "end_ns": time.time_ns(), - "result": result, - "responses": self.responses[first:], - } - if not timed: - validate_response(row, allow_loading=cold) - return row - - def close(self) -> None: - try: - self.sdk.close() - finally: - # Callers serialize their evidence before closing. Preserve unrelated - # hooks, and break only the observer/history this benchmark owns. - hooks = self._http.event_hooks["response"] - hooks[:] = [ - hook - for hook in hooks - if not ( - getattr(hook, "__self__", None) is self and getattr(hook, "__func__", None) is type(self).observe - ) - ] - self.responses.clear() - - -def describe_response(response: httpx.Response | dict[str, Any]) -> dict[str, Any]: - if not isinstance(response, httpx.Response): - return response - row: dict[str, Any] = { - "status": response.status_code, - "timing_headers": { - name: response.headers[name] - for name in ( - "x-queue-time", - "x-tokenization-time", - "x-inference-time", - "x-postprocessing-time", - "x-total-time", - ) - if name in response.headers - }, - } - if response.status_code >= HTTPStatus.BAD_REQUEST: - row["content_type"] = response.headers.get("content-type") - row["retry_after"] = response.headers.get("retry-after") - try: - response.read() - except (httpx.StreamError, httpx.HTTPError) as error: - # A partially read error stream can already have failed in the SDK. - # Preserve that original failure and its wire metadata; diagnostic - # re-reading must not replace it or discard earlier attempts. - row.update(body_unavailable=True, body_read_error=type(error).__name__, code=None) - return row - row["body_hex"] = response.content.hex() - try: - body = ( - unpackb(response.content, numeric_arrays=False) - if "msgpack" in response.headers.get("content-type", "") - else response.json() - ) - except ValueError as error: - row["body_decode_error"] = type(error).__name__ - body = None - error = body.get("error", body.get("detail", body)) if isinstance(body, dict) else None - row["code"] = error.get("code") if isinstance(error, dict) else None - return row - - -def serialize_responses(row: dict[str, Any]) -> None: - row["responses"] = [ - describe_response(value) if isinstance(value, httpx.Response) else value for value in row["responses"] - ] - - -def validate_response(row: dict[str, Any], *, allow_loading: bool = False) -> None: - serialize_responses(row) - statuses = row["responses"] - require(statuses and statuses[-1]["status"] == HTTPStatus.OK, "missing successful HTTP response") - if allow_loading: - require( - all( - value["status"] == HTTPStatus.SERVICE_UNAVAILABLE and value.get("code") == "LORA_LOADING" - for value in statuses[:-1] - ), - "unexpected retry during cold adapter request", - ) - else: - require(len(statuses) == 1, "hidden SDK retry disqualifies this request") - scores = row["result"]["scores"] - expected_ids = [f"{row['label']}:{i}" for i in range(REQUEST_ITEMS)] - by_id = {item["item_id"]: item["score"] for item in scores} - require(len(scores) == REQUEST_ITEMS and set(by_id) == set(expected_ids), "score result lost or changed item IDs") - require([item["rank"] for item in scores] == list(range(REQUEST_ITEMS)), "score ranks changed") - values = [by_id[key] for key in expected_ids] - row["scores"] = values - require(all(math.isfinite(value) for value in values), "nonfinite SDK score") - require([item["score"] for item in scores] == sorted(values, reverse=True), "score ranks are unordered") - - @contextmanager def unobserved(adapter: Any) -> Iterator[None]: """Time production methods with the original forward attribute lookup restored.""" @@ -395,60 +248,17 @@ def matching_vector(scores: list[float], oracles: dict[str, Any]) -> dict[str, A def performance(url: str, concurrency: int, evidence: Evidence, *, smoke: bool) -> dict[str, Any]: - clients = [Client(url) for _ in range(concurrency)] - samples: list[dict[str, Any]] = [] - warmups: list[dict[str, Any]] = [] - lock = threading.Lock() - try: - for i in range(2 if smoke else WARMUP): - warmups.append(clients[0].score(None, f"warm-C{concurrency}-{i}", timed=True)) - barrier = threading.Barrier(concurrency + 1) - clock: dict[str, float] = {} - - def lane(index: int) -> None: - barrier.wait(timeout=30) - counter = 0 - while ( - counter < (4 if smoke else SEQUENTIAL) if concurrency == 1 else time.perf_counter() < clock["deadline"] - ): - row = clients[index].score(None, f"C{concurrency}-lane{index}-{counter}", timed=True) - with lock: - samples.append(row) - counter += 1 - - with ThreadPoolExecutor(max_workers=concurrency) as executor: - futures = [executor.submit(lane, i) for i in range(concurrency)] - clock["start"] = time.perf_counter() - clock["deadline"] = clock["start"] + (0.15 if smoke else STEADY_SECONDS) - barrier.wait(timeout=30) - for future in futures: - future.result(timeout=PERFORMANCE_FUTURE_TIMEOUT_S) - elapsed = time.perf_counter() - clock["start"] - for row in [*warmups, *samples]: - validate_response(row) - latencies = [row["latency_ms"] for row in samples] - return { - "concurrency": concurrency, - "count": len(samples), - "elapsed_s": elapsed, - "mean_ms": statistics.mean(latencies), - "p95_ms": percentile(latencies, 0.95), - "rps": len(samples) / elapsed, - "samples": samples, - } - finally: - with ExitStack() as cleanup: - for client in clients: - cleanup.callback(client.close) - for row in [*warmups, *samples]: - serialize_responses(row) - evidence.emit( - "performance_attempt", - concurrency=concurrency, - completed_samples=samples, - warmups=warmups, - wire_responses=[describe_response(response) for client in clients for response in client.responses], - ) + """Retain the historical in-process diagnostic with its explicit settings.""" + return client_workload.performance( + url, + concurrency, + evidence, + smoke=smoke, + warmup=WARMUP, + sequential=SEQUENTIAL, + steady_seconds=STEADY_SECONDS, + future_timeout=PERFORMANCE_FUTURE_TIMEOUT_S, + ) def validate_usage(rows: list[dict[str, Any]], batches: list[dict[str, Any]]) -> None: @@ -766,29 +576,43 @@ def concurrent(index: int) -> dict[str, Any]: ) summary["resident_memory"] = memory() summary["performance"] = {} + summary["pre_timing_history"] = { + "batches": len(adapter.batch_observations), + "selections": len(adapter.selection_observations), + "fetches": len(adapter.fetch_observations), + "main_client_wire_attempts": len(client.responses), + "retained_untimed_replays": sum(len(value["requests"]) for value in summary["untimed_replays"].values()), + "policy": "Existing server histories retained; memory() just completed its existing gc.collect/CUDA synchronization outside timing. No new collection or history release is introduced.", + } with unobserved(adapter): - for concurrency in (1, *CONCURRENCIES): - phase = f"performance-C{concurrency}" - evidence.emit("gpu_before_phase", phase=phase, telemetry=telemetry()) - measured = await asyncio.to_thread(performance, url, concurrency, evidence, smoke=args.smoke) - summary["performance"][str(concurrency)] = measured - evidence.emit("performance", phase=phase, **measured) - atomic_json(evidence.directory / "trial.json", summary) - for row in measured["samples"]: - require(row["result"]["usage"] == bases[0]["result"]["usage"], "timed SDK usage differs") - if concurrency == 1: - require(vector_delta(row["scores"], baseline) <= RESTORE_ATOL, "timed C1 base output differs") - row["diagnostic_matching_oracle"] = matching_vector(row["scores"], summary["shape_oracles"]) - measured["actual_timed_batch_shape"] = "unobserved" - evidence.emit( - "timed_output_diagnostics", - concurrency=concurrency, - witnesses=[ - {"label": row["label"], "match": row["diagnostic_matching_oracle"]} - for row in measured["samples"] - ], - ) - evidence.emit("gpu_after_phase", phase=phase, telemetry=telemetry()) + async with IsolatedLoadgen(url, args.source, evidence, smoke=args.smoke) as loadgen: + for concurrency in (1, *CONCURRENCIES): + phase = f"performance-C{concurrency}" + evidence.emit("gpu_before_phase", phase=phase, telemetry=telemetry()) + await loadgen.phase(concurrency) + evidence.emit("gpu_after_phase", phase=phase, telemetry=telemetry()) + # Reaping and all raw-row materialization/validation happen after + # every measured phase, so their allocations never enter the + # serving heap during a subsequent timed phase. + summary["loadgen"] = loadgen.receipt + summary["performance"] = loadgen.results() + for concurrency in (1, *CONCURRENCIES): + measured = summary["performance"][str(concurrency)] + evidence.emit("performance", phase=f"performance-C{concurrency}", **measured) + atomic_json(evidence.directory / "trial.json", summary) + for row in measured["samples"]: + require(row["result"]["usage"] == bases[0]["result"]["usage"], "timed SDK usage differs") + if concurrency == 1: + require(vector_delta(row["scores"], baseline) <= RESTORE_ATOL, "timed C1 base output differs") + row["diagnostic_matching_oracle"] = matching_vector(row["scores"], summary["shape_oracles"]) + measured["actual_timed_batch_shape"] = "unobserved" + evidence.emit( + "timed_output_diagnostics", + concurrency=concurrency, + witnesses=[ + {"label": row["label"], "match": row["diagnostic_matching_oracle"]} for row in measured["samples"] + ], + ) adapter.observation_phase = "lifecycle" if args.arm == "R": previous = set(registry.loaded_lora_names(MODEL)) From 4eb6a0daae909ad7e5c69f59cbd28fd73810dc6b Mon Sep 17 00:00:00 2001 From: svonava Date: Sat, 17 Oct 2026 09:55:36 -0700 Subject: [PATCH 13/16] test: observe cross-encoder GC only at phase boundaries --- tools/benchmarks/cross_encoder_lora/README.md | 53 ++++--- .../cross_encoder_lora/isolation.py | 36 ++--- .../benchmarks/cross_encoder_lora/loadgen.py | 34 +++-- .../cross_encoder_lora/process_metrics.py | 70 ++++++++- .../tests/test_isolation.py | 134 +++++++++++++++--- .../tests/test_process_metrics.py | 74 ++++++++++ 6 files changed, 326 insertions(+), 75 deletions(-) diff --git a/tools/benchmarks/cross_encoder_lora/README.md b/tools/benchmarks/cross_encoder_lora/README.md index 87392d700..e2ae7b911 100644 --- a/tools/benchmarks/cross_encoder_lora/README.md +++ b/tools/benchmarks/cross_encoder_lora/README.md @@ -43,7 +43,7 @@ benchmark only inspects them and fails on a mismatch. ```console mise exec -- uv run --no-sync python -m tools.benchmarks.cross_encoder_lora.runner \ --inputs /tmp/ce-sources --fixtures /tmp/ce-fixtures \ - --output /tmp/ce-evidence --deadline-s 4200 --trial-deadline-s 270 + --output /tmp/ce-evidence --deadline-s 3900 --trial-deadline-s 270 ``` Use a fresh output directory. Set the outer compute-session deadline before @@ -55,8 +55,11 @@ more than its 270-second limit plus 90 seconds for wait, termination and kill grace (45/35/10). The proposed outer limits are 4,500 seconds for the operator and 4,800 seconds from provider start, with a 300-second startup cutoff and an additional absolute ceiling of application creation plus 5,100 seconds. The -operator must reserve export time before starting the runner; these settings -do not themselves authorize or allocate compute. Never change thresholds or +operator reserves 300 seconds for export and requires strictly more than 4,255 +seconds remaining before starting the 3,900-second runner (plus 45 seconds wait +and 10 seconds cleanup). The public runner's 4,200-second ceiling remains +available, but is not this operator preset. These settings do not themselves +authorize or allocate compute. Never change thresholds or drop failed trials to obtain a pass, or retry the same failed protocol unchanged. ## Frozen comparison @@ -215,24 +218,42 @@ created. Cancellation, malformed protocol and child errors fail the trial and trigger bounded termination/reaping. SIGKILL or immediate parent-death exit can lose the interrupted phase; prior completed atomic phase files remain evidence. -Each PID records preallocated numeric GC events and self-duration. The combined -server-plus-client estimated callback cost must be at most **0.1%** of each -measured interval, including a fixed 10-microsecond allowance per intersecting -callback. A bounded local calibration must be retained before hardware use; -it checks the allowance rather than changing it after outcomes. This estimate -cannot prove zero scheduling or GC perturbation. Overflow, missing event pairs, -clock contradictions, callback-list changes or the cost limit fail qualification. -Run the public local calibration once from the frozen checkout and retain its -full JSON, including failures and outliers: +### Boundary-only observation amendment + +The first isolated run at `250bd9a91c8f465aafe69de94aee98aad6ac3298` completed +the first baseline trial's three SDK phases, then failed its prospective +combined GC-callback cost limit. Its C16 estimate was **0.183546%** against +**0.1%**: measured inner callback self-time contributed **0.055258%** and the +fixed 10-microsecond allowance contributed **0.128287%**. That failed run remains +failed; no candidate trials or paired performance gates were qualified. The +allowance was not reduced after observing the result. + +The qualification harness now adds **no per-event GC callbacks** in either +PID; pre-existing callbacks are preserved. Outside-clock snapshots retain normal GC enable state, thresholds, +collection counters and ordered callback identities. The original callback +objects stay referenced through both boundaries, and changes to their identities +or GC configuration fail the phase. These checks cannot detect an intervening +change undone before the next snapshot. No automatic collection is disabled, +and no forced collection is added during timing. + +Actual GC pause intervals and callback self-time/cost are explicitly **unavailable** +(`null`), not measured zero; the absent observer's cost gate is `not_applicable`, +not passed. Boundary snapshots still have overhead outside the request clocks; +this is not a zero-overhead claim. The SDK process separation, workload, +statistical gates, numerical and memory limits, wire checks and process cleanup +remain unchanged. The historical recorder and calibration remain standalone +diagnostics and are not prerequisites for a run that does not install callbacks. +To run that optional diagnostic from a frozen checkout, retain its full JSON, +including failures and outliers: ```console mise exec -- uv run --no-sync python -m tools.benchmarks.cross_encoder_lora.calibrate_gc --output /tmp/ce-gc-calibration.json ``` -It estimates local mean cost, not worst-case event cost or another host's cost. -Short CPU smoke applies integrity checks but does not qualify performance or -apply a meaningful wall-time cost budget. No automatic collection is disabled, -and no forced collection is added during timing. +It estimates local mean callback cost, not worst-case event cost or another +host's cost. Exact replay of the instrumented method requires its recorded +`250bd9a9` checkout. Short CPU smoke checks evidence integrity but does not +qualify performance. CPU/context-switch/cgroup snapshots bracket the actual shared monotonic interval; they include boundary work and are not exact request-level or GPU-only timers. diff --git a/tools/benchmarks/cross_encoder_lora/isolation.py b/tools/benchmarks/cross_encoder_lora/isolation.py index 3524b7270..4cadc6031 100644 --- a/tools/benchmarks/cross_encoder_lora/isolation.py +++ b/tools/benchmarks/cross_encoder_lora/isolation.py @@ -3,6 +3,7 @@ from __future__ import annotations import asyncio +import gc import json import math import os @@ -15,7 +16,12 @@ from tools.benchmarks.cross_encoder_lora.evidence import Evidence, atomic_json, digest from tools.benchmarks.cross_encoder_lora.loadgen import CONTROL_LIMIT, CONTROL_TIMEOUT_S, PHASES -from tools.benchmarks.cross_encoder_lora.process_metrics import MAX_CALLBACK_FRACTION, GCRecorder, bracket, snapshot +from tools.benchmarks.cross_encoder_lora.process_metrics import ( + boundary_gc_report, + bracket, + snapshot, + validate_boundary_gc, +) from tools.benchmarks.cross_encoder_lora.protocol import PERFORMANCE_FUTURE_TIMEOUT_S, require @@ -115,12 +121,11 @@ async def phase(self, concurrency: int) -> dict[str, Any]: len(self.records) < len(PHASES) and concurrency == PHASES[len(self.records)], "duplicate or reordered measured phase", ) - monitor = GCRecorder() observation: dict[str, Any] = {"server_pid": os.getpid(), "client_identity": self.receipt["identity"]} try: await self._send("phase", concurrency) await self._read("ready", concurrency) - monitor.start() + callbacks = tuple(gc.callbacks) observation["before"] = snapshot() await self._send("go", concurrency) measured = await self._read("measured", concurrency, wait_s=PERFORMANCE_FUTURE_TIMEOUT_S) @@ -135,8 +140,7 @@ async def phase(self, concurrency: int) -> dict[str, Any]: ) observation["clock"] = clock observation["after"] = snapshot() - monitor.close() - observation["gc"] = monitor.report(clock["start"], clock["end"]) + observation["gc"] = boundary_gc_report(observation["before"], observation["after"], callbacks) observation["resources"] = bracket(observation["before"], observation["after"], clock) await self._send("observed", concurrency) complete = await self._read("phase_complete", concurrency) @@ -159,7 +163,6 @@ async def phase(self, concurrency: int) -> dict[str, Any]: ) return record finally: - monitor.close() primary_error = sys.exception() try: atomic_json(self.evidence.directory / f"server-C{concurrency}-observations.json", observation) @@ -279,24 +282,11 @@ def _validate_phase(self, record: dict[str, Any], value: dict[str, Any]) -> dict require(client["gc"]["pid"] == identity["pid"], "client GC observation PID differs") require(server["gc"]["pid"] == os.getpid(), "server GC observation PID differs") for observation in (client, server): - gc_report = observation["gc"] - require( - gc_report["window_start_ns"] == round(server["clock"]["start"] * 1e9) - and gc_report["window_end_ns"] == round(server["clock"]["end"] * 1e9), - "GC observation window differs", - ) - require(gc_report["integrity_passed"] is True, "GC observer integrity gate failed") - cost = gc_report["estimated_observer_fraction"] - require(type(cost) in (int, float) and math.isfinite(cost) and cost >= 0, "invalid observer cost") - combined = server["gc"]["estimated_observer_fraction"] + client["gc"]["estimated_observer_fraction"] - # Tiny CPU smoke has no performance qualification. It still must prove - # evidence integrity; the unchanged prospective cost gate is mandatory - # for every real measurement window on the qualifying allocation. - if not self.smoke: - require(combined <= MAX_CALLBACK_FRACTION, "combined observer cost gate failed") + validate_boundary_gc(observation) value["server_observation"] = server - value["combined_estimated_observer_fraction"] = combined - value["observer_cost_gate_applied"] = not self.smoke + value["combined_estimated_observer_fraction"] = None + value["observer_cost_gate_applied"] = False + value["observer_cost_gate_status"] = "not_applicable" return value def results(self) -> dict[str, dict[str, Any]]: diff --git a/tools/benchmarks/cross_encoder_lora/loadgen.py b/tools/benchmarks/cross_encoder_lora/loadgen.py index eb1fdae7a..f1fb26dd8 100644 --- a/tools/benchmarks/cross_encoder_lora/loadgen.py +++ b/tools/benchmarks/cross_encoder_lora/loadgen.py @@ -3,6 +3,7 @@ from __future__ import annotations import argparse +import gc import inspect import json import os @@ -19,7 +20,12 @@ from tools.benchmarks.cross_encoder_lora.client import performance from tools.benchmarks.cross_encoder_lora.evidence import Evidence, atomic_json, digest -from tools.benchmarks.cross_encoder_lora.process_metrics import GCRecorder, bracket, snapshot +from tools.benchmarks.cross_encoder_lora.process_metrics import ( + boundary_gc_report, + bracket, + snapshot, + validate_boundary_gc, +) from tools.benchmarks.cross_encoder_lora.protocol import require CONTROL_LIMIT = 65_536 @@ -133,11 +139,12 @@ def run(args: argparse.Namespace) -> None: for concurrency in PHASES: control.receive("phase", concurrency) require_sdk_only() - monitor = GCRecorder() - monitor.start() + callbacks: tuple[Any, ...] = () observations: dict[str, Any] = {} def before_measure() -> None: + nonlocal callbacks + callbacks = tuple(gc.callbacks) observations["before"] = snapshot() control.send("ready", concurrency) control.receive("go", concurrency) @@ -145,7 +152,7 @@ def before_measure() -> None: def after_measure(clock: dict[str, float]) -> None: observations["clock"] = clock observations["after"] = snapshot() - monitor.close() + observations["gc"] = boundary_gc_report(observations["before"], observations["after"], callbacks) control.send("measured", concurrency, clock=clock) control.receive("observed", concurrency) @@ -159,15 +166,16 @@ def after_measure(clock: dict[str, float]) -> None: after_measure=after_measure, ) finally: - monitor.close() - if "clock" in observations: - observations["gc"] = monitor.report(observations["clock"]["start"], observations["clock"]["end"]) - atomic_json(evidence.directory / f"phase-C{concurrency}-observations.json", observations) - require(observations["gc"]["integrity_passed"], "client GC observation integrity failed") - if not args.smoke: - require( - observations["gc"]["cost_budget_passed"], "client GC observation exceeded its prospective cost gate" - ) + primary_error = sys.exception() + try: + atomic_json(evidence.directory / f"phase-C{concurrency}-observations.json", observations) + except BaseException as write_error: + if primary_error is None: + raise + primary_error.add_note( + f"Secondary client observation write failure: {type(write_error).__name__}: {write_error}" + ) + validate_boundary_gc(observations) observations["resources"] = bracket( observations["before"], observations["after"], result["measurement_clock"] ) diff --git a/tools/benchmarks/cross_encoder_lora/process_metrics.py b/tools/benchmarks/cross_encoder_lora/process_metrics.py index 2cde82443..ee19fe7fb 100644 --- a/tools/benchmarks/cross_encoder_lora/process_metrics.py +++ b/tools/benchmarks/cross_encoder_lora/process_metrics.py @@ -137,6 +137,7 @@ def snapshot() -> dict[str, Any]: "rusage": {key: getattr(usage, key) for key in RUSAGE_FIELDS}, "gc_enabled": gc.isenabled(), "gc_thresholds": list(gc.get_threshold()), + "gc_callback_ids": [id(callback) for callback in gc.callbacks], "gc_count": list(gc.get_count()), "gc_stats": gc.get_stats(), } @@ -244,7 +245,72 @@ def telemetry_delta(before: dict[str, Any], after: dict[str, Any]) -> dict[str, } -# Prospective observation limits, independent of the performance gates. +def _boundary_gc_projection(before: dict[str, Any], after: dict[str, Any]) -> dict[str, Any]: + first, last = before["core"], after["core"] + errors = [] + for core in (first, last): + require( + isinstance(core["gc_callback_ids"], list) + and all(type(value) is int and value > 0 for value in core["gc_callback_ids"]), + "invalid boundary GC callback identities", + ) + require( + len(core["gc_thresholds"]) == GC_GENERATIONS + and all(type(value) is int and value >= 0 for value in core["gc_thresholds"]), + "invalid boundary GC thresholds", + ) + if first["pid"] != last["pid"]: + errors.append("GC boundary process identity changed") + if first["gc_callback_ids"] != last["gc_callback_ids"]: + errors.append("pre-existing GC callback identities changed") + if first["gc_enabled"] is not True or last["gc_enabled"] is not True: + errors.append("ordinary automatic GC must remain enabled") + if first["gc_thresholds"] != last["gc_thresholds"] or first["gc_thresholds"][0] <= 0: + errors.append("GC thresholds changed or disabled automatic collection") + return { + "mode": "boundary-only", + "pid": first["pid"], + "scope": "Boundary state only; intervening changes and GC pauses are unobserved.", + "integrity_passed": not errors, + "errors": errors, + "records": None, + "intervals": None, + "callback_self_ns": None, + "estimated_observer_fraction": None, + "cost_budget_passed": None, + "unavailable_reason": "No per-event GC callbacks are installed by the qualification harness.", + } + + +def boundary_gc_report(before: dict[str, Any], after: dict[str, Any], callbacks: tuple[Any, ...]) -> dict[str, Any]: + """Check outside-clock state without installing callbacks or changing GC. + + Holding the original objects through both snapshots prevents identity reuse. + This cannot observe an intervening change that is undone before the boundary. + """ + result = _boundary_gc_projection(before, after) + if ( + before["core"]["gc_callback_ids"] != [id(callback) for callback in callbacks] + or len(gc.callbacks) != len(callbacks) + or any(current is not prior for current, prior in zip(gc.callbacks, callbacks, strict=False)) + ): + result["errors"].append("retained GC callback identity differs at a boundary") + result["integrity_passed"] = False + return result + + +def validate_boundary_gc(observation: dict[str, Any]) -> None: + """Reject measured-event claims or contradictory raw boundary evidence.""" + expected = _boundary_gc_projection(observation["before"], observation["after"]) + require( + expected["integrity_passed"] is True + and observation["gc"]["integrity_passed"] is True + and observation["gc"] == expected, + "boundary-only GC state or unavailable observation fields differ", + ) + + +# Historical per-event diagnostic limits; qualification does not install this recorder. MAX_CALLBACK_FRACTION = 0.001 CALLBACK_TAIL_ALLOWANCE_NS = 10_000 GC_RECORD_CAPACITY = 65_536 @@ -391,7 +457,7 @@ def report(self, start_s: float, end_s: float) -> dict[str, Any]: def bracket(before: dict[str, Any], after: dict[str, Any], clock: dict[str, float]) -> dict[str, Any]: - """Attribute exact GC intervals separately from enclosing CPU/cgroup deltas.""" + """Keep enclosing GC/CPU/cgroup deltas distinct from request-level timing.""" require(before["core"]["pid"] == after["core"]["pid"], "process identity changed") begin, end = clock["start"], clock["end"] require( diff --git a/tools/benchmarks/cross_encoder_lora/tests/test_isolation.py b/tools/benchmarks/cross_encoder_lora/tests/test_isolation.py index 05fb7311e..1afdbc43b 100644 --- a/tools/benchmarks/cross_encoder_lora/tests/test_isolation.py +++ b/tools/benchmarks/cross_encoder_lora/tests/test_isolation.py @@ -5,6 +5,7 @@ import ast import asyncio import copy +import gc import json import os import subprocess @@ -18,6 +19,7 @@ import pytest from sie_sdk._msgpack import packb, unpackb +from tools.benchmarks.cross_encoder_lora import isolation, loadgen, process_metrics from tools.benchmarks.cross_encoder_lora.evidence import Evidence, atomic_json, digest from tools.benchmarks.cross_encoder_lora.isolation import IsolatedLoadgen from tools.benchmarks.cross_encoder_lora.loadgen import PHASES @@ -67,7 +69,32 @@ def log_message(self, format: str, *args: Any) -> None: @pytest.mark.asyncio -async def test_one_sdk_only_exec_pid_reused_across_all_phases_and_reaped(sdk_server: str, tmp_path: Path) -> None: +async def test_one_sdk_only_exec_pid_reused_across_all_phases_and_reaped( + sdk_server: str, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + def forbidden(*_args: Any, **_kwargs: Any) -> None: + raise AssertionError("qualification must not construct a GCRecorder") + + # Exercise both real qualification paths with the historical recorder + # constructor made fatal. The SDK-only exec still uses its ordinary CLI. + monkeypatch.setattr(isolation, "GCRecorder", forbidden, raising=False) + monkeypatch.setattr(process_metrics, "GCRecorder", forbidden) + original_exec = asyncio.create_subprocess_exec + + async def guarded_exec(*args: Any, **kwargs: Any) -> asyncio.subprocess.Process: + assert args[1:3] == ("-m", "tools.benchmarks.cross_encoder_lora.loadgen") + bootstrap = ( + "from tools.benchmarks.cross_encoder_lora import process_metrics, loadgen\n" + "def forbidden(*args, **kwargs):\n" + " raise AssertionError('qualification must not construct a GCRecorder')\n" + "process_metrics.GCRecorder = forbidden\n" + "loadgen.GCRecorder = forbidden\n" + "loadgen.main()\n" + ) + return await original_exec(args[0], "-c", bootstrap, *args[3:], **kwargs) + + monkeypatch.setattr(asyncio, "create_subprocess_exec", guarded_exec) + callbacks, thresholds = tuple(gc.callbacks), gc.get_threshold() evidence = Evidence(tmp_path / "out") async with IsolatedLoadgen(sdk_server, ROOT, evidence, smoke=True) as owner: assert owner.process is not None @@ -94,8 +121,21 @@ async def test_one_sdk_only_exec_pid_reused_across_all_phases_and_reaped(sdk_ser assert phase["client_observation"]["gc"]["integrity_passed"] assert phase["server_observation"]["gc"]["integrity_passed"] assert phase["measurement_clock"] == phase["server_observation"]["clock"] - assert phase["observer_cost_gate_applied"] is False # CPU smoke never qualifies performance. + assert phase["combined_estimated_observer_fraction"] is None + assert phase["observer_cost_gate_applied"] is False + assert phase["observer_cost_gate_status"] == "not_applicable" + for observation in (phase["client_observation"], phase["server_observation"]): + assert observation["gc"]["mode"] == "boundary-only" + assert observation["gc"]["records"] is None + assert observation["gc"]["intervals"] is None + assert observation["gc"]["callback_self_ns"] is None + assert observation["gc"]["estimated_observer_fraction"] is None + assert observation["gc"]["cost_budget_passed"] is None assert all(len(row["responses"]) == 1 for row in phase["samples"]) + assert len(gc.callbacks) == len(callbacks) + assert all(current is prior for current, prior in zip(gc.callbacks, callbacks, strict=True)) + assert gc.isenabled() + assert gc.get_threshold() == thresholds terminal = json.loads((evidence.directory / "loadgen/terminal.json").read_text()) assert terminal["phases"] == owner.records with pytest.raises(ProcessLookupError): @@ -109,18 +149,13 @@ def synthetic_owner(tmp_path: Path) -> tuple[IsolatedLoadgen, dict[str, Any], di clock = {"start": 1.0, "deadline": 61.0, "end": 61.1} def observation(pid: int) -> dict[str, Any]: - return { - "before": {"core": {"pid": pid}}, - "after": {"core": {"pid": pid}}, + value = { + "before": {"core": {"pid": pid, "gc_enabled": True, "gc_thresholds": [700, 10, 10], "gc_callback_ids": []}}, + "after": {"core": {"pid": pid, "gc_enabled": True, "gc_thresholds": [700, 10, 10], "gc_callback_ids": []}}, "clock": clock, - "gc": { - "pid": pid, - "integrity_passed": True, - "window_start_ns": 1_000_000_000, - "window_end_ns": 61_100_000_000, - "estimated_observer_fraction": 0.0001, - }, } + value["gc"] = process_metrics._boundary_gc_projection(value["before"], value["after"]) + return value owner.observations[1] = observation(os.getpid()) record = {"concurrency": 1} @@ -142,16 +177,21 @@ def observation(pid: int) -> dict[str, Any]: "observation_clock", "identity", "pid", - "window", + "mode", + "callbacks", + "thresholds", + "disabled", "cost_nan", - "combined_budget", + "zero_cost", + "empty_events", + "false_gate", "integrity", ], ) -def test_phase_payload_must_match_observed_ipc_provenance_and_combined_budget(tmp_path: Path, mutation: str) -> None: +def test_phase_payload_must_match_observed_ipc_provenance_and_boundary_state(tmp_path: Path, mutation: str) -> None: owner, record, original = synthetic_owner(tmp_path) value = copy.deepcopy(original) - assert owner._validate_phase(record, copy.deepcopy(original))["observer_cost_gate_applied"] + assert owner._validate_phase(record, copy.deepcopy(original))["observer_cost_gate_applied"] is False if mutation == "swapped_phase": value["concurrency"] = 4 elif mutation == "clock": @@ -162,19 +202,71 @@ def test_phase_payload_must_match_observed_ipc_provenance_and_combined_budget(tm value["identity"]["source"] = "/wrong" elif mutation == "pid": value["client_observation"]["after"]["core"]["pid"] += 1 - elif mutation == "window": - value["client_observation"]["gc"]["window_start_ns"] += 1 + elif mutation == "mode": + value["client_observation"]["gc"]["mode"] = "per-event" + elif mutation == "callbacks": + value["client_observation"]["after"]["core"]["gc_callback_ids"] = [123] + elif mutation == "thresholds": + value["client_observation"]["after"]["core"]["gc_thresholds"][0] += 1 + elif mutation == "disabled": + value["client_observation"]["after"]["core"]["gc_enabled"] = False elif mutation == "cost_nan": value["client_observation"]["gc"]["estimated_observer_fraction"] = float("nan") - elif mutation == "combined_budget": - owner.observations[1]["gc"]["estimated_observer_fraction"] = 0.0006 - value["client_observation"]["gc"]["estimated_observer_fraction"] = 0.0006 + elif mutation == "zero_cost": + value["client_observation"]["gc"]["estimated_observer_fraction"] = 0 + elif mutation == "empty_events": + value["client_observation"]["gc"]["records"] = [] + elif mutation == "false_gate": + value["client_observation"]["gc"]["cost_budget_passed"] = False else: value["client_observation"]["gc"]["integrity_passed"] = False with pytest.raises(AssertionError): owner._validate_phase(record, value) +def test_child_observation_write_failure_cannot_replace_primary_phase_failure( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + import argparse + + class Control: + def __init__(self, _token: str) -> None: + pass + + def send(self, *_args: Any, **_kwargs: Any) -> None: + pass + + def receive(self, *_args: Any) -> None: + pass + + def close(self) -> None: + pass + + def failed_performance(*_args: Any, before_measure: Any, **_kwargs: Any) -> None: + before_measure() + raise ValueError("primary SDK failure") + + original_write = loadgen.atomic_json + + def failed_write(path: Path, value: Any) -> None: + if path.name.endswith("-observations.json"): + raise OSError("observation write failed") + original_write(path, value) + + monkeypatch.setattr(loadgen, "Control", Control) + monkeypatch.setattr(loadgen, "identity", lambda _source: {"pid": os.getpid()}) + monkeypatch.setattr(loadgen, "require_sdk_only", lambda: None) + monkeypatch.setattr(loadgen, "performance", failed_performance) + monkeypatch.setattr(loadgen, "atomic_json", failed_write) + args = argparse.Namespace(output=tmp_path / "child", source=ROOT, token=str(os.getpid()), smoke=True, url="unused") + with pytest.raises(ValueError, match="primary SDK failure") as caught: + loadgen.run(args) + assert any("observation write failed" in note for note in caught.value.__notes__) + terminal = json.loads((args.output / "terminal.json").read_text()) + assert terminal["error"]["type"] == "ValueError" + assert terminal["passed"] is False + + @pytest.mark.asyncio @pytest.mark.parametrize("mutation", ["failed", "identity", "schedule", "smoke"]) async def test_valid_terminal_hash_cannot_mask_wrong_terminal_payload( diff --git a/tools/benchmarks/cross_encoder_lora/tests/test_process_metrics.py b/tools/benchmarks/cross_encoder_lora/tests/test_process_metrics.py index bbbf35f80..0bd0fe6ee 100644 --- a/tools/benchmarks/cross_encoder_lora/tests/test_process_metrics.py +++ b/tools/benchmarks/cross_encoder_lora/tests/test_process_metrics.py @@ -2,6 +2,7 @@ from __future__ import annotations +import copy import gc import time from array import array @@ -13,6 +14,79 @@ from tools.benchmarks.cross_encoder_lora import process_metrics as metrics +def test_boundary_observation_preserves_gc_without_measuring_events() -> None: + callbacks, thresholds = tuple(gc.callbacks), gc.get_threshold() + before = metrics.snapshot() + after = metrics.snapshot() + value = metrics.boundary_gc_report(before, after, callbacks) + metrics.validate_boundary_gc({"before": before, "after": after, "gc": value}) + assert value["mode"] == "boundary-only" + assert value["integrity_passed"] + assert all( + value[key] is None + for key in ("records", "intervals", "callback_self_ns", "estimated_observer_fraction", "cost_budget_passed") + ) + assert "unobserved" in value["scope"] + assert before["core"]["gc_callback_ids"] == after["core"]["gc_callback_ids"] == [id(item) for item in callbacks] + assert len(gc.callbacks) == len(callbacks) + assert all(current is prior for current, prior in zip(gc.callbacks, callbacks, strict=True)) + assert gc.isenabled() + assert gc.get_threshold() == thresholds + + +def test_boundary_callback_identity_check_does_not_use_callable_equality() -> None: + class Permissive: + def __call__(self, _phase: str, _info: dict[str, Any]) -> None: + pass + + def __eq__(self, _other: object) -> bool: + return True + + def __hash__(self) -> int: + return 0 + + original, replacement = Permissive(), Permissive() + previous = tuple(gc.callbacks) + gc.callbacks.append(original) + callbacks = tuple(gc.callbacks) + try: + before = metrics.snapshot() + gc.callbacks[-1] = replacement + after = metrics.snapshot() + value = metrics.boundary_gc_report(before, after, callbacks) + assert not value["integrity_passed"] + with pytest.raises(AssertionError, match="boundary-only"): + metrics.validate_boundary_gc({"before": before, "after": after, "gc": value}) + # Even forged matching snapshot IDs cannot hide a changed live object + # from the local retained-reference check. + after["core"]["gc_callback_ids"] = before["core"]["gc_callback_ids"] + assert not metrics.boundary_gc_report(before, after, callbacks)["integrity_passed"] + finally: + gc.callbacks[:] = previous + + +def test_boundary_threshold_mutation_and_disabled_gc_fail_without_restoring_them_silently() -> None: + thresholds = gc.get_threshold() + callbacks = tuple(gc.callbacks) + before = metrics.snapshot() + try: + gc.set_threshold(thresholds[0] + 1, *thresholds[1:]) + after = metrics.snapshot() + value = metrics.boundary_gc_report(before, after, callbacks) + assert not value["integrity_passed"] + assert gc.get_threshold()[0] == thresholds[0] + 1 + gc.disable() + with pytest.raises(AssertionError, match="GC state"): + metrics.snapshot() + after = copy.deepcopy(before) + after["core"]["gc_enabled"] = False + assert not metrics.boundary_gc_report(before, after, callbacks)["integrity_passed"] + assert not gc.isenabled() + finally: + gc.set_threshold(*thresholds) + gc.enable() + + def report(rows: list[list[int]], start: float = 1.0, end: float = 2.0) -> dict[str, Any]: recorder = metrics.GCRecorder(capacity=max(1, len(rows))) recorder.start() From e90614482952db3c351bcb75508d17b2ed721ebf Mon Sep 17 00:00:00 2001 From: svonava Date: Sat, 17 Oct 2026 12:20:58 -0700 Subject: [PATCH 14/16] test(benchmarks): add bounded interleaved baseline pilot --- tools/benchmarks/cross_encoder_lora/README.md | 63 ++ .../cross_encoder_lora/group_control.py | 252 ++++++++ .../cross_encoder_lora/interleaved.py | 560 ++++++++++++++++++ .../cross_encoder_lora/isolation.py | 28 +- .../benchmarks/cross_encoder_lora/loadgen.py | 28 +- .../tests/test_group_control.py | 361 +++++++++++ .../tests/test_interleaved.py | 355 +++++++++++ tools/benchmarks/cross_encoder_lora/worker.py | 68 ++- 8 files changed, 1701 insertions(+), 14 deletions(-) create mode 100644 tools/benchmarks/cross_encoder_lora/group_control.py create mode 100644 tools/benchmarks/cross_encoder_lora/interleaved.py create mode 100644 tools/benchmarks/cross_encoder_lora/tests/test_group_control.py create mode 100644 tools/benchmarks/cross_encoder_lora/tests/test_interleaved.py diff --git a/tools/benchmarks/cross_encoder_lora/README.md b/tools/benchmarks/cross_encoder_lora/README.md index e2ae7b911..dac2ebff3 100644 --- a/tools/benchmarks/cross_encoder_lora/README.md +++ b/tools/benchmarks/cross_encoder_lora/README.md @@ -272,6 +272,69 @@ server boundary snapshots. Keep those files in the full durable archive. ## Evidence and CPU checks +### Whole-phase triplet feasibility mode + +The separate `interleaved` entry point holds three fresh server/SDK pairs resident +for one triplet. It prepares every pair before timing, then runs C1 for all three +roles, C4 for all three, and C16 for all three, using the six existing balanced +role orders. Each arm keeps its contiguous 2,000-request C1 phase, 60-second +C4/C16 admission windows, 30 warmups, drain, and original numerical, memory and +12 statistical limits. The sequential `runner` entry point remains available. + +The `baseline-pilot` mode maps **every** logical B/N/R role to the same archived +baseline source and actual B configuration, with no adapter declarations. It +retains the peer numerical and memory checks. Its schema-3 summary reports +`completed_valid` separately from `qualification_passed`, which is always false +for this descriptive mode. A successful diagnostic exit does not qualify a +candidate or authorize a subsequent run. The separately selected `qualification` +mode requires the original B/N/R configurations and source mapping; it cannot +silently substitute baseline-only R. Pilot and candidate observations must not +be pooled or appended until a gate passes. + +From a clean frozen checkout, after the ordinary immutable source and fixture +preparation above, the prospective baseline-only invocation is: + +```console +mise exec -- uv run --no-sync python -m tools.benchmarks.cross_encoder_lora.interleaved \ + --inputs /tmp/ce-inputs --fixtures /tmp/ce-fixtures --output /tmp/ce-baseline-pilot \ + --mode baseline-pilot --deadline-s 3900 --group-work-s 600 +``` + +These are bounds, not a promise that six groups fit. Strict group admission +reserves 600 seconds of work plus one shared 90-second cleanup interval. Each +worker also has a cumulative 270-second grant-to-ACK active budget, with a +120-second cap for each phase; explicit idle waits consume the group budget. +The smaller group cap can stop work before three full active budgets are spent. +The enclosing prospective operator keeps its 300-second archive reserve within +4,500 seconds, provider lifetime at most 4,800 seconds, and outer bound 5,100 +seconds. This entry point does not allocate resources or extend those bounds. + +A dedicated socket carries immutable mode, role, actual configuration, source, +group deadline and lease identities; stdout/stderr remain ordinary logs. One +lease covers warmup, measurement, complete drain, atomic phase persistence and +an outside-timer CUDA synchronization with no pending/in-flight model requests. +Inactive workers wait on events, without a benchmark polling loop. All nine +phase ACKs precede any finish grant, coordinator/server result materialization, lifecycle +churn, unload or archive work. Pre-existing server housekeeping and three idle +CUDA contexts remain part of this changed resource condition; single-active +inference does not establish their neutrality. + +The coordinator owns each server process group; its SDK child inherits that +group. The dedicated EOF monitor starts before heavy server imports, explicitly +cancels the experiment even under uvicorn's signal handler, and has a bounded +group-kill fallback. Group mode also watches unexpected idle SDK exit. Shared +cleanup signals every owned group before waiting and joins teardown through +repeated cancellation. A hard kill may prevent the interrupted phase from +being finalized; prior atomic files remain evidence, not a reconstructed pass. + +This prospective method follows the failed complete `4eb6a0da` sequential +experiment. A separately retained CPU mechanics experiment verified scheduling, +cleanup and sensitivity to an injected arm delay, but only 6/12 baseline gates +passed under either schedule and 0/12 passed with the planted common drift. +Those controls do not demonstrate improved precision, explain the earlier +hardware result, or qualify L4 performance. Six triplets remain the statistical +units; shared-device observations are not independent hardware replications. + `summary.json` schema 2 retains trial outcomes, aggregate metrics, statistical gates and source hashes. Each entry links its complete `trial.json` by relative path and SHA-256; raw request arrays and oracle corpora stay in those per-trial diff --git a/tools/benchmarks/cross_encoder_lora/group_control.py b/tools/benchmarks/cross_encoder_lora/group_control.py new file mode 100644 index 000000000..d598e8f10 --- /dev/null +++ b/tools/benchmarks/cross_encoder_lora/group_control.py @@ -0,0 +1,252 @@ +"""Dedicated group control and parent-death ownership, outside request timing.""" + +from __future__ import annotations + +import argparse +import asyncio +import json +import math +import os +import queue +import select +import signal +import socket +import sys +import threading +import time +from pathlib import Path +from typing import Any + +from tools.benchmarks.cross_encoder_lora.evidence import atomic_json, digest, fingerprint +from tools.benchmarks.cross_encoder_lora.protocol import BASE_REVISION, MAX_TRIAL_SECONDS, ORDERS, require + +LIMIT = 65_536 +GROUP_WORK_S = 600 +GROUP_CLEANUP_S = 90 +PHASE_LIMIT_S = 120 +RUNNER_LIMIT_S = 3900 +STAGES = ("prepare", "C1", "C4", "C16", "finish") +MODES = ("baseline-pilot", "qualification") +NONCE_LENGTH = 32 + + +def selection(mode: str, role: str) -> tuple[str, str]: + require(mode in MODES and role in "BNR" and len(role) == 1, "invalid group mode/role") + return ("B", "B") if mode == "baseline-pilot" or role == "B" else (role, "candidate") + + +def validate_binding(binding: dict[str, Any]) -> None: + require(binding["schema"] == 1 and binding["schedule"] == "whole-phase", "invalid group binding schema") + arm, archive = selection(binding["mode"], binding["logical_role"]) + require((binding["actual_arm"], binding["source_key"]) == (arm, archive), "role/config/source mismatch") + require(type(binding["pair"]) is int and 0 <= binding["pair"] < len(ORDERS), "invalid group pair") + require(isinstance(binding["nonce"], str) and len(binding["nonce"]) == NONCE_LENGTH, "invalid group nonce") + require(type(binding["coordinator_pid"]) is int and binding["coordinator_pid"] > 0, "invalid coordinator PID") + for name in ("work_deadline", "hard_deadline", "group_started"): + require(type(binding[name]) in (float, int) and math.isfinite(binding[name]), "invalid group deadline") + require( + 0 < binding["work_deadline"] - binding["group_started"] <= GROUP_WORK_S + and 0 < binding["hard_deadline"] - binding["work_deadline"] <= GROUP_CLEANUP_S, + "group deadlines exceed fixed bounds", + ) + require(binding["active_limit_s"] == MAX_TRIAL_SECONDS, "active budget differs") + require(binding["phase_limit_s"] == PHASE_LIMIT_S, "phase budget differs") + if archive == "B": + require(binding["source_commit"] == BASE_REVISION, "baseline commit differs") + + +def validate_source(binding: dict[str, Any], source: Path, arm: str, pair: int) -> None: + """Called under the prepare lease, before preflight/model initialization.""" + validate_binding(binding) + require(binding["actual_arm"] == arm and binding["pair"] == pair, "worker arguments differ from binding") + require(str(source.resolve()) == binding["source_root"], "worker source path differs") + manifest_path = Path(binding["source_manifest"]) + require(digest(manifest_path) == binding["source_manifest_sha256"], "source manifest changed") + manifest = json.loads(manifest_path.read_text()) + require(manifest["schema"] == 1 and manifest["dependency_lock_identical"], "source manifest mode differs") + record = manifest["sources"][binding["source_key"]] + require(record["commit"] == binding["source_commit"], "source commit differs") + require(record["sha256"] == binding["archive_sha256"], "source archive binding differs") + archive = manifest_path.parent / record["archive"] + require(archive.resolve().is_relative_to(manifest_path.parent.resolve()), "source archive escapes inputs") + require(digest(archive) == record["sha256"], "source archive changed") + for name, expected in binding["source_files"].items(): + path = source / name + require(path.resolve().is_relative_to(source.resolve()) and digest(path) == expected, "source file changed") + + +def envelope(binding: dict[str, Any], **fields: Any) -> dict[str, Any]: + return {"binding_sha256": fingerprint(binding), **fields} + + +def decode(line: bytes, binding: dict[str, Any]) -> dict[str, Any]: + require(bool(line) and len(line) <= LIMIT and line.endswith(b"\n"), "missing/oversized group message") + value = json.loads(line) + require(isinstance(value, dict) and value.get("binding_sha256") == fingerprint(binding), "group identity differs") + return value + + +class Member: + """One raw-fd monitor exists before heavy worker imports or uvicorn handlers. + + Its only writer is the coordinator. EOF cancels the asyncio experiment; + if teardown or import is stuck, one hard deadline kills this worker's own + process group (including its SDK child). No model work occurs in this thread. + """ + + def __init__(self, fd: int, binding: dict[str, Any], output: Path) -> None: + validate_binding(binding) + require(os.getpgrp() == os.getpid(), "group worker must own its process group") + require(os.getppid() == binding["coordinator_pid"], "coordinator lineage differs") + self.binding, self.output = binding, output + self.socket = socket.socket(fileno=fd) + self.socket.set_inheritable(False) + self.socket.settimeout(5) + self.messages: queue.Queue[dict[str, Any]] = queue.Queue(maxsize=2) + self.stopped = threading.Event() + self.wake_read, self.wake_write = os.pipe() + self.available: asyncio.Event | None = None + self.failure: str | None = None + self.loop: asyncio.AbstractEventLoop | None = None + self.owner: asyncio.Task | None = None + self.index = 0 + self.current: dict[str, Any] | None = None + self.history: list[dict[str, Any]] = [] + self.reader = threading.Thread(target=self._monitor, name="group-coordinator", daemon=True) + self.reader.start() + self.send(event="hello", pid=os.getpid(), ppid=os.getppid(), process_group=os.getpgrp()) + + def _fail(self, reason: str) -> None: + self.failure = reason + if self.loop is not None and self.owner is not None: + try: + self.loop.call_soon_threadsafe(self.owner.cancel, reason) + except RuntimeError: + # A loop that already closed must not disable the raw-fd + # monitor's independent hard group kill. + pass + + def _monitor(self) -> None: + pending = bytearray() + deadline = self.binding["hard_deadline"] + try: + while not self.stopped.is_set(): + remaining = deadline - time.monotonic() + if remaining <= 0: + break + ready, _, _ = select.select([self.socket, self.wake_read], [], [], remaining) + if self.wake_read in ready: + return + if not ready: + continue + data = self.socket.recv(4096) + require(bool(data), "coordinator EOF") + pending.extend(data) + while b"\n" in pending: + end = pending.index(b"\n") + 1 + line = bytes(pending[:end]) + del pending[:end] + self.messages.put_nowait(decode(line, self.binding)) + if self.loop is not None and self.available is not None: + self.loop.call_soon_threadsafe(self.available.set) + require(len(pending) <= LIMIT, "unterminated group message") + except BaseException as error: # noqa: BLE001 -- losing lifetime monitoring must fail closed + if not self.stopped.is_set(): + self._fail(f"{type(error).__name__}: {error}") + deadline = min(deadline, time.monotonic() + GROUP_CLEANUP_S) + if not self.stopped.is_set(): + self._fail(self.failure or "group hard deadline") + if not self.stopped.wait(max(0, deadline - time.monotonic())): + os.killpg(os.getpid(), signal.SIGKILL) + + def bind_task(self) -> None: + self.loop = asyncio.get_running_loop() + self.owner = asyncio.current_task() + self.available = asyncio.Event() + if not self.messages.empty(): + self.available.set() + require(self.failure is None, self.failure or "coordinator lost during imports") + + def send(self, **fields: Any) -> None: + line = (json.dumps(envelope(self.binding, **fields), allow_nan=False) + "\n").encode() + require(len(line) <= LIMIT, "group output too large") + self.socket.sendall(line) + + async def grant(self, stage: str) -> None: + require( + self.current is None and self.index < len(STAGES) and stage == STAGES[self.index], "group stage reordered" + ) + assert self.available is not None + while self.messages.empty(): + require(self.failure is None, self.failure or "group monitor failed") + remaining = self.binding["work_deadline"] - time.monotonic() + require(remaining > 0, "group idle deadline") + self.available.clear() + if self.messages.empty(): + await asyncio.wait_for(self.available.wait(), remaining) + value = self.messages.get_nowait() + require(value.get("command") == "grant" and value.get("stage") == stage, "unexpected group grant") + require(value.get("ordinal") == self.index, "stale group grant") + require(isinstance(value.get("lease"), str) and len(value["lease"]) == NONCE_LENGTH, "invalid group lease") + require(time.monotonic() < self.binding["work_deadline"], "group work deadline") + self.current = value + + def acknowledge(self, **fields: Any) -> None: + require(self.current is not None, "group ACK without active lease") + assert self.current is not None + event = { + "event": "ack", + "stage": self.current["stage"], + "ordinal": self.index, + "lease": self.current["lease"], + "monotonic_s": time.monotonic(), + **fields, + } + self.history.append(event) + atomic_json(self.output / "group-control.json", {"binding": self.binding, "events": self.history}) + self.send(**event) + self.current = None + self.index += 1 + + def close(self) -> None: + self.stopped.set() + os.write(self.wake_write, b"x") + self.reader.join(timeout=2) + require(not self.reader.is_alive(), "group lifetime monitor did not stop") + self.socket.close() + os.close(self.wake_read) + os.close(self.wake_write) + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--binding", type=Path, required=True) + parser.add_argument("--control-fd", type=int, required=True) + own, remainder = parser.parse_known_args() + binding = json.loads(own.binding.read_text()) + member = Member(own.control_fd, binding, Path(binding["output"])) + try: + # Optional serving dependencies are imported only after the lifetime + # monitor is live. Ordinary worker.py invocations do not use this path. + from tools.benchmarks.cross_encoder_lora import worker # noqa: PLC0415 + + args = worker.arguments(remainder) + args.group = member + + async def launch() -> None: + member.bind_task() + await worker.run(args) + + asyncio.run(launch()) + finally: + primary = sys.exception() + try: + member.close() + except BaseException as error: + if primary is None: + raise + primary.add_note(f"Group monitor close failed: {type(error).__name__}: {error}") + + +if __name__ == "__main__": + main() diff --git a/tools/benchmarks/cross_encoder_lora/interleaved.py b/tools/benchmarks/cross_encoder_lora/interleaved.py new file mode 100644 index 000000000..422726b32 --- /dev/null +++ b/tools/benchmarks/cross_encoder_lora/interleaved.py @@ -0,0 +1,560 @@ +"""Six fresh resident triplets, one whole-phase lease at a time; no provisioning.""" + +from __future__ import annotations + +import argparse +import asyncio +import json +import os +import signal +import socket +import sys +import time +import uuid +from pathlib import Path +from types import FunctionType +from typing import Any + +from tools.benchmarks.cross_encoder_lora.evidence import atomic_json, digest +from tools.benchmarks.cross_encoder_lora.group_control import ( + GROUP_CLEANUP_S, + GROUP_WORK_S, + LIMIT, + MODES, + PHASE_LIMIT_S, + RUNNER_LIMIT_S, + STAGES, + decode, + envelope, + selection, + validate_binding, +) +from tools.benchmarks.cross_encoder_lora.protocol import MAX_TRIAL_SECONDS, ORDERS, compare, require +from tools.benchmarks.cross_encoder_lora.runner import ( + cross_checks, + extract, + harness_manifest, + load_trial, + source_environment, + trial_summary, +) + + +def admission(now: float, deadline: float, work_s: float = GROUP_WORK_S) -> dict[str, Any]: + require(0 < work_s <= GROUP_WORK_S, "group work cap differs") + margin = deadline - (now + work_s + GROUP_CLEANUP_S) + return { + "monotonic_s": now, + "deadline_monotonic_s": deadline, + "reserve_s": work_s + GROUP_CLEANUP_S, + "margin_s": margin, + "admitted": margin > 0, + } + + +def binding_for( + args: argparse.Namespace, + manifest: dict[str, Any], + pair: int, + role: str, + nonce: str, + started: float, +) -> dict[str, Any]: + arm, source_key = selection(args.mode, role) + source = (args.output / "sources" / source_key).resolve() + record = manifest["sources"][source_key] + value = { + "schema": 1, + "schedule": "whole-phase", + "mode": args.mode, + "pair": pair, + "logical_role": role, + "actual_arm": arm, + "source_key": source_key, + "source_commit": record["commit"], + "archive_sha256": record["sha256"], + "source_root": str(source), + "source_manifest": str((args.inputs / "manifest.json").resolve()), + "source_manifest_sha256": digest(args.inputs / "manifest.json"), + "source_files": { + str(path.relative_to(source)): digest(path) + for path in sorted(source.rglob("*")) + if path.is_file() and "__pycache__" not in path.parts and path.suffix != ".pyc" + }, + "nonce": nonce, + "coordinator_pid": os.getpid(), + "group_started": started, + "work_deadline": started + args.group_work_s, + "hard_deadline": started + args.group_work_s + GROUP_CLEANUP_S, + "active_limit_s": MAX_TRIAL_SECONDS, + "phase_limit_s": PHASE_LIMIT_S, + "output": str((args.output / f"trial-{pair}-{role}").resolve()), + } + validate_binding(value) + return value + + +class Peer: + def __init__(self, binding: dict[str, Any], log: Any) -> None: + self.binding, self.log = binding, log + self.process: asyncio.subprocess.Process | None = None + self.reader: asyncio.StreamReader | None = None + self.writer: asyncio.StreamWriter | None = None + self.active_s = 0.0 + self.ordinal = 0 + self.events: list[dict[str, Any]] = [] + self.hello: dict[str, Any] | None = None + self.finishing = False + self.stopping = False + + async def start(self, args: argparse.Namespace, binding_path: Path) -> None: + parent, child = socket.socketpair() + try: + command = [ + sys.executable, + "-m", + "tools.benchmarks.cross_encoder_lora.group_control", + "--binding", + str(binding_path), + "--control-fd", + str(child.fileno()), + "--source", + self.binding["source_root"], + "--fixtures", + str(args.fixtures.resolve()), + "--output", + self.binding["output"], + "--arm", + self.binding["actual_arm"], + "--pair", + str(self.binding["pair"]), + ] + if args.snapshot: + command.extend(["--snapshot", str(args.snapshot.resolve())]) + if args.smoke: + command.append("--smoke") + self.process = await asyncio.create_subprocess_exec( + *command, + cwd=self.binding["source_root"], + env=source_environment(Path(self.binding["source_root"])), + stdout=self.log, + stderr=asyncio.subprocess.STDOUT, + pass_fds=(child.fileno(),), + close_fds=True, + start_new_session=True, + ) + self.reader, self.writer = await asyncio.open_connection(sock=parent, limit=LIMIT) + except BaseException: + parent.close() + raise + finally: + child.close() + + async def read(self, deadline: float) -> dict[str, Any]: + assert self.reader is not None + remaining = deadline - time.monotonic() + require(remaining > 0, "group control deadline") + line = await asyncio.wait_for(self.reader.readline(), remaining) + return decode(line, self.binding) + + async def receive_hello(self) -> None: + value = await self.read(self.binding["work_deadline"]) + assert self.process is not None + require( + value.get("event") == "hello" + and value.get("pid") == self.process.pid + and value.get("ppid") == os.getpid() + and value.get("process_group") == self.process.pid, + "worker hello ownership differs", + ) + self.hello = value + + async def lease(self, stage: str) -> dict[str, Any]: + require(self.ordinal < len(STAGES) and stage == STAGES[self.ordinal], "coordinator stage reordered") + assert self.writer is not None + lease = uuid.uuid4().hex + started = time.monotonic() + remaining = MAX_TRIAL_SECONDS - self.active_s + limit = min(remaining, PHASE_LIMIT_S) if stage.startswith("C") else remaining + deadline = min(self.binding["work_deadline"], started + limit) + require(deadline > started, "worker cumulative active budget exhausted") + request = envelope(self.binding, command="grant", stage=stage, ordinal=self.ordinal, lease=lease) + self.writer.write((json.dumps(request, allow_nan=False) + "\n").encode()) + await asyncio.wait_for(self.writer.drain(), min(5, deadline - time.monotonic())) + event = { + "stage": stage, + "ordinal": self.ordinal, + "lease": lease, + "started": started, + "deadline": deadline, + "active_before_s": self.active_s, + } + self.events.append(event) + self.finishing = stage == "finish" + try: + value = await self.read(deadline) + require( + value.get("event") == "ack" + and value.get("stage") == stage + and value.get("ordinal") == self.ordinal + and value.get("lease") == lease, + "stale/reordered group ACK", + ) + require(started <= value["monotonic_s"] <= time.monotonic(), "worker ACK clock differs") + if stage == "prepare": + require(value["actual_arm"] == self.binding["actual_arm"], "prepared configuration differs") + require( + value["manager_present"] is (self.binding["actual_arm"] == "R"), "prepared LoRA manager differs" + ) + sdk = value["sdk_identity"] + assert self.process is not None + require( + sdk["ppid"] == self.process.pid and sdk["process_group"] == self.process.pid, "SDK lineage differs" + ) + require( + sdk["source"] == self.binding["source_root"] and not sdk["forbidden_imports_present"], + "SDK source differs", + ) + config_path = Path(self.binding["output"]) / "models/model.yaml" + require(digest(config_path) == value["config_sha256"], "prepared config hash differs") + if stage != "finish": + require(value["quiet"]["pending"] == 0 and value["quiet"]["in_flight"] == 0, "worker ACK was not quiet") + if stage.startswith("C"): + concurrency = int(stage[1:]) + record = value["record"] + require( + record["concurrency"] == concurrency and record["path"] == f"phase-C{concurrency}.json", + "ACK phase differs", + ) + path = Path(self.binding["output"]) / "loadgen" / record["path"] + # The worker verified the full hash before ACK. Keep only + # metadata here; even the coordinator's redundant full-file + # hash read waits until all nine timers have ended. + require( + path.stat().st_size == record["bytes"], + "phase persistence differs", + ) + event["ack"] = value + self.ordinal += 1 + return value + finally: + event["ended"] = time.monotonic() + self.active_s += event["ended"] - started + event["active_total_s"] = self.active_s + if self.active_s > MAX_TRIAL_SECONDS: + error = sys.exception() + if error is None: + raise AssertionError("worker cumulative active budget exceeded") + error.add_note("Worker cumulative active budget also exceeded") + + +async def cleanup(peers: list[Peer], hard_deadline: float) -> list[dict[str, Any]]: + """One shared deadline, all signals first; no per-child multiplication.""" + deadline = min(hard_deadline, time.monotonic() + GROUP_CLEANUP_S) + for peer in peers: + peer.stopping = True + if peer.writer is not None: + peer.writer.close() # sole parent endpoint; explicit worker EOF cancellation + if peer.process is not None: + try: + os.killpg(peer.process.pid, signal.SIGTERM) + except ProcessLookupError: + pass + processes = [peer.process for peer in peers if peer.process is not None] + waits = [asyncio.create_task(process.wait()) for process in processes] + if waits: + _, pending = await asyncio.wait(waits, timeout=max(0, min(35, deadline - time.monotonic() - 5))) + for process in processes: + # A worker may have exited while a descendant remains. Signal its + # entire owned group, not only PIDs whose returncode is still None. + try: + os.killpg(process.pid, signal.SIGKILL) + except ProcessLookupError: + pass + if pending: + _, pending = await asyncio.wait(pending, timeout=max(0, deadline - time.monotonic())) + require(not pending, "owned worker reap exceeded shared cleanup deadline") + receipt = [] + for peer in peers: + if peer.process is not None: + receipt.append( + { + "role": peer.binding["logical_role"], + "pid": peer.process.pid, + "returncode": peer.process.returncode, + "reaped": peer.process.returncode is not None, + } + ) + peer.log.close() + return receipt + + +async def watch_peer(peer: Peer, owner: asyncio.Task) -> None: + assert peer.process is not None + await peer.process.wait() + if not peer.finishing and not peer.stopping: + owner.cancel(f"worker {peer.binding['logical_role']} exited before group finish") + + +def verify_phase_files(peers: list[Peer]) -> None: + require( + len(peers) == len(ORDERS[0]) and all(peer.ordinal == len(STAGES) - 1 for peer in peers), + "group timers incomplete", + ) + for peer in peers: + for event in peer.events[1:]: + record = event["ack"]["record"] + path = Path(peer.binding["output"]) / "loadgen" / record["path"] + require( + path.stat().st_size == record["bytes"] and digest(path) == record["sha256"], "phase persistence differs" + ) + + +async def join_cleanup(task: asyncio.Task, primary: BaseException | None) -> Any: + """Repeated coordinator TERM/cancellation cannot abandon owned teardown. + + cleanup itself owns the one absolute deadline. This join does not reset it + and does not cancel the shared teardown task when its caller is cancelled. + """ + cancelled = None + while not task.done(): + try: + await asyncio.shield(task) + except asyncio.CancelledError as error: + cancelled = error + except BaseException: # noqa: BLE001 -- retrieve the same task exception below + break # retrieve the completed task's original exception below + result = task.result() + if cancelled is not None: + if primary is None: + raise cancelled + primary.add_note("Coordinator cancellation during cleanup retained after owned reaping") + return result + + +async def run_group(args: argparse.Namespace, manifest: dict[str, Any], pair: int) -> dict[str, Any]: + started = time.monotonic() + hard_deadline = started + args.group_work_s + GROUP_CLEANUP_S + group: dict[str, Any] = { + "pair": pair, + "order": list(ORDERS[pair]), + "started": started, + "work_deadline": started + args.group_work_s, + "hard_deadline": hard_deadline, + "completed_valid": False, + "leases": [], + "trials": [], + } + peers: list[Peer] = [] + watchers: list[asyncio.Task] = [] + owner = asyncio.current_task() + assert owner is not None + nonce = uuid.uuid4().hex + try: + for role in ORDERS[pair]: + binding = binding_for(args, manifest, pair, role, nonce, started) + path = args.output / f"binding-{pair}-{role}.json" + atomic_json(path, binding) + peer = Peer(binding, (args.output / f"trial-{pair}-{role}.log").open("wb")) + peers.append(peer) + await peer.start(args, path) + watchers.append(asyncio.create_task(watch_peer(peer, owner))) + for peer in peers: + await peer.receive_hello() + for stage in STAGES: + # Phase-major scheduling, and finalization only after all nine + # persisted phase ACKs. No background task grants another lease. + if stage == "finish": + verify_phase_files(peers) + group["phase_hashes_verified_at"] = time.monotonic() + for peer in peers: + await peer.lease(stage) + group["leases"].append({"role": peer.binding["logical_role"], **peer.events[-1]}) + atomic_json(args.output / f"group-{pair}.json", group) + if stage == "finish": + assert peer.process is not None + await asyncio.wait_for(peer.process.wait(), max(0.001, min(10, hard_deadline - time.monotonic()))) + require(peer.process.returncode == 0, "finalized worker exited unsuccessfully") + # Reopen raw trial payloads only after every timer, every finish grant, + # SDK reap and server lifecycle check in the triplet is complete. + for peer in peers: + path = Path(peer.binding["output"]) / "trial.json" + require(digest(path) == peer.events[-1]["ack"]["terminal_sha256"], "worker terminal hash differs") + trial = json.loads(path.read_text()) + require( + trial.get("group") == peer.binding and trial.get("actual_arm") == peer.binding["actual_arm"], + "terminal binding differs", + ) + require(trial["pair"] == pair and trial["arm"] == peer.binding["logical_role"], "terminal role differs") + require(trial["passed"] and trial["shutdown_complete"], "worker correctness/lifecycle failed") + trial.update( + source_commit=peer.binding["source_commit"], + process_returncode=0, + terminal_record=str(path.relative_to(args.output)), + terminal_record_sha256=digest(path), + ) + group["trials"].append(trial_summary(trial)) + group["completed_valid"] = True + except BaseException as error: + group["error"] = {"type": type(error).__name__, "message": str(error)} + raise + finally: + primary = sys.exception() + task = asyncio.create_task(cleanup(peers, hard_deadline)) + try: + group["cleanup"] = await join_cleanup(task, primary) + except BaseException as error: + group["completed_valid"] = False + group["cleanup_error"] = {"type": type(error).__name__, "message": str(error)} + if primary is None: + raise + primary.add_note(f"Group cleanup failed: {type(error).__name__}: {error}") + finally: + for watcher in watchers: + watcher.cancel() + await asyncio.gather(*watchers, return_exceptions=True) + group["peer_events"] = {peer.binding["logical_role"]: peer.events for peer in peers} + group["finished"] = time.monotonic() + try: + atomic_json(args.output / f"group-{pair}.json", group) + except BaseException as error: + if primary is None: + raise + primary.add_note(f"Group receipt failed: {type(error).__name__}: {error}") + return group + + +def validate_trials(trials: list[dict[str, Any]], mode: str, *, smoke: bool) -> None: + schedule = ( + [(0, role) for role in ORDERS[0]] if smoke else [(i, role) for i, order in enumerate(ORDERS) for role in order] + ) + require([(t["pair"], t["arm"]) for t in trials] == schedule, "group trial schedule incomplete") + for trial in trials: + binding = trial["group"] + validate_binding(binding) + require( + binding["mode"] == mode and binding["logical_role"] == trial["arm"] and binding["pair"] == trial["pair"], + "trial group binding differs", + ) + require(trial["actual_arm"] == selection(mode, trial["arm"])[0], "actual trial configuration differs") + require(trial["manager_present"] is (trial["actual_arm"] == "R"), "trial manager differs") + require(trial["passed"] and trial["shutdown_complete"], "group trial failed") + require( + trial["environment"]["qualification"] is (mode == "qualification" and not smoke), + "trial qualification mode differs", + ) + + +async def run(args: argparse.Namespace) -> dict[str, Any]: + require(args.mode in MODES and 0 < args.deadline_s <= RUNNER_LIMIT_S, "invalid interleaved mode/deadline") + require(0 < args.group_work_s <= GROUP_WORK_S, "invalid group work deadline") + args.output = args.output.resolve() + args.output.mkdir(parents=True, exist_ok=False) + started = time.monotonic() + deadline = started + args.deadline_s + summary: dict[str, Any] = { + "schema": 3, + "mode": args.mode, + "schedule": "whole-phase", + "smoke": args.smoke, + "completed_valid": False, + "qualification_passed": False, + "passed": False, + "orders": ORDERS, + "groups": [], + "trials": [], + "admissions": [], + "harness_manifest": harness_manifest(), + "runner_clock": { + "monotonic_start_s": started, + "wall_start_ns": time.time_ns(), + "deadline_monotonic_s": deadline, + "budget_s": args.deadline_s, + }, + } + try: + manifest = extract(args.inputs.resolve(), args.output / "sources") + summary["source_manifest"] = manifest + if args.environment_receipt is not None: + summary["environment_preparation"] = { + "sha256": digest(args.environment_receipt), + "receipt": json.loads(args.environment_receipt.read_text()), + } + for pair in range(1 if args.smoke else len(ORDERS)): + check = {"pair": pair, **admission(time.monotonic(), deadline, args.group_work_s)} + summary["admissions"].append(check) + require(check["admitted"], "insufficient time for complete bounded group and shared cleanup") + print(f"Starting whole-phase group {pair + 1} ({args.mode})", flush=True) + group = await run_group(args, manifest, pair) + summary["groups"].append( + { + "pair": pair, + "completed_valid": group["completed_valid"], + "record": f"group-{pair}.json", + "sha256": digest(args.output / f"group-{pair}.json"), + } + ) + summary["trials"].extend(group["trials"]) + atomic_json(args.output / "summary.json", summary) + print(f"Finished whole-phase group {pair + 1}", flush=True) + trials = [load_trial(args.output, record) for record in summary["trials"]] + validate_trials(trials, args.mode, smoke=args.smoke) + # Smoke still validates all three peers' numerics/memory: fill only the + # iteration count via a dedicated single-triplet check, not fabricated + # replicas. Full acceptance always uses the original six-triplet check. + if args.smoke: + summary["correctness_memory"] = smoke_cross_checks(trials) + else: + summary["correctness_memory"] = cross_checks(trials) + summary["statistics"] = compare(trials) + summary["completed_valid"] = True + summary["qualification_passed"] = ( + not args.smoke and args.mode == "qualification" and summary["statistics"]["passed"] + ) + summary["passed"] = ( + summary["completed_valid"] + if args.mode == "baseline-pilot" or args.smoke + else summary["qualification_passed"] + ) + require(summary["passed"], "fixed non-inferiority gates did not pass") + except BaseException as error: + summary["error"] = {"type": type(error).__name__, "message": str(error)} + raise + finally: + atomic_json(args.output / "summary.json", summary) + return summary + + +def smoke_cross_checks(trials: list[dict[str, Any]]) -> dict[str, Any]: + # Reuse the exact peer checks without weakening them or inventing six + # observations. The helper reads ORDERS only to determine the pair count. + check = FunctionType(cross_checks.__code__, {**cross_checks.__globals__, "ORDERS": ORDERS[:1]}) + return check(trials) + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--inputs", type=Path, required=True) + parser.add_argument("--fixtures", type=Path, required=True) + parser.add_argument("--snapshot", type=Path) + parser.add_argument("--environment-receipt", type=Path) + parser.add_argument("--output", type=Path, required=True) + parser.add_argument("--mode", choices=MODES, required=True) + parser.add_argument("--deadline-s", type=float, default=RUNNER_LIMIT_S) + parser.add_argument("--group-work-s", type=float, default=GROUP_WORK_S) + parser.add_argument("--smoke", action="store_true") + args = parser.parse_args() + + async def launch() -> None: + task = asyncio.current_task() + assert task is not None + loop = asyncio.get_running_loop() + for sig in (signal.SIGTERM, signal.SIGINT): + loop.add_signal_handler(sig, task.cancel) + await run(args) + + asyncio.run(launch()) + + +if __name__ == "__main__": + main() diff --git a/tools/benchmarks/cross_encoder_lora/isolation.py b/tools/benchmarks/cross_encoder_lora/isolation.py index 4cadc6031..121c54539 100644 --- a/tools/benchmarks/cross_encoder_lora/isolation.py +++ b/tools/benchmarks/cross_encoder_lora/isolation.py @@ -28,14 +28,19 @@ class IsolatedLoadgen: """Leave uvicorn running while awaiting a bounded, same-group SDK child.""" - def __init__(self, url: str, source: Path, evidence: Evidence, *, smoke: bool) -> None: + def __init__( + self, url: str, source: Path, evidence: Evidence, *, smoke: bool, idle_deadline: float | None = None + ) -> None: self.url, self.source, self.evidence, self.smoke = url, source.resolve(), evidence, smoke + self.idle_deadline = idle_deadline self.token = uuid.uuid4().hex self.process: asyncio.subprocess.Process | None = None self.log: Any = None self.records: list[dict[str, Any]] = [] self.observations: dict[int, dict[str, Any]] = {} self.receipt: dict[str, Any] = {"passed": False, "reaped": False, "phases": self.records} + self.exit_watcher: asyncio.Task | None = None + self.expected_exit = False async def _send(self, command: str, concurrency: int | None = None) -> None: assert self.process is not None @@ -75,6 +80,11 @@ async def __aenter__(self) -> Self: ] if self.smoke: command.append("--smoke") + if self.idle_deadline is not None: + require( + math.isfinite(self.idle_deadline) and self.idle_deadline > time.monotonic(), "invalid idle deadline" + ) + command.extend(["--idle-deadline", str(self.idle_deadline)]) self.log = (self.evidence.directory / "loadgen.log").open("wb") try: self.process = await asyncio.create_subprocess_exec( @@ -96,6 +106,7 @@ async def __aenter__(self) -> Self: identity["source"] == str(self.source) and identity["forbidden_imports_present"] is False, "loadgen source/import preflight failed", ) + require(hello.get("idle_deadline") == self.idle_deadline, "loadgen idle deadline differs") sdk_path = Path(identity["sdk_path"]) require( sdk_path.is_relative_to(self.source / "packages/sie_sdk/src"), "loadgen SDK source escaped its archive" @@ -108,6 +119,10 @@ async def __aenter__(self) -> Self: ) self.receipt["identity"] = identity self.evidence.emit("loadgen_started", **identity) + if self.idle_deadline is not None: + owner = asyncio.current_task() + assert owner is not None + self.exit_watcher = asyncio.create_task(self._watch_group_exit(owner)) return self except BaseException as error: try: @@ -116,6 +131,12 @@ async def __aenter__(self) -> Self: error.add_note(f"Loadgen startup cleanup failed: {type(cleanup_error).__name__}: {cleanup_error}") raise + async def _watch_group_exit(self, owner: asyncio.Task) -> None: + assert self.process is not None + await self.process.wait() + if not self.expected_exit: + owner.cancel("group SDK child exited before finalization") + async def phase(self, concurrency: int) -> dict[str, Any]: require( len(self.records) < len(PHASES) and concurrency == PHASES[len(self.records)], @@ -174,6 +195,7 @@ async def phase(self, concurrency: int) -> dict[str, Any]: ) async def _cleanup(self, successful: bool) -> None: + self.expected_exit = True try: if self.process is not None: try: @@ -209,6 +231,9 @@ async def _cleanup(self, successful: bool) -> None: else: self.receipt["reaped"] = True finally: + if self.exit_watcher is not None: + self.exit_watcher.cancel() + await asyncio.gather(self.exit_watcher, return_exceptions=True) if self.log is not None: self.log.close() atomic_json(self.evidence.directory / "loadgen-process.json", self.receipt) @@ -254,6 +279,7 @@ def _validate_terminal(self, terminal: dict[str, Any]) -> None: require(terminal.get("passed") is True and "error" not in terminal, "loadgen terminal did not pass") require(terminal.get("identity") == self.receipt["identity"], "loadgen terminal identity differs") require(terminal.get("smoke") is self.smoke, "loadgen terminal mode differs") + require(terminal.get("idle_deadline") == self.idle_deadline, "loadgen terminal idle deadline differs") require(terminal.get("phases") == self.records, "loadgen terminal phase records differ") require(tuple(record["concurrency"] for record in self.records) == PHASES, "loadgen terminal schedule differs") require( diff --git a/tools/benchmarks/cross_encoder_lora/loadgen.py b/tools/benchmarks/cross_encoder_lora/loadgen.py index f1fb26dd8..96ecc04ea 100644 --- a/tools/benchmarks/cross_encoder_lora/loadgen.py +++ b/tools/benchmarks/cross_encoder_lora/loadgen.py @@ -6,6 +6,7 @@ import gc import inspect import json +import math import os import queue import select @@ -95,8 +96,16 @@ def close(self) -> None: os.close(self.stop_read) os.close(self.stop_write) - def receive(self, command: str, concurrency: int | None = None) -> dict[str, Any]: - value = self.messages.get(timeout=CONTROL_TIMEOUT_S) + def receive( + self, command: str, concurrency: int | None = None, *, idle_deadline: float | None = None + ) -> dict[str, Any]: + wait_s = CONTROL_TIMEOUT_S + if idle_deadline is not None: + require(command in {"phase", "finish"}, "group idle deadline used for active handshake") + require(math.isfinite(idle_deadline), "invalid group idle deadline") + wait_s = idle_deadline - time.monotonic() + require(wait_s > 0, "group idle deadline expired") + value = self.messages.get(timeout=wait_s) require("reader_error" not in value, "control reader failed") require(value.get("token") == self.token and value.get("command") == command, "unexpected parent command") require(value.get("concurrency") == concurrency, "parent phase identity differs") @@ -134,10 +143,15 @@ def run(args: argparse.Namespace) -> None: control = Control(args.token) terminal: dict[str, Any] = {"passed": False, "phases": [], "identity": identity(args.source), "smoke": args.smoke} retained_results = [] - control.send("hello", identity=terminal["identity"]) + idle_deadline = getattr(args, "idle_deadline", None) + terminal["idle_deadline"] = idle_deadline + control.send("hello", identity=terminal["identity"], idle_deadline=idle_deadline) try: for concurrency in PHASES: - control.receive("phase", concurrency) + if idle_deadline is None: + control.receive("phase", concurrency) + else: + control.receive("phase", concurrency, idle_deadline=idle_deadline) require_sdk_only() callbacks: tuple[Any, ...] = () observations: dict[str, Any] = {} @@ -198,7 +212,10 @@ def after_measure(clock: dict[str, float]) -> None: # rows until every timed phase is complete. retained_results.append(result) control.send("phase_complete", concurrency, record=record) - control.receive("finish") + if idle_deadline is None: + control.receive("finish") + else: + control.receive("finish", idle_deadline=idle_deadline) terminal["passed"] = True except BaseException as error: terminal["error"] = {"type": type(error).__name__, "message": str(error)} @@ -218,6 +235,7 @@ def main() -> None: parser.add_argument("--url", required=True) parser.add_argument("--token", required=True) parser.add_argument("--smoke", action="store_true") + parser.add_argument("--idle-deadline", type=float) run(parser.parse_args()) diff --git a/tools/benchmarks/cross_encoder_lora/tests/test_group_control.py b/tools/benchmarks/cross_encoder_lora/tests/test_group_control.py new file mode 100644 index 000000000..5e84aff02 --- /dev/null +++ b/tools/benchmarks/cross_encoder_lora/tests/test_group_control.py @@ -0,0 +1,361 @@ +"""Dedicated socket ownership, real uvicorn cancellation and bounded teardown.""" + +# All Python children and signals are scoped to new owned process groups. +# ruff: noqa: S603 + +from __future__ import annotations + +import asyncio +import json +import os +import signal +import socket +import subprocess +import sys +import time +from pathlib import Path +from typing import cast + +import pytest + +from tools.benchmarks.cross_encoder_lora.evidence import atomic_json, digest +from tools.benchmarks.cross_encoder_lora.group_control import Member, decode, envelope +from tools.benchmarks.cross_encoder_lora.interleaved import Peer, cleanup, join_cleanup, watch_peer +from tools.benchmarks.cross_encoder_lora.loadgen import Control +from tools.benchmarks.cross_encoder_lora.runner import ROOT, source_environment +from tools.benchmarks.cross_encoder_lora.tests.test_interleaved import binding +from tools.benchmarks.cross_encoder_lora.tests.test_isolation import sdk_server # noqa: F401 -- pytest fixture + +HELPER = r""" +import asyncio, json, os, signal, socket, sys +from pathlib import Path +from tools.benchmarks.cross_encoder_lora.group_control import Member +from tools.benchmarks.cross_encoder_lora.evidence import Evidence, atomic_json +value=json.loads(Path(sys.argv[1]).read_text()) +owner=Member(int(sys.argv[2]), value, Path(value['output'])) +# These are imported only after ownership is active, as in the actual entrypoint. +import uvicorn +from tools.benchmarks.cross_encoder_lora.isolation import IsolatedLoadgen +from tools.benchmarks.cross_encoder_lora.runner import ROOT +async def app(scope, receive, send): + if scope['type']=='lifespan': + while True: + message=await receive() + await send({'type': message['type']+'.complete'}) + if message['type']=='lifespan.shutdown': return + else: + await send({'type':'http.response.start','status':200,'headers':[]}) + await send({'type':'http.response.body','body':b'ok'}) +async def run(): + owner.bind_task() + await owner.grant('prepare') + sock=socket.socket(); sock.bind(('127.0.0.1',0)) + server=uvicorn.Server(uvicorn.Config(app, log_level='warning', access_log=False)) + task=asyncio.create_task(server.serve(sockets=[sock])) + loadgen=None + try: + while not server.started: await asyncio.sleep(.01) + print('ordinary model/log chatter {not protocol}',flush=True) + async with IsolatedLoadgen(sys.argv[3],ROOT,Evidence(Path(value['output'])/'sdk'),smoke=True, + idle_deadline=value['work_deadline']) as loadgen: + owner.acknowledge(sdk_pid=loadgen.process.pid,uvicorn_handler=repr(signal.getsignal(signal.SIGTERM))) + await owner.grant('C1') + record=await loadgen.phase(1) + owner.acknowledge(record=record) + await owner.grant('C4') + finally: + server.should_exit=True + await asyncio.wait_for(task,5) + sock.close() + atomic_json(Path(value['output'])/'helper-cleanup.json',{'loadgen':loadgen.receipt if loadgen else None}) +try: + asyncio.run(run()) +finally: + owner.close() +""" + + +@pytest.mark.parametrize("failure", ["eof", "wrong_identity", "stale_stage", "sdk_crash"]) +def test_actual_uvicorn_eof_cancels_and_reaps_sdk_preserving_phase( + request: pytest.FixtureRequest, tmp_path: Path, failure: str +) -> None: + url = request.getfixturevalue("sdk_server") + value = binding(tmp_path) + value["coordinator_pid"] = os.getpid() + path = tmp_path / "binding.json" + atomic_json(path, value) + parent, child = socket.socketpair() + process = None + with (tmp_path / "worker.log").open("wb") as log: + try: + process = subprocess.Popen( + [sys.executable, "-c", HELPER, str(path), str(child.fileno()), url], + pass_fds=(child.fileno(),), + close_fds=True, + start_new_session=True, + stdout=log, + stderr=log, + env=source_environment(ROOT), + ) + child.close() + parent.settimeout(10) + reader = parent.makefile("rb", buffering=0) + hello = decode(reader.readline(), value) + assert hello["pid"] == process.pid + + def grant(stage: str, ordinal: int) -> None: + parent.sendall( + ( + json.dumps(envelope(value, command="grant", stage=stage, ordinal=ordinal, lease="b" * 32)) + + "\n" + ).encode() + ) + + grant("prepare", 0) + prepared = decode(reader.readline(), value) + assert ( + "Server.capture_signals" in prepared["uvicorn_handler"] + or "Server.handle_exit" in prepared["uvicorn_handler"] + ) + sdk_pid = prepared["sdk_pid"] + grant("C1", 1) + complete = decode(reader.readline(), value) + completed_file = tmp_path / "sdk/loadgen/phase-C1.json" + original_hash = digest(completed_file) + assert original_hash == complete["record"]["sha256"] + if failure == "wrong_identity": + parent.sendall(b'{"binding_sha256":"wrong"}\n') + elif failure == "stale_stage": + grant("C1", 1) + elif failure == "sdk_crash": + os.kill(sdk_pid, signal.SIGKILL) + else: + reader.close() + parent.close() + assert process.wait(timeout=12) != 0 + receipt = json.loads((tmp_path / "helper-cleanup.json").read_text()) + assert receipt["loadgen"]["reaped"] + assert digest(completed_file) == original_hash + with pytest.raises(ProcessLookupError): + os.kill(sdk_pid, 0) + assert "ordinary model/log chatter" in (tmp_path / "worker.log").read_text() + finally: + parent.close() + child.close() + if process is not None and process.poll() is None: + os.killpg(process.pid, signal.SIGKILL) + process.wait(timeout=5) + + +@pytest.mark.asyncio +async def test_repeated_cancellation_joins_one_shared_cleanup_and_preserves_primary() -> None: + reaped = [] + + async def cleanup(): + await asyncio.sleep(0.08) + reaped.extend(range(6)) + return "complete" + + primary = ValueError("original phase failure") + task = asyncio.create_task(cleanup()) + owner = asyncio.create_task(join_cleanup(task, primary)) + await asyncio.sleep(0.01) + owner.cancel() + await asyncio.sleep(0.01) + owner.cancel() + assert await owner == "complete" + assert reaped == list(range(6)) + assert task.done() + assert str(primary) == "original phase failure" + assert "owned reaping" in primary.__notes__[0] + + +@pytest.mark.asyncio +async def test_cancel_during_clean_exit_propagates_after_reaping() -> None: + reaped = [] + + async def cleanup(): + await asyncio.sleep(0.03) + reaped.append(True) + + task = asyncio.create_task(cleanup()) + owner = asyncio.create_task(join_cleanup(task, None)) + await asyncio.sleep(0.01) + owner.cancel() + with pytest.raises(asyncio.CancelledError): + await owner + assert reaped + assert task.done() + + +def test_closed_event_loop_cannot_disable_hard_fallback() -> None: + owner = object.__new__(Member) + owner.loop = asyncio.new_event_loop() + owner.loop.close() + owner.owner = type("Owner", (), {"cancel": lambda *_: None})() + owner._fail("parent gone") + assert owner.failure == "parent gone" + + +def test_group_idle_deadline_is_finite_and_only_for_idle_commands() -> None: + owner = object.__new__(Control) + + class Messages: + def get(self, *, timeout): + assert 30 < timeout <= 60 + return {"token": "owned", "command": "phase", "concurrency": 4} + + owner.messages = Messages() + owner.token = "owned" # noqa: S105 -- local control nonce fixture + assert owner.receive("phase", 4, idle_deadline=time.monotonic() + 60)["concurrency"] == 4 + with pytest.raises(AssertionError, match="active handshake"): + owner.receive("go", 4, idle_deadline=time.monotonic() + 60) + with pytest.raises(AssertionError, match="expired"): + owner.receive("phase", 4, idle_deadline=time.monotonic() - 1) + + +COORDINATOR = r""" +import json, os, socket, subprocess, sys, time +from pathlib import Path +from tools.benchmarks.cross_encoder_lora.evidence import atomic_json,digest +from tools.benchmarks.cross_encoder_lora.group_control import decode,envelope +from tools.benchmarks.cross_encoder_lora.runner import ROOT,source_environment +root=Path(sys.argv[1]); value=json.loads((root/'template.json').read_text()) +owned=[]; sockets=[]; records=[] +for role in 'BNR': + directory=root/role; directory.mkdir() + b={**value,'logical_role':role,'output':str(directory),'coordinator_pid':os.getpid()} + path=directory/'binding.json';atomic_json(path,b) + parent,child=socket.socketpair();sockets.append(parent) + log=(directory/'worker.log').open('wb') + p=subprocess.Popen([sys.executable,str(root/'helper.py'),str(path),str(child.fileno()),sys.argv[2]], + pass_fds=(child.fileno(),),close_fds=True,start_new_session=True,stdout=log,stderr=log,env=source_environment(ROOT)) + child.close();parent.settimeout(15);reader=parent.makefile('rb',buffering=0) + decode(reader.readline(),b) + def grant(stage,ordinal): + parent.sendall((json.dumps(envelope(b,command='grant',stage=stage,ordinal=ordinal,lease='b'*32))+'\n').encode()) + grant('prepare',0); prepared=decode(reader.readline(),b) + grant('C1',1); complete=decode(reader.readline(),b) + records.append({'pid':p.pid,'sdk_pid':prepared['sdk_pid'],'directory':str(directory), + 'phase_sha256':complete['record']['sha256']}) + owned.append((p,reader,log)) +atomic_json(root/'ready.json',{'children':records}) +# Parent is deliberately SIGKILLed by the test. No Python finally is expected. +while True: time.sleep(60) +""" + + +def test_actual_coordinator_sigkill_ends_all_three_uvicorn_sdk_pairs( + request: pytest.FixtureRequest, tmp_path: Path +) -> None: + url = request.getfixturevalue("sdk_server") + atomic_json(tmp_path / "template.json", binding(tmp_path)) + (tmp_path / "helper.py").write_text(HELPER) + process = subprocess.Popen( + [sys.executable, "-c", COORDINATOR, str(tmp_path), url], + env=source_environment(ROOT), + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + start_new_session=True, + ) + rows = [] + try: + deadline = time.monotonic() + 15 + while not (tmp_path / "ready.json").exists(): + assert process.poll() is None, process.communicate() + assert time.monotonic() < deadline + time.sleep(0.02) + rows = json.loads((tmp_path / "ready.json").read_text())["children"] + assert len(rows) == 3 + process.kill() + assert process.wait(timeout=5) == -signal.SIGKILL + deadline = time.monotonic() + 10 + while not all((Path(row["directory"]) / "helper-cleanup.json").exists() for row in rows): + assert time.monotonic() < deadline + time.sleep(0.02) + for row in rows: + directory = Path(row["directory"]) + receipt = json.loads((directory / "helper-cleanup.json").read_text()) + assert receipt["loadgen"]["reaped"] + assert digest(directory / "sdk/loadgen/phase-C1.json") == row["phase_sha256"] + with pytest.raises(ProcessLookupError): + os.kill(row["sdk_pid"], 0) + deadline = time.monotonic() + 10 + for row in rows: + while True: + try: + os.kill(row["pid"], 0) + except ProcessLookupError: + break + assert time.monotonic() < deadline, "server survived coordinator loss" + time.sleep(0.02) + finally: + if process.poll() is None: + process.kill() + process.wait(timeout=5) + for row in rows: + try: + os.killpg(row["pid"], signal.SIGKILL) + except ProcessLookupError: + pass + + +@pytest.mark.asyncio +async def test_simultaneous_teardown_uses_one_shared_deadline(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + signalled = [] + gates = [asyncio.Event() for _ in range(3)] + + class Process: + def __init__(self, index): + self.pid = index + 1 + self.returncode = None + + async def wait(self): + await gates[self.pid - 1].wait() + self.returncode = -signal.SIGKILL + return self.returncode + + def killpg(pid, sig): + signalled.append((pid, sig, time.monotonic())) + if sig == signal.SIGKILL: + gates[pid - 1].set() + + monkeypatch.setattr(os, "killpg", killpg) + peers = [Peer({"logical_role": role}, (tmp_path / f"{role}.log").open("wb")) for role in "BNR"] + for index, peer in enumerate(peers): + peer.process = cast("asyncio.subprocess.Process", Process(index)) + started = time.monotonic() + receipt = await cleanup(peers, started + 0.5) + assert time.monotonic() - started < 0.5 + assert [row[1] for row in signalled[:3]] == [signal.SIGTERM] * 3 + assert [row[1] for row in signalled[3:]] == [signal.SIGKILL] * 3 + assert all(row["reaped"] for row in receipt) + + +@pytest.mark.asyncio +async def test_inactive_worker_failure_cancels_group() -> None: + done = asyncio.Event() + + class Process: + async def wait(self): + await done.wait() + + peer = object.__new__(Peer) + peer.process = Process() + peer.binding = {"logical_role": "R"} + peer.finishing = peer.stopping = False + + async def active_group(): + owner = asyncio.current_task() + assert owner is not None + watcher = asyncio.create_task(watch_peer(peer, owner)) + try: + await asyncio.sleep(10) + finally: + await watcher + + task = asyncio.create_task(active_group()) + await asyncio.sleep(0) + done.set() + with pytest.raises(asyncio.CancelledError, match="before group finish"): + await task diff --git a/tools/benchmarks/cross_encoder_lora/tests/test_interleaved.py b/tools/benchmarks/cross_encoder_lora/tests/test_interleaved.py new file mode 100644 index 000000000..703720d2a --- /dev/null +++ b/tools/benchmarks/cross_encoder_lora/tests/test_interleaved.py @@ -0,0 +1,355 @@ +"""Actual archived loader/SDK triplets and fail-closed group semantics.""" + +# All subprocesses use the current interpreter and owned temporary artifacts. +# ruff: noqa: S603 + +from __future__ import annotations + +import argparse +import asyncio +import copy +import json +import os +import signal +import subprocess +import sys +import time +from pathlib import Path +from typing import cast + +import pytest + +from tools.benchmarks.cross_encoder_lora import interleaved, prepare +from tools.benchmarks.cross_encoder_lora.evidence import atomic_json, digest +from tools.benchmarks.cross_encoder_lora.fixtures import prepare as fixtures_prepare +from tools.benchmarks.cross_encoder_lora.group_control import envelope, selection, validate_binding, validate_source +from tools.benchmarks.cross_encoder_lora.protocol import BASE_REVISION +from tools.benchmarks.cross_encoder_lora.runner import ROOT, source_environment +from tools.benchmarks.cross_encoder_lora.tests.test_probe import tiny_snapshot + + +def test_strict_group_admission() -> None: + assert not interleaved.admission(100, 789)["admitted"] + assert not interleaved.admission(100, 790)["admitted"] + assert interleaved.admission(100, 791)["admitted"] + + +@pytest.mark.parametrize("role", ["B", "N", "R"]) +def test_explicit_role_configuration_source(role: str) -> None: + assert selection("baseline-pilot", role) == ("B", "B") + assert selection("qualification", role) == (("B", "B") if role == "B" else (role, "candidate")) + + +def binding(tmp_path: Path) -> dict: + start = time.monotonic() + return { + "schema": 1, + "schedule": "whole-phase", + "mode": "baseline-pilot", + "pair": 0, + "logical_role": "R", + "actual_arm": "B", + "source_key": "B", + "source_commit": BASE_REVISION, + "nonce": "a" * 32, + "coordinator_pid": 1, + "group_started": start, + "work_deadline": start + 30, + "hard_deadline": start + 120, + "active_limit_s": 270, + "phase_limit_s": 120, + "output": str(tmp_path), + } + + +@pytest.mark.parametrize( + "change", + [ + {"actual_arm": "R"}, + {"source_key": "candidate"}, + {"source_commit": "f" * 40}, + {"phase_limit_s": 121}, + {"active_limit_s": 271}, + {"pair": True}, + ], +) +def test_binding_rejects_relabeling_and_budget_changes(tmp_path: Path, change: dict) -> None: + value = binding(tmp_path) + validate_binding(value) + value.update(change) + with pytest.raises(AssertionError): + validate_binding(value) + + +def test_qualification_resident_cannot_be_base(tmp_path: Path) -> None: + value = {**binding(tmp_path), "mode": "qualification"} + with pytest.raises(AssertionError, match="mismatch"): + validate_binding(value) + + +def test_manifest_and_extracted_source_hashes_checked_before_load(tmp_path: Path) -> None: + source = tmp_path / "source" + source.mkdir() + (source / "uv.lock").write_text("fixed") + archive = tmp_path / "B.tar" + archive.write_bytes(b"archive bytes") + manifest = tmp_path / "manifest.json" + atomic_json( + manifest, + { + "schema": 1, + "dependency_lock_identical": True, + "sources": {"B": {"commit": BASE_REVISION, "archive": "B.tar", "sha256": digest(archive)}}, + }, + ) + value = { + **binding(tmp_path), + "source_root": str(source), + "source_manifest": str(manifest), + "source_manifest_sha256": digest(manifest), + "archive_sha256": digest(archive), + "source_files": {"uv.lock": digest(source / "uv.lock")}, + } + validate_source(value, source, "B", 0) + with pytest.raises(AssertionError, match="arguments"): + validate_source(value, source, "R", 0) + (source / "uv.lock").write_text("changed") + with pytest.raises(AssertionError, match="source file"): + validate_source(value, source, "B", 0) + + +@pytest.mark.parametrize("mode", ["baseline-pilot", "qualification"]) +def test_real_archived_whole_phase_triplet(tmp_path: Path, mode: str) -> None: + snapshot = tiny_snapshot(tmp_path / "model") + fixtures = tmp_path / "fixtures" + fixtures_prepare(fixtures, snapshot, smoke=True) + inputs = tmp_path / "inputs" + prepare.prepare(ROOT, inputs, "HEAD") + output = tmp_path / "out" + command = [ + sys.executable, + "-m", + "tools.benchmarks.cross_encoder_lora.interleaved", + "--inputs", + str(inputs), + "--fixtures", + str(fixtures), + "--output", + str(output), + "--mode", + mode, + "--smoke", + "--group-work-s", + "120", + "--deadline-s", + "240", + ] + result = subprocess.run( + command, cwd=ROOT, env=source_environment(ROOT), capture_output=True, text=True, timeout=225, check=False + ) + assert result.returncode == 0, ( + result.stdout + + result.stderr + + "\n" + + "\n".join(path.read_text()[-8000:] for path in sorted(output.glob("*.log"))) + ) + summary = json.loads((output / "summary.json").read_text()) + assert summary["completed_valid"] + assert summary["passed"] + assert summary["qualification_passed"] is False + group = json.loads((output / "group-0.json").read_text()) + assert [event["stage"] for event in group["leases"]] == [ + stage for stage in ("prepare", "C1", "C4", "C16", "finish") for _ in range(3) + ] + for left, right in zip(group["leases"], group["leases"][1:], strict=False): + assert left["ended"] <= right["started"] + phase_end = max(event["ended"] for event in group["leases"] if event["stage"].startswith("C")) + assert all(event["started"] >= phase_end for event in group["leases"] if event["stage"] == "finish") + assert len(group["cleanup"]) == 3 + assert all(row["reaped"] and row["returncode"] == 0 for row in group["cleanup"]) + trials = [json.loads((output / record["terminal_record"]).read_text()) for record in summary["trials"]] + interleaved.validate_trials(trials, mode, smoke=True) + assert {row["arm"] for row in trials} == {"B", "N", "R"} + for trial in trials: + actual, source_key = selection(mode, trial["arm"]) + assert trial["actual_arm"] == actual + assert trial["group"]["source_key"] == source_key + assert trial["manager_present"] == (actual == "R") + assert trial["loadgen"]["passed"] + assert trial["loadgen"]["reaped"] + assert Path(trial["environment"]["adapter_source"]).is_relative_to(output / "sources" / source_key) + assert all( + phase["client_observation"]["gc"]["mode"] == "boundary-only" for phase in trial["performance"].values() + ) + invalid = copy.deepcopy(trials) + invalid[-1]["actual_arm"] = "B" if mode == "qualification" else "R" + with pytest.raises(AssertionError, match="configuration"): + interleaved.validate_trials(invalid, mode, smoke=True) + + +def test_incomplete_group_cannot_pass(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + args = argparse.Namespace( + mode="baseline-pilot", + deadline_s=1000, + group_work_s=600, + output=tmp_path / "out", + inputs=tmp_path, + fixtures=tmp_path, + environment_receipt=None, + smoke=False, + ) + monkeypatch.setattr(interleaved, "extract", lambda *_: {}) + + async def fail(*_): + raise RuntimeError("owned child failed") + + monkeypatch.setattr(interleaved, "run_group", fail) + with pytest.raises(RuntimeError, match="owned child"): + asyncio.run(interleaved.run(args)) + value = json.loads((args.output / "summary.json").read_text()) + assert not value["completed_valid"] + assert not value["qualification_passed"] + assert not value["passed"] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("mutation", ["lease", "ordinal", "stage", "phase"]) +async def test_phase_ack_must_bind_exact_lease_and_persisted_payload(tmp_path: Path, mutation: str) -> None: + value = binding(tmp_path) + (tmp_path / "loadgen").mkdir() + path = tmp_path / "loadgen/phase-C1.json" + path.write_text("{}") + peer = interleaved.Peer(value, (tmp_path / "log").open("wb")) + peer.ordinal = 1 + peer.reader = asyncio.StreamReader() + + class Writer: + def write(self, raw): + request = json.loads(raw) + ack = envelope( + value, + event="ack", + stage="C1", + ordinal=1, + lease=request["lease"], + monotonic_s=time.monotonic(), + quiet={"pending": 0, "in_flight": 0}, + record={"concurrency": 1, "path": "phase-C1.json", "bytes": 2, "sha256": digest(path)}, + ) + if mutation == "phase": + ack["record"]["concurrency"] = 4 + elif mutation == "ordinal": + ack[mutation] = 0 + else: + ack[mutation] = "wrong" + assert peer.reader is not None + peer.reader.feed_data((json.dumps(ack) + "\n").encode()) + + async def drain(self): + pass + + peer.writer = cast("asyncio.StreamWriter", Writer()) + try: + with pytest.raises(AssertionError): + await peer.lease("C1") + assert peer.ordinal == 1 + finally: + peer.log.close() + + +def test_coordinator_full_file_hashes_wait_for_all_nine_timers(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + peers = [] + read = [] + for role in "BNR": + directory = tmp_path / role + (directory / "loadgen").mkdir(parents=True) + peer = interleaved.Peer({"logical_role": role, "output": str(directory)}, (directory / "log").open("wb")) + peer.events = [{}] + for c in (1, 4, 16): + path = directory / "loadgen" / f"phase-C{c}.json" + path.write_text("{}") + peer.events.append({"ack": {"record": {"path": path.name, "bytes": 2, "sha256": digest(path)}}}) + peer.ordinal = 4 + peers.append(peer) + original = interleaved.digest + + def observed(path): + read.append(path) + return original(path) + + monkeypatch.setattr(interleaved, "digest", observed) + try: + peers[-1].ordinal = 3 + with pytest.raises(AssertionError, match="timers incomplete"): + interleaved.verify_phase_files(peers) + assert not read + peers[-1].ordinal = 4 + interleaved.verify_phase_files(peers) + assert len(read) == 9 + peers[-1].events[-1]["ack"]["record"]["sha256"] = "f" * 64 + with pytest.raises(AssertionError, match="persistence"): + interleaved.verify_phase_files(peers) + finally: + for peer in peers: + peer.log.close() + + +def test_actual_coordinator_term_reaps_archived_workers_and_sdk_children(tmp_path: Path) -> None: + snapshot = tiny_snapshot(tmp_path / "model") + fixtures = tmp_path / "fixtures" + fixtures_prepare(fixtures, snapshot, smoke=True) + inputs = tmp_path / "inputs" + prepare.prepare(ROOT, inputs, "HEAD") + output = tmp_path / "out" + command = [ + sys.executable, + "-m", + "tools.benchmarks.cross_encoder_lora.interleaved", + "--inputs", + str(inputs), + "--fixtures", + str(fixtures), + "--output", + str(output), + "--mode", + "baseline-pilot", + "--smoke", + "--group-work-s", + "120", + "--deadline-s", + "240", + ] + with (tmp_path / "coordinator.log").open("wb") as log: + process = subprocess.Popen(command, cwd=ROOT, env=source_environment(ROOT), stdout=log, stderr=log) + try: + deadline = time.monotonic() + 60 + group_path = output / "group-0.json" + while True: + assert process.poll() is None, (tmp_path / "coordinator.log").read_text() + assert time.monotonic() < deadline + if group_path.exists(): + group = json.loads(group_path.read_text()) + if len(group["leases"]) >= 4: + break + time.sleep(0.01) + completed = output / "trial-0-B/loadgen/phase-C1.json" + before = digest(completed) + process.send_signal(signal.SIGTERM) + assert process.wait(timeout=20) != 0 + group = json.loads(group_path.read_text()) + assert not group["completed_valid"] + assert group["error"]["type"] == "CancelledError" + assert digest(completed) == before + assert all(row["reaped"] for row in group["cleanup"]) + for row in group["cleanup"]: + with pytest.raises(ProcessLookupError): + os.kill(row["pid"], 0) + for role in "BNR": + receipt = json.loads((output / f"trial-0-{role}/loadgen-process.json").read_text()) + assert receipt["reaped"] + with pytest.raises(ProcessLookupError): + os.kill(receipt["identity"]["pid"], 0) + finally: + if process.poll() is None: + process.kill() + process.wait(timeout=5) diff --git a/tools/benchmarks/cross_encoder_lora/worker.py b/tools/benchmarks/cross_encoder_lora/worker.py index 01c56d044..8c4d21bf4 100644 --- a/tools/benchmarks/cross_encoder_lora/worker.py +++ b/tools/benchmarks/cross_encoder_lora/worker.py @@ -42,6 +42,7 @@ describe_response, ) from tools.benchmarks.cross_encoder_lora.evidence import Evidence, atomic_json, digest, fingerprint, vector_delta +from tools.benchmarks.cross_encoder_lora.group_control import Member, validate_source from tools.benchmarks.cross_encoder_lora.isolation import IsolatedLoadgen from tools.benchmarks.cross_encoder_lora.protocol import ( CONCURRENCIES, @@ -354,7 +355,8 @@ def preflight(args: argparse.Namespace, evidence: Evidence) -> dict[str, Any]: "threads": torch.get_num_threads(), "device": str(torch.cuda.get_device_name(0)) if torch.cuda.is_available() else "cpu", "torch_cuda": torch.version.cuda, - "qualification": not args.smoke, + "qualification": not args.smoke + and (getattr(args, "group", None) is None or args.group.binding["mode"] == "qualification"), "installed_distributions": sorted( (dist.metadata["Name"], dist.version) for dist in importlib.metadata.distributions() ), @@ -369,6 +371,7 @@ def preflight(args: argparse.Namespace, evidence: Evidence) -> dict[str, Any]: async def experiment(args: argparse.Namespace, evidence: Evidence, summary: dict[str, Any]) -> None: + group: Member | None = getattr(args, "group", None) fixtures = args.fixtures.resolve() manifest = json.loads((fixtures / "manifest.json").read_text()) snapshot = args.snapshot.resolve() if args.snapshot else Path(manifest["snapshot"]) @@ -585,12 +588,34 @@ def concurrent(index: int) -> dict[str, Any]: "policy": "Existing server histories retained; memory() just completed its existing gc.collect/CUDA synchronization outside timing. No new collection or history release is introduced.", } with unobserved(adapter): - async with IsolatedLoadgen(url, args.source, evidence, smoke=args.smoke) as loadgen: + async with IsolatedLoadgen( + url, + args.source, + evidence, + smoke=args.smoke, + idle_deadline=group.binding["work_deadline"] if group is not None else None, + ) as loadgen: + if group is not None: + group.acknowledge( + actual_arm=args.arm, + manager_present=summary["manager_present"], + config_sha256=digest(evidence.directory / "models/model.yaml"), + sdk_identity=loadgen.receipt["identity"], + quiet=group_quiet(worker), + ) for concurrency in (1, *CONCURRENCIES): + if group is not None: + await group.grant(f"C{concurrency}") phase = f"performance-C{concurrency}" evidence.emit("gpu_before_phase", phase=phase, telemetry=telemetry()) - await loadgen.phase(concurrency) + record = await loadgen.phase(concurrency) evidence.emit("gpu_after_phase", phase=phase, telemetry=telemetry()) + if group is not None: + group.acknowledge(record=record, quiet=group_quiet(worker)) + if group is not None: + # No SDK shutdown, raw payload loading, churn, unload or + # finalization until all nine group phase ACKs exist. + await group.grant("finish") # Reaping and all raw-row materialization/validation happen after # every measured phase, so their allocations never enter the # serving heap during a subsequent timed phase. @@ -687,16 +712,33 @@ async def unload_adapters() -> None: sock.close() +def group_quiet(worker: Any) -> dict[str, Any]: + """A lease ends only after drained requests and an outside-timer CUDA fence.""" + if torch.cuda.is_available(): + torch.cuda.synchronize() + state = { + "pending": worker.pending_count, + "in_flight": len(worker._in_flight), + "cuda_synchronized": torch.cuda.is_available(), + } + require(state["pending"] == 0 and state["in_flight"] == 0, "worker was not quiet at lease boundary") + return state + + async def run(args: argparse.Namespace) -> dict[str, Any]: + group: Member | None = getattr(args, "group", None) evidence = Evidence(args.output.resolve()) summary: dict[str, Any] = { - "arm": args.arm, + "arm": group.binding["logical_role"] if group is not None else args.arm, "pair": args.pair, "passed": False, "shutdown_complete": False, "smoke": args.smoke, "pid": os.getpid(), } + if group is not None: + summary["group"] = group.binding + summary["actual_arm"] = args.arm os.environ.update( { "SIE_TELEMETRY_DISABLED": "1", @@ -709,8 +751,12 @@ async def run(args: argparse.Namespace) -> dict[str, Any]: torch.set_num_threads(THREADS) torch.set_num_interop_threads(1) try: + if group is not None: + await group.grant("prepare") + validate_source(group.binding, args.source, args.arm, args.pair) summary["environment"] = preflight(args, evidence) - async with asyncio.timeout(args.deadline_s): + timeout = max(0, group.binding["work_deadline"] - time.monotonic()) if group is not None else args.deadline_s + async with asyncio.timeout(timeout): await experiment(args, evidence, summary) except BaseException as error: summary["passed"] = False @@ -720,10 +766,12 @@ async def run(args: argparse.Namespace) -> dict[str, Any]: finally: atomic_json(evidence.directory / "trial.json", summary) evidence.emit("finished", passed=summary["passed"], shutdown_complete=summary["shutdown_complete"]) + if group is not None and summary["passed"] and summary["shutdown_complete"]: + group.acknowledge(terminal_sha256=digest(evidence.directory / "trial.json")) return summary -def main() -> None: +def arguments(argv: list[str] | None = None) -> argparse.Namespace: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--source", type=Path, required=True) parser.add_argument("--fixtures", type=Path, required=True) @@ -733,12 +781,16 @@ def main() -> None: parser.add_argument("--pair", type=int, default=0) parser.add_argument("--deadline-s", type=float, default=MAX_TRIAL_SECONDS) parser.add_argument("--smoke", action="store_true") - args = parser.parse_args() + args = parser.parse_args(argv) require( 0 < args.deadline_s <= MAX_TRIAL_SECONDS, f"worker deadline must be in (0, {MAX_TRIAL_SECONDS}]", ) - asyncio.run(run(args)) + return args + + +def main() -> None: + asyncio.run(run(arguments())) if __name__ == "__main__": From 255ccd5d92b2c51c9e22b5c985739fc1e3b495e7 Mon Sep 17 00:00:00 2001 From: svonava Date: Sat, 17 Oct 2026 15:17:23 -0700 Subject: [PATCH 15/16] test(benchmarks): guard pilot checks on shallow history --- tools/benchmarks/cross_encoder_lora/README.md | 2 +- .../cross_encoder_lora/tests/test_interleaved.py | 8 ++++++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/tools/benchmarks/cross_encoder_lora/README.md b/tools/benchmarks/cross_encoder_lora/README.md index dac2ebff3..1482be0e5 100644 --- a/tools/benchmarks/cross_encoder_lora/README.md +++ b/tools/benchmarks/cross_encoder_lora/README.md @@ -296,7 +296,7 @@ preparation above, the prospective baseline-only invocation is: ```console mise exec -- uv run --no-sync python -m tools.benchmarks.cross_encoder_lora.interleaved \ - --inputs /tmp/ce-inputs --fixtures /tmp/ce-fixtures --output /tmp/ce-baseline-pilot \ + --inputs /tmp/ce-sources --fixtures /tmp/ce-fixtures --output /tmp/ce-baseline-pilot \ --mode baseline-pilot --deadline-s 3900 --group-work-s 600 ``` diff --git a/tools/benchmarks/cross_encoder_lora/tests/test_interleaved.py b/tools/benchmarks/cross_encoder_lora/tests/test_interleaved.py index 703720d2a..b6e8c2d78 100644 --- a/tools/benchmarks/cross_encoder_lora/tests/test_interleaved.py +++ b/tools/benchmarks/cross_encoder_lora/tests/test_interleaved.py @@ -28,6 +28,12 @@ from tools.benchmarks.cross_encoder_lora.tests.test_probe import tiny_snapshot +def require_baseline_history() -> None: + available = subprocess.run(["git", "cat-file", "-e", BASE_REVISION], cwd=ROOT, capture_output=True, check=False) # noqa: S607 + if available.returncode: + pytest.skip("immutable archive smoke requires the documented baseline commit in git history") + + def test_strict_group_admission() -> None: assert not interleaved.admission(100, 789)["admitted"] assert not interleaved.admission(100, 790)["admitted"] @@ -120,6 +126,7 @@ def test_manifest_and_extracted_source_hashes_checked_before_load(tmp_path: Path @pytest.mark.parametrize("mode", ["baseline-pilot", "qualification"]) def test_real_archived_whole_phase_triplet(tmp_path: Path, mode: str) -> None: + require_baseline_history() snapshot = tiny_snapshot(tmp_path / "model") fixtures = tmp_path / "fixtures" fixtures_prepare(fixtures, snapshot, smoke=True) @@ -295,6 +302,7 @@ def observed(path): def test_actual_coordinator_term_reaps_archived_workers_and_sdk_children(tmp_path: Path) -> None: + require_baseline_history() snapshot = tiny_snapshot(tmp_path / "model") fixtures = tmp_path / "fixtures" fixtures_prepare(fixtures, snapshot, smoke=True) From fc7eeac318d429a34b9660311d72007fae077c25 Mon Sep 17 00:00:00 2001 From: svonava Date: Sat, 17 Oct 2026 16:51:26 -0700 Subject: [PATCH 16/16] feat(adapters): make CrossEncoder runtime LoRA explicitly opt in --- .../adapters/cross_encoder/__init__.py | 74 +-------- .../adapters/cross_encoder/runtime_lora.py | 88 +++++++++++ .../tests/adapters/test_cross_encoder_lora.py | 147 ++++++++++++++++-- tools/benchmarks/cross_encoder_lora/README.md | 48 +++++- .../benchmarks/cross_encoder_lora/adapter.py | 12 ++ .../cross_encoder_lora/runtime_adapter.py | 9 ++ .../cross_encoder_lora/stability.py | 2 +- .../tests/test_forward_observer.py | 31 ++++ .../tests/test_stability.py | 19 ++- .../cross_encoder_lora/tests/test_storage.py | 4 +- tools/benchmarks/cross_encoder_lora/worker.py | 24 +-- 11 files changed, 355 insertions(+), 103 deletions(-) create mode 100644 packages/sie_server/src/sie_server/adapters/cross_encoder/runtime_lora.py create mode 100644 tools/benchmarks/cross_encoder_lora/runtime_adapter.py diff --git a/packages/sie_server/src/sie_server/adapters/cross_encoder/__init__.py b/packages/sie_server/src/sie_server/adapters/cross_encoder/__init__.py index 9b5c96ba1..4104fffe2 100644 --- a/packages/sie_server/src/sie_server/adapters/cross_encoder/__init__.py +++ b/packages/sie_server/src/sie_server/adapters/cross_encoder/__init__.py @@ -17,7 +17,7 @@ """ from pathlib import Path -from typing import TYPE_CHECKING, Any +from typing import Any import torch from sentence_transformers import CrossEncoder @@ -28,10 +28,6 @@ from sie_server.core.inference_output import ScoreOutput from sie_server.types.inputs import Item -if TYPE_CHECKING: - from sie_server.adapters.cross_encoder._lora import CrossEncoderLora - from sie_server.adapters.lora import LoraRuntimeConfig - class CrossEncoderAdapter(BaseAdapter): """Adapter for cross-encoder reranker models. @@ -44,15 +40,6 @@ class CrossEncoderAdapter(BaseAdapter): These models take (query, document) pairs and output relevance scores. Higher scores indicate more relevant documents. - Runtime LoRA is opt-in through ``lora_paths={served_name: source}``. - Only ordinary A/B LoRA matrices targeting encoder Linear modules in a - standard, unquantized ``BertForSequenceClassification`` are supported. - The scoring head and activation stay those of the base checkpoint; - trained replacement heads, causal rerankers and custom model classes - are rejected when LoRA is enabled. Adapters load on demand under the - registry's resident limit. ``options.lora=None`` selects the base model. - Configurations without declarations use the original model and scoring path. - Performance optimizations: - SDPA attention (default): Uses PyTorch's scaled_dot_product_attention which can dispatch to flash attention when available @@ -74,9 +61,6 @@ def __init__( compute_precision: ComputePrecision | None = None, attn_implementation: AttnImplementation = "sdpa", revision: str | None = None, - lora_paths: dict[str, str] | None = None, - lora_revisions: dict[str, str | None] | None = None, - max_loaded_loras: int = 10, **kwargs: Any, # Accept extra args from loader (e.g., pooling) ) -> None: """Initialize the adapter. @@ -92,12 +76,6 @@ def __init__( revision: Optional HuggingFace revision/branch/commit SHA to pin when loading the model. If None, the default branch is used. Forwarded to ``CrossEncoder(..., revision=...)``. - lora_paths: Declared adapter aliases/sources. Enables deferred runtime - loading for standard BERT encoder projections only. Served names - use letters, digits, underscores, dots and hyphens; sources may - be local directories or Hugging Face repositories. - lora_revisions: Source revisions supplied by the model loader. - max_loaded_loras: Resident adapter limit, enforced with registry LRU eviction. **kwargs: Additional arguments (ignored, for compatibility). """ _ = kwargs # Unused, but accepted for loader compatibility @@ -110,12 +88,6 @@ def __init__( self._model: CrossEncoder | None = None self._device: str | None = None - self._lora: CrossEncoderLora | None = None - if lora_paths: - # Keep optional dependency imports and all LoRA state off the ordinary path. - from sie_server.adapters.cross_encoder._lora import CrossEncoderLora as LoraManager - - self._lora = LoraManager(lora_paths, lora_revisions, max_loaded_loras) def load(self, device: str) -> None: """Load the model onto the specified device. @@ -162,50 +134,6 @@ def load(self, device: str) -> None: if self._max_length is not None: self._model.max_length = self._max_length - if self._lora is not None: - self._lora.bind(self._model) - - def unload(self) -> None: - try: - if self._lora is not None: - self._lora.close() - finally: - super().unload() - - def supports_lora(self) -> bool: - return self._lora is not None - - def supports_hot_lora_reload(self) -> bool: - return self._lora is not None - - def lora_runtime_config(self) -> "LoraRuntimeConfig | None": - return self._lora.runtime_config() if self._lora is not None else None - - def fetch_lora(self, lora_path: str, revision: str | None = None) -> str: - if self._lora is None: - return super().fetch_lora(lora_path, revision) - return self._lora.fetch(lora_path, revision) - - def discard_lora_source(self, lora_path: str) -> None: - if self._lora is not None: - self._lora.discard(lora_path) - - def load_lora(self, lora_path: str, revision: str | None = None, *, source: str | None = None) -> int: - if self._lora is None: - return super().load_lora(lora_path, revision, source=source) - return self._lora.load(lora_path, revision, source=source) - - def unload_lora(self, lora_name: str) -> None: - if self._lora is None: - super().unload_lora(lora_name) - else: - self._lora.unload(lora_name) - - def set_active_lora(self, lora_name: str | None) -> None: - if self._lora is not None: - self._lora.select(lora_name) - elif lora_name is not None: - raise ValueError("No LoRA sources are declared for this cross-encoder") def score( self, diff --git a/packages/sie_server/src/sie_server/adapters/cross_encoder/runtime_lora.py b/packages/sie_server/src/sie_server/adapters/cross_encoder/runtime_lora.py new file mode 100644 index 000000000..f042d478e --- /dev/null +++ b/packages/sie_server/src/sie_server/adapters/cross_encoder/runtime_lora.py @@ -0,0 +1,88 @@ +"""Explicit runtime-LoRA adapter; ordinary CrossEncoder profiles stay unchanged.""" + +from pathlib import Path +from typing import Any + +from sie_server.adapters._types import AttnImplementation, ComputePrecision +from sie_server.adapters.cross_encoder import CrossEncoderAdapter +from sie_server.adapters.cross_encoder._lora import CrossEncoderLora +from sie_server.adapters.lora import LoraRuntimeConfig + + +class CrossEncoderLoraAdapter(CrossEncoderAdapter): + """Opt-in runtime LoRA for a standard, unquantized BERT cross-encoder. + + Select ``sie_server.adapters.cross_encoder.runtime_lora:CrossEncoderLoraAdapter`` + explicitly and declare ``lora_paths={served_name: source}``. Only ordinary + A/B matrices targeting BERT encoder Linear modules are accepted. The outer + CrossEncoder, classifier, activation and scoring methods are inherited; + trained replacement heads and custom/causal reranker classes are rejected. + + ``options.lora=None`` restores the original base modules while adapters stay + resident off-tree. Loading, selection and eviction use the worker's existing + mutation guard. This lifecycle contract does not establish zero overhead or + retrieval quality. Ordinary ``CrossEncoderAdapter`` profiles do not import + this module or construct any of its state. + """ + + def __init__( + self, + model_name_or_path: str | Path, + *, + trust_remote_code: bool = False, + max_length: int | None = None, + compute_precision: ComputePrecision | None = None, + attn_implementation: AttnImplementation = "sdpa", + revision: str | None = None, + lora_paths: dict[str, str] | None = None, + lora_revisions: dict[str, str | None] | None = None, + max_loaded_loras: int = 10, + **kwargs: Any, + ) -> None: + if not lora_paths: + raise ValueError("CrossEncoderLoraAdapter requires nonempty lora_paths declarations") + super().__init__( + model_name_or_path, + trust_remote_code=trust_remote_code, + max_length=max_length, + compute_precision=compute_precision, + attn_implementation=attn_implementation, + revision=revision, + **kwargs, + ) + self._lora = CrossEncoderLora(lora_paths, lora_revisions, max_loaded_loras) + + def load(self, device: str) -> None: + super().load(device) + assert self._model is not None + self._lora.bind(self._model) + + def unload(self) -> None: + try: + self._lora.close() + finally: + super().unload() + + def supports_lora(self) -> bool: + return True + + def supports_hot_lora_reload(self) -> bool: + return True + + def lora_runtime_config(self) -> LoraRuntimeConfig: + return self._lora.runtime_config() + + def fetch_lora(self, lora_path: str, revision: str | None = None) -> str: + return self._lora.fetch(lora_path, revision) + + def discard_lora_source(self, lora_path: str) -> None: + self._lora.discard(lora_path) + + def load_lora(self, lora_path: str, revision: str | None = None, *, source: str | None = None) -> int: + return self._lora.load(lora_path, revision, source=source) + + def unload_lora(self, lora_name: str) -> None: + self._lora.unload(lora_name) + + def set_active_lora(self, lora_name: str | None) -> None: + self._lora.select(lora_name) diff --git a/packages/sie_server/tests/adapters/test_cross_encoder_lora.py b/packages/sie_server/tests/adapters/test_cross_encoder_lora.py index 26bd47253..6defe57df 100644 --- a/packages/sie_server/tests/adapters/test_cross_encoder_lora.py +++ b/packages/sie_server/tests/adapters/test_cross_encoder_lora.py @@ -5,6 +5,8 @@ import builtins import gc import json +import subprocess +import sys import time import weakref from concurrent.futures import ThreadPoolExecutor @@ -18,10 +20,14 @@ from peft import LoraConfig from peft.tuners.tuners_utils import BaseTunerLayer from safetensors.torch import load_file, save_file +from sie_server.adapters._base_adapter import BaseAdapter from sie_server.adapters.cross_encoder import CrossEncoderAdapter +from sie_server.adapters.cross_encoder.runtime_lora import CrossEncoderLoraAdapter from sie_server.adapters.lora import LoraStateUncertainError, LoraUnavailableError from sie_server.app.app_factory import AppFactory from sie_server.app.app_state_config import AppStateConfig +from sie_server.config.model import ModelConfig +from sie_server.core.loader import load_adapter from sie_server.core.registry import ModelRegistry from transformers import BertConfig, BertForSequenceClassification, BertTokenizerFast @@ -92,7 +98,7 @@ def save_adapter(path, seed=1, *, targets=("query", "value"), **overrides): @pytest.fixture def adapter(checkpoint, tmp_path): paths = {name: save_adapter(tmp_path / name, seed) for seed, name in enumerate(("a", "b", "c"), 1)} - adapter = CrossEncoderAdapter(checkpoint, lora_paths=paths, max_loaded_loras=2) + adapter = CrossEncoderLoraAdapter(checkpoint, lora_paths=paths, max_loaded_loras=2) adapter.load("cpu") yield adapter adapter.unload() @@ -149,12 +155,12 @@ def test_never_lora_has_no_manager_or_optional_imports(checkpoint, monkeypatch): original = builtins.__import__ def guarded_import(name, *args, **kwargs): - assert name != "sie_server.adapters.cross_encoder._lora" + assert name not in {"sie_server.adapters.cross_encoder._lora", "sie_server.adapters.cross_encoder.runtime_lora"} return original(name, *args, **kwargs) monkeypatch.setattr(builtins, "__import__", guarded_import) adapter = CrossEncoderAdapter(checkpoint) - assert adapter._lora is None + assert not hasattr(adapter, "_lora") assert not adapter.supports_lora() assert adapter.lora_capability() is None assert not adapter.supports_hot_lora_reload() @@ -165,11 +171,94 @@ def guarded_import(name, *args, **kwargs): adapter.set_active_lora(None) np.testing.assert_array_equal(scores(adapter), baseline) assert_base_preserved(adapter, before) - with pytest.raises(ValueError, match="No LoRA sources"): - adapter.set_active_lora("a") + # Preserve the original base hook contract; registry capability checks + # reject adapter requests before any model mutation. + adapter.set_active_lora("a") + np.testing.assert_array_equal(scores(adapter), baseline) adapter.unload() +def test_ordinary_module_import_and_hooks_stay_independent_of_runtime_lora(tmp_path): + script = """ +import builtins, sys +original = builtins.__import__ +def guarded(name, *args, **kwargs): + assert name not in { + 'sie_server.adapters.cross_encoder.runtime_lora', + 'sie_server.adapters.cross_encoder._lora', + }, name + return original(name, *args, **kwargs) +builtins.__import__ = guarded +from sie_server.adapters._base_adapter import BaseAdapter +from sie_server.adapters.cross_encoder import CrossEncoderAdapter +adapter = CrossEncoderAdapter('unused-local-construction-only', lora_paths={'a': 'unused'}) +assert not hasattr(adapter, '_lora') +for name in ('unload', 'supports_lora', 'supports_hot_lora_reload', 'lora_runtime_config', + 'fetch_lora', 'discard_lora_source', 'load_lora', 'unload_lora', 'set_active_lora'): + assert getattr(CrossEncoderAdapter, name) is getattr(BaseAdapter, name), name +assert not adapter.supports_lora() +assert 'sie_server.adapters.cross_encoder.runtime_lora' not in sys.modules +assert 'sie_server.adapters.cross_encoder._lora' not in sys.modules +""" + result = subprocess.run( # noqa: S603 -- fixed local import-only check, no model or network + [sys.executable, "-c", script], cwd=tmp_path, capture_output=True, text=True, timeout=45, check=False + ) + assert result.returncode == 0, result.stdout + result.stderr + + +@pytest.mark.parametrize("declarations", [None, {}, []]) +def test_explicit_runtime_class_requires_declarations(checkpoint, declarations): + with pytest.raises(ValueError, match="requires nonempty lora_paths"): + CrossEncoderLoraAdapter(checkpoint, lora_paths=declarations, lora_revisions={"org/other": "a" * 40}) + + +@pytest.mark.parametrize("runtime", [False, True]) +def test_real_loader_routes_only_the_explicit_profile_and_preserves_base_options(checkpoint, runtime): + adapter_type = CrossEncoderLoraAdapter if runtime else CrossEncoderAdapter + loadtime = {"max_length": 24, "trust_remote_code": True} + if runtime: + loadtime["lora_paths"] = {"a": {"id": "org/declared", "revision": "b" * 40}} + config = ModelConfig.model_validate( + { + "sie_id": "test/explicit-cross-encoder", + "weights_path": str(checkpoint), + "hf_revision": "a" * 40, + "inputs": {"text": True}, + "tasks": {"score": {}}, + "profiles": { + "default": { + "adapter_path": f"{adapter_type.__module__}:{adapter_type.__name__}", + "max_batch_tokens": 8192, + "adapter_options": {"loadtime": loadtime}, + } + }, + } + ) + adapter = load_adapter(config, checkpoint, device="cpu", attention_backend="eager", max_loras_per_model=2) + assert type(adapter) is adapter_type + assert adapter._model_name_or_path == str(checkpoint) + assert adapter._max_length == 24 + assert adapter._trust_remote_code + assert adapter._compute_precision == "float32" + assert adapter._attn_implementation == "eager" + assert adapter._revision == "a" * 40 + assert adapter_type.score is CrossEncoderAdapter.score + assert adapter_type.score_pairs is CrossEncoderAdapter.score_pairs + if runtime: + assert adapter._lora._revisions == {"org/declared": "b" * 40} + assert adapter.lora_runtime_config().max_loaded == 2 + assert adapter.lora_runtime_config().aliases == {"a": "org/declared"} + else: + assert not hasattr(adapter, "_lora") + assert adapter.set_active_lora.__func__ is BaseAdapter.set_active_lora + # Profile resolution is cached. Construct a distinct invalid config, + # rather than mutating the already-resolved instance behind its cache. + invalid = config.model_dump() + invalid["profiles"]["default"]["adapter_options"]["loadtime"]["lora_paths"] = {"a": "org/declared"} + with pytest.raises(ValueError, match=r"does not accept.*lora_paths"): + load_adapter(ModelConfig.model_validate(invalid), checkpoint, device="cpu") + + def test_actual_effect_restoration_head_and_final_module_identity(adapter): snapshot = base_snapshot(adapter) baseline = scores(adapter) @@ -216,6 +305,7 @@ def test_resident_adapters_leave_the_exact_base_tree_and_preserve_layer_hooks(ad snapshot = base_snapshot(adapter) baseline = scores(adapter) original_query = snapshot[1].bert.encoder.layer[0].attention.self.query + original_storage = [(parameter, parameter.data_ptr()) for parameter in snapshot[1].parameters()] original_calls, wrapper_calls = [], [] base_hook = original_query.register_forward_hook(lambda *_: original_calls.append(True)) adapter.load_lora("a") @@ -246,18 +336,53 @@ def test_resident_adapters_leave_the_exact_base_tree_and_preserve_layer_hooks(ad np.testing.assert_array_equal(actual, expected[selection]) expected[selection] = actual assert_base_preserved(adapter, snapshot) + assert all(parameter.data_ptr() == pointer for parameter, pointer in original_storage) assert {name: id(layer) for name, layer in manager._resident_layers.items()} == identities assert np.max(np.abs(expected["a"] - expected["b"])) > 1e-6 base_hook.remove() wrapper_hook.remove() +def test_repeated_disabled_selection_checks_readiness_without_tree_or_peft_work(adapter, monkeypatch): + adapter.load_lora("a") + adapter.load_lora("b") + adapter.set_active_lora(None) + baseline = scores(adapter) + snapshot = base_snapshot(adapter) + manager = adapter._lora + original_ready = manager._ready + checks = [] + + def ready(): + checks.append(True) + original_ready() + + def unexpected(*_args, **_kwargs): + pytest.fail("disabled selection traversed or changed adapter layers") + + with monkeypatch.context() as context: + context.setattr(manager, "_ready", ready) + for name in ("_attach_layers", "_detach_layers", "_restore_grad_flags"): + context.setattr(manager, name, unexpected) + for name in ("set_adapter", "enable_adapters", "disable_adapters"): + context.setattr(adapter._model, name, unexpected) + for _ in range(100): + adapter.set_active_lora(None) + assert len(checks) == 100 + np.testing.assert_array_equal(scores(adapter), baseline) + assert dict(snapshot[1].named_modules()) == snapshot[5] + manager._uncertain = True + with pytest.raises(LoraStateUncertainError): + adapter.set_active_lora(None) + manager._uncertain = False + + def test_disjoint_targets_evict_only_their_wrappers_and_final_unload_releases_weights(checkpoint, tmp_path): paths = { "a": save_adapter(tmp_path / "a", 1, targets=("query",)), "b": save_adapter(tmp_path / "b", 2, targets=("value",)), } - adapter = CrossEncoderAdapter(checkpoint, lora_paths=paths, max_loaded_loras=2) + adapter = CrossEncoderLoraAdapter(checkpoint, lora_paths=paths, max_loaded_loras=2) adapter.load("cpu") try: snapshot, baseline = base_snapshot(adapter), scores(adapter) @@ -522,8 +647,8 @@ def test_discard_and_model_unload_release_owned_sources(adapter): def test_revision_and_capacity_validation(checkpoint, tmp_path): path = save_adapter(tmp_path / "a") with pytest.raises(ValueError, match="positive integer"): - CrossEncoderAdapter(checkpoint, lora_paths={"a": path}, max_loaded_loras=0) - adapter = CrossEncoderAdapter(checkpoint, lora_paths={"a": path}, lora_revisions={path: "a" * 40}) + CrossEncoderLoraAdapter(checkpoint, lora_paths={"a": path}, max_loaded_loras=0) + adapter = CrossEncoderLoraAdapter(checkpoint, lora_paths={"a": path}, lora_revisions={path: "a" * 40}) adapter.load("cpu") try: with pytest.raises(ValueError, match="differs"): @@ -536,14 +661,14 @@ def test_revision_and_capacity_validation(checkpoint, tmp_path): def test_revisions_alone_do_not_enable_or_authorize_lora(checkpoint): adapter = CrossEncoderAdapter(checkpoint, lora_revisions={"other-profile/source": "a" * 40}) - assert adapter._lora is None + assert not hasattr(adapter, "_lora") assert adapter.lora_capability() is None @pytest.mark.parametrize("paths", [["org/source"], {"/private/source": "org/source"}, {"a": "b", "b": "c"}]) def test_aliases_are_public_names_and_cannot_shadow_sources(checkpoint, paths): with pytest.raises(ValueError, match=r"aliases|shadows"): - CrossEncoderAdapter(checkpoint, lora_paths=paths) + CrossEncoderLoraAdapter(checkpoint, lora_paths=paths) def test_fetch_rechecks_model_generation_before_publishing_source(adapter, monkeypatch): @@ -669,7 +794,7 @@ def test_real_score_routes_authorize_aliases_isolate_batches_and_evict( "max_sequence_length": 64, "profiles": { "default": { - "adapter_path": "sie_server.adapters.cross_encoder:CrossEncoderAdapter", + "adapter_path": "sie_server.adapters.cross_encoder.runtime_lora:CrossEncoderLoraAdapter", "max_batch_tokens": 8192, "adapter_options": {"loadtime": {"lora_paths": {name: paths[name] for name in ("a", "b", "c")}}}, }, diff --git a/tools/benchmarks/cross_encoder_lora/README.md b/tools/benchmarks/cross_encoder_lora/README.md index 1482be0e5..17dd57c01 100644 --- a/tools/benchmarks/cross_encoder_lora/README.md +++ b/tools/benchmarks/cross_encoder_lora/README.md @@ -1,7 +1,8 @@ # CrossEncoder runtime LoRA qualification -This probe measures the ordinary `CrossEncoderAdapter` using the real SIE loader, -worker, loopback HTTP API and `sie_sdk.SIEClient`. It does not provision compute, +This probe measures the ordinary `CrossEncoderAdapter` and the explicit opt-in +`CrossEncoderLoraAdapter` using the real SIE loader, worker, loopback HTTP API +and `sie_sdk.SIEClient`. It does not provision compute, install dependencies, change precision, or change the production catalog. The catalog model is overridden locally to use the ordinary adapter rather than the catalog's BertFlash implementation. @@ -14,6 +15,46 @@ every BERT encoder query/value projection. They establish reversible numerical effects and lifecycle behavior, **not retrieval quality**. Source seeds, model files and adapter files are hashed in the fixture manifest. +## Explicit opt-in adapter + +Ordinary profiles keep `sie_server.adapters.cross_encoder:CrossEncoderAdapter`. +That module is restored exactly to the pinned baseline source, including its +constructor, load/unload and inherited LoRA hooks. It does not import the +runtime-LoRA module or construct a manager. The public catalog and fallback +paths are unchanged. + +Experimental runtime profiles must select the separate class explicitly: + +```yaml +profiles: + default: + adapter_path: sie_server.adapters.cross_encoder.runtime_lora:CrossEncoderLoraAdapter + max_batch_tokens: 8192 + adapter_options: + loadtime: + lora_paths: + task_adapter: + id: example/bert-encoder-lora + revision: 0123456789abcdef0123456789abcdef01234567 +``` + +The repository and revision above are placeholders. Use declared, compatible +adapter weights. The explicit class requires nonempty declarations; revisions +alone do not enable it. Older experimental PR775 configurations that added +`lora_paths` to the ordinary class must change their `adapter_path`; the existing +loader rejects those options on the ordinary class. Dependent architecture +extensions must likewise opt into the separate class and retain their own +qualification requirements; this change does not add XLM-R support. + +The opt-in class inherits the ordinary scoring methods. With base selected, its +manager restores the original module objects and retains LoRA layers in a plain +off-tree dictionary, without copying the base weights. Repeated base selection +still checks manager readiness and active state; declared profiles retain their +existing admission, reservation and mutation controls. These structural facts +do not prove equal end-to-end latency or throughput for enabled profiles. All +previous failed measurements and performance gates remain in force; the +baseline-only pilot is not candidate qualification. + ## Prepare immutable sources and fixtures From a clean public checkout with the candidate committed: @@ -66,7 +107,8 @@ drop failed trials to obtain a pass, or retry the same failed protocol unchanged There are six fresh-process blocks, in order `BNR`, `NRB`, `RBN`, `BRN`, `RNB`, `NBR` (18 fresh servers). B uses the pinned baseline with no declared LoRA. -N uses the candidate with no declared LoRA. R uses the candidate after A/B +N uses the candidate's ordinary class with no declared LoRA. R uses its explicit +runtime-LoRA class after A/B activity, with base selected and two adapters resident but disabled. Every arm uses the same password-reset query and four documents in `protocol.py`. diff --git a/tools/benchmarks/cross_encoder_lora/adapter.py b/tools/benchmarks/cross_encoder_lora/adapter.py index a9c6ce828..68c80c6c1 100644 --- a/tools/benchmarks/cross_encoder_lora/adapter.py +++ b/tools/benchmarks/cross_encoder_lora/adapter.py @@ -14,6 +14,16 @@ INPUT_DIMENSIONS = 2 +def production_class(arm: str) -> type[CrossEncoderAdapter]: + require(arm in {"B", "N", "R"}, "unknown benchmark arm") + if arm == "R": + # Optional production adapter does not exist in the archived B source. + from sie_server.adapters.cross_encoder.runtime_lora import CrossEncoderLoraAdapter # noqa: PLC0415 + + return CrossEncoderLoraAdapter + return CrossEncoderAdapter + + def require_forward_coverage(batch: dict[str, Any]) -> None: """An outer score batch alone does not prove the model forward was observed.""" forwards = batch["forwards"] @@ -47,6 +57,8 @@ def require_forward_selection(batch: dict[str, Any], expected_adapter: str | Non class ObservedCrossEncoder(CrossEncoderAdapter): + production_adapter_class = CrossEncoderAdapter + def load(self, device: str) -> None: self.batch_observations = [] self.selection_observations = [] diff --git a/tools/benchmarks/cross_encoder_lora/runtime_adapter.py b/tools/benchmarks/cross_encoder_lora/runtime_adapter.py new file mode 100644 index 000000000..1146de478 --- /dev/null +++ b/tools/benchmarks/cross_encoder_lora/runtime_adapter.py @@ -0,0 +1,9 @@ +"""Untimed observation of the explicitly selected runtime-LoRA production class.""" + +from sie_server.adapters.cross_encoder.runtime_lora import CrossEncoderLoraAdapter + +from tools.benchmarks.cross_encoder_lora.adapter import ObservedCrossEncoder + + +class ObservedRuntimeCrossEncoder(ObservedCrossEncoder, CrossEncoderLoraAdapter): + production_adapter_class = CrossEncoderLoraAdapter diff --git a/tools/benchmarks/cross_encoder_lora/stability.py b/tools/benchmarks/cross_encoder_lora/stability.py index 479900b66..beac02760 100644 --- a/tools/benchmarks/cross_encoder_lora/stability.py +++ b/tools/benchmarks/cross_encoder_lora/stability.py @@ -414,7 +414,7 @@ async def child(args: argparse.Namespace) -> dict[str, Any]: torch.set_num_interop_threads(1) try: summary["required_telemetry_preflight"] = snapshot() - environment = worker.preflight(args, evidence) + environment = worker.preflight(args, evidence, actual_arm="B") environment["hardware_preflight_only"] = environment.pop("qualification") environment["qualification"] = False environment["source_imports"] = {} diff --git a/tools/benchmarks/cross_encoder_lora/tests/test_forward_observer.py b/tools/benchmarks/cross_encoder_lora/tests/test_forward_observer.py index 46c4d0f25..ec9b01812 100644 --- a/tools/benchmarks/cross_encoder_lora/tests/test_forward_observer.py +++ b/tools/benchmarks/cross_encoder_lora/tests/test_forward_observer.py @@ -9,6 +9,8 @@ import pytest from sie_server.adapters.cross_encoder import CrossEncoderAdapter +from sie_server.adapters.cross_encoder.runtime_lora import CrossEncoderLoraAdapter +from sie_server.adapters.lora import LoraStateUncertainError from sie_server.types.inputs import Item from tools.benchmarks.cross_encoder_lora.adapter import ( @@ -16,6 +18,8 @@ require_forward_coverage, require_forward_selection, ) +from tools.benchmarks.cross_encoder_lora.fixtures import prepare +from tools.benchmarks.cross_encoder_lora.runtime_adapter import ObservedRuntimeCrossEncoder from tools.benchmarks.cross_encoder_lora.tests.test_probe import tiny_snapshot from tools.benchmarks.cross_encoder_lora.worker import unobserved @@ -137,6 +141,33 @@ def observed_batch() -> dict: } +def test_runtime_timing_uses_actual_opt_in_selection_and_preserves_poison_refusal(tmp_path: Path) -> None: + snapshot = tiny_snapshot(tmp_path / "model") + fixtures = tmp_path / "fixtures" + prepare(fixtures, snapshot, smoke=True) + adapter = ObservedRuntimeCrossEncoder(snapshot, lora_paths={"a": str(fixtures / "a")}) + adapter.load("cpu") + try: + adapter.load_lora("a") + before = len(adapter.selection_observations) + with unobserved(adapter): + assert adapter.production_adapter_class is CrossEncoderLoraAdapter + assert adapter.score_pairs.__func__ is CrossEncoderAdapter.score_pairs + assert adapter.set_active_lora.__func__ is CrossEncoderLoraAdapter.set_active_lora + adapter.set_active_lora("a") + assert adapter._lora._active == str(fixtures / "a") + adapter.set_active_lora(None) + assert adapter._lora._active is None + adapter.score_pairs(*pairs()) + assert len(adapter.selection_observations) == before + adapter._lora._uncertain = True + with pytest.raises(LoraStateUncertainError), unobserved(adapter): + adapter.set_active_lora(None) + assert adapter._observer_wrapper is not None + finally: + adapter.unload() + + @pytest.mark.parametrize("failure", ["empty", "rows", "logits", "nonfinite", "wrong_adapter", "enabled_base"]) def test_missing_partial_or_wrong_tuner_observation_cannot_qualify(failure: str) -> None: batch = copy.deepcopy(observed_batch()) diff --git a/tools/benchmarks/cross_encoder_lora/tests/test_stability.py b/tools/benchmarks/cross_encoder_lora/tests/test_stability.py index f0973a6e8..044c00657 100644 --- a/tools/benchmarks/cross_encoder_lora/tests/test_stability.py +++ b/tools/benchmarks/cross_encoder_lora/tests/test_stability.py @@ -349,7 +349,7 @@ def test_negative_raw_latency_cannot_hide_in_positive_aggregates() -> None: probe.metrics(phase) -def test_serving_observer_and_gc_are_unmodified() -> None: +def test_memory_and_observer_flow_preserved_with_explicit_production_class() -> None: baseline = subprocess.run( ["git", "show", "a2932d81:tools/benchmarks/cross_encoder_lora/worker.py"], # noqa: S607 cwd=ROOT, @@ -361,13 +361,28 @@ def test_serving_observer_and_gc_are_unmodified() -> None: pytest.skip("invariant control requires its documented parent commit in git history") def code(text: str) -> dict[str, str]: + class ClassBinding(ast.NodeTransformer): + def visit_Name(self, node): + if node.id == "production": + node.id = "CrossEncoderAdapter" + return node + + tree = ast.parse(text) + for node in tree.body: + if isinstance(node, ast.FunctionDef) and node.name == "unobserved": + binding = ast.parse("production = adapter.production_adapter_class").body[0] + node.body = [entry for entry in node.body if ast.dump(entry) != ast.dump(binding)] + ClassBinding().visit(node) return { node.name: ast.dump(node) - for node in ast.parse(text).body + for node in tree.body if isinstance(node, ast.FunctionDef) and node.name in {"unobserved", "memory"} } assert code(Path(worker.__file__).read_text()) == code(baseline.stdout) + # The explicit opt-in subclass now supplies selection; all timing/restore + # control flow and memory semantics stay the same. Real R forwarding is + # checked separately, including refusal of a poisoned resident manager. # Historical diagnostic timing still uses the in-process wrapper and does # not force GC. The new qualification load loop has its own extraction and # clock-boundary tests; exact historical replay remains pinned to 482b. diff --git a/tools/benchmarks/cross_encoder_lora/tests/test_storage.py b/tools/benchmarks/cross_encoder_lora/tests/test_storage.py index 976d953a7..0be591cf1 100644 --- a/tools/benchmarks/cross_encoder_lora/tests/test_storage.py +++ b/tools/benchmarks/cross_encoder_lora/tests/test_storage.py @@ -4,7 +4,7 @@ import pytest import torch -from sie_server.adapters.cross_encoder import CrossEncoderAdapter +from sie_server.adapters.cross_encoder.runtime_lora import CrossEncoderLoraAdapter from tools.benchmarks.cross_encoder_lora.fixtures import prepare from tools.benchmarks.cross_encoder_lora.tests.test_probe import tiny_snapshot @@ -16,7 +16,7 @@ def resident(tmp_path): snapshot = tiny_snapshot(tmp_path / "model") fixtures = tmp_path / "fixtures" prepare(fixtures, snapshot, smoke=True) - adapter = CrossEncoderAdapter(snapshot, lora_paths={name: str(fixtures / name) for name in ("a", "b")}) + adapter = CrossEncoderLoraAdapter(snapshot, lora_paths={name: str(fixtures / name) for name in ("a", "b")}) adapter.load("cpu") adapter.load_lora("a") adapter.load_lora("b") diff --git a/tools/benchmarks/cross_encoder_lora/worker.py b/tools/benchmarks/cross_encoder_lora/worker.py index 8c4d21bf4..db7910e27 100644 --- a/tools/benchmarks/cross_encoder_lora/worker.py +++ b/tools/benchmarks/cross_encoder_lora/worker.py @@ -28,14 +28,13 @@ import uvicorn import yaml from sie_sdk.client.errors import RequestError -from sie_server.adapters.cross_encoder import CrossEncoderAdapter from sie_server.adapters.peft_lora_mixin import PEFTLoRAMixin from sie_server.app.app_factory import AppFactory from sie_server.app.app_state_config import AppStateConfig from sie_server.types.inputs import Item from tools.benchmarks.cross_encoder_lora import client as client_workload -from tools.benchmarks.cross_encoder_lora.adapter import require_forward_selection +from tools.benchmarks.cross_encoder_lora.adapter import production_class, require_forward_selection from tools.benchmarks.cross_encoder_lora.client import ( REQUEST_ITEMS, Client, @@ -208,15 +207,14 @@ def adapter_storage(adapter: Any) -> dict[str, int]: def unobserved(adapter: Any) -> Iterator[None]: """Time production methods with the original forward attribute lookup restored.""" score_pairs, select = adapter.score_pairs, adapter.set_active_lora + production = adapter.production_adapter_class before = (len(adapter.batch_observations), len(adapter.forward_observations), len(adapter.selection_observations)) adapter.remove_forward_observer() - adapter.score_pairs = CrossEncoderAdapter.score_pairs.__get__(adapter) - adapter.set_active_lora = CrossEncoderAdapter.set_active_lora.__get__(adapter) + adapter.score_pairs = production.score_pairs.__get__(adapter) + adapter.set_active_lora = production.set_active_lora.__get__(adapter) try: - require(adapter.score_pairs.__func__ is CrossEncoderAdapter.score_pairs, "timed scoring is instrumented") - require( - adapter.set_active_lora.__func__ is CrossEncoderAdapter.set_active_lora, "timed selection is instrumented" - ) + require(adapter.score_pairs.__func__ is production.score_pairs, "timed scoring is instrumented") + require(adapter.set_active_lora.__func__ is production.set_active_lora, "timed selection is instrumented") adapter.assert_original_forward() yield adapter.assert_original_forward() @@ -330,19 +328,21 @@ def configs(directory: Path, source: Path, fixtures: Path, arm: str, snapshot: P profile = config["profiles"]["default"] profile["adapter_path"] = f"{HERE / 'adapter.py'}:ObservedCrossEncoder" if arm == "R": + profile["adapter_path"] = f"{HERE / 'runtime_adapter.py'}:ObservedRuntimeCrossEncoder" profile["adapter_options"]["loadtime"] = { "lora_paths": {name: str(fixtures / name) for name in ("a", "b", "c")} } (directory / "model.yaml").write_text(yaml.safe_dump(config)) -def preflight(args: argparse.Namespace, evidence: Evidence) -> dict[str, Any]: +def preflight(args: argparse.Namespace, evidence: Evidence, *, actual_arm: str) -> dict[str, Any]: versions = {name: importlib.metadata.version(name) for name in EXPECTED_VERSIONS} try: torchao = importlib.metadata.version("torchao") except importlib.metadata.PackageNotFoundError: torchao = None - module_path = sys.modules[CrossEncoderAdapter.__module__].__file__ + production = production_class(actual_arm) + module_path = sys.modules[production.__module__].__file__ assert module_path is not None actual_source = Path(module_path).resolve() require(actual_source.is_relative_to(args.source.resolve()), "adapter was imported from a different source tree") @@ -351,6 +351,7 @@ def preflight(args: argparse.Namespace, evidence: Evidence) -> dict[str, Any]: "torchao": torchao, "adapter_source": str(actual_source), "adapter_sha256": digest(actual_source), + "adapter_class": f"{production.__module__}:{production.__name__}", "python": sys.version, "threads": torch.get_num_threads(), "device": str(torch.cuda.get_device_name(0)) if torch.cuda.is_available() else "cpu", @@ -424,6 +425,7 @@ async def experiment(args: argparse.Namespace, evidence: Evidence, summary: dict summary["loaded_before_forward_memory"] = memory() summary["manager_present"] = getattr(adapter, "_lora", None) is not None require(summary["manager_present"] == (args.arm == "R"), "unexpected manager activation") + require(adapter.production_adapter_class is production_class(args.arm), "wrong production adapter class") async def call(selection: str | None, label: str, *, cold: bool = False) -> dict[str, Any]: row = await asyncio.to_thread(client.score, selection, label, cold=cold) @@ -754,7 +756,7 @@ async def run(args: argparse.Namespace) -> dict[str, Any]: if group is not None: await group.grant("prepare") validate_source(group.binding, args.source, args.arm, args.pair) - summary["environment"] = preflight(args, evidence) + summary["environment"] = preflight(args, evidence, actual_arm=args.arm) timeout = max(0, group.binding["work_deadline"] - time.monotonic()) if group is not None else args.deadline_s async with asyncio.timeout(timeout): await experiment(args, evidence, summary)