|
| 1 | +"""Exclusive host-GPU handoff between the resident local reasoner and Motif jobs.""" |
| 2 | +from __future__ import annotations |
| 3 | + |
| 4 | +from contextlib import contextmanager |
| 5 | +import fcntl |
| 6 | +import os |
| 7 | +from pathlib import Path |
| 8 | +import subprocess |
| 9 | +import threading |
| 10 | +import time |
| 11 | +from typing import Callable, Iterator, Sequence |
| 12 | + |
| 13 | +import failures |
| 14 | + |
| 15 | + |
| 16 | +class SharedGpuCoordinator: |
| 17 | + """Stop the resident Qwen service while one admitted scientific GPU job runs. |
| 18 | +
|
| 19 | + The process lock serializes controller threads; the file lock extends the same |
| 20 | + exclusion boundary across API processes. The service is restarted only when |
| 21 | + this coordinator observed it active before the handoff. |
| 22 | + """ |
| 23 | + |
| 24 | + def __init__(self, *, lock_path: Path, service: str = "dirac-qwen.service", |
| 25 | + runner: Callable[..., subprocess.CompletedProcess] = subprocess.run, |
| 26 | + sleep: Callable[[float], None] = time.sleep, |
| 27 | + timeout_seconds: float = 120.0, |
| 28 | + poll_seconds: float = 1.0) -> None: |
| 29 | + self.lock_path = lock_path.resolve() |
| 30 | + self.service = service |
| 31 | + self._runner = runner |
| 32 | + self._sleep = sleep |
| 33 | + self.timeout_seconds = max(1.0, float(timeout_seconds)) |
| 34 | + self.poll_seconds = max(.05, float(poll_seconds)) |
| 35 | + self._thread_lock = threading.Lock() |
| 36 | + |
| 37 | + def _run(self, command: Sequence[str], *, check: bool = True |
| 38 | + ) -> subprocess.CompletedProcess: |
| 39 | + try: |
| 40 | + return self._runner( |
| 41 | + list(command), check=check, capture_output=True, text=True, |
| 42 | + timeout=min(30.0, self.timeout_seconds)) |
| 43 | + except (OSError, subprocess.SubprocessError) as error: |
| 44 | + raise failures.DiracUnsupported( |
| 45 | + "shared GPU handoff command failed", |
| 46 | + details={"command": list(command), |
| 47 | + "error": f"{type(error).__name__}: {error}"}) from error |
| 48 | + |
| 49 | + def _is_reasoner_active(self) -> bool: |
| 50 | + result = self._run( |
| 51 | + ["systemctl", "--user", "is-active", self.service], check=False) |
| 52 | + if result.returncode == 0 and result.stdout.strip() == "active": |
| 53 | + return True |
| 54 | + if result.stdout.strip() in {"inactive", "failed", "unknown"}: |
| 55 | + return False |
| 56 | + raise failures.DiracUnsupported( |
| 57 | + "cannot establish local reasoner service state before GPU handoff", |
| 58 | + details={"service": self.service, "returncode": result.returncode, |
| 59 | + "stdout": result.stdout.strip(), |
| 60 | + "stderr": result.stderr.strip()}) |
| 61 | + |
| 62 | + def _free_memory_bytes(self) -> int: |
| 63 | + result = self._run([ |
| 64 | + "nvidia-smi", "--query-gpu=memory.free", |
| 65 | + "--format=csv,noheader,nounits", |
| 66 | + ]) |
| 67 | + try: |
| 68 | + return int(result.stdout.splitlines()[0].strip()) * (1 << 20) |
| 69 | + except (ValueError, IndexError) as error: |
| 70 | + raise failures.DiracUnsupported( |
| 71 | + "cannot parse free GPU memory during shared GPU handoff", |
| 72 | + details={"stdout": result.stdout[:256]}) from error |
| 73 | + |
| 74 | + def _wait_for_memory(self, required_bytes: int) -> None: |
| 75 | + deadline = time.monotonic() + self.timeout_seconds |
| 76 | + observed = 0 |
| 77 | + while time.monotonic() <= deadline: |
| 78 | + observed = self._free_memory_bytes() |
| 79 | + if observed >= required_bytes: |
| 80 | + return |
| 81 | + self._sleep(self.poll_seconds) |
| 82 | + raise failures.DiracUnsupported( |
| 83 | + "local reasoner stopped but the scientific GPU memory requirement " |
| 84 | + "was not released before timeout", |
| 85 | + details={"required_bytes": required_bytes, |
| 86 | + "observed_free_bytes": observed, |
| 87 | + "timeout_seconds": self.timeout_seconds}) |
| 88 | + |
| 89 | + @contextmanager |
| 90 | + def exclusive(self, *, required_bytes: int) -> Iterator[None]: |
| 91 | + if type(required_bytes) is not int or required_bytes < 1: |
| 92 | + raise failures.DiracInternal( |
| 93 | + "shared GPU handoff requires a positive byte capacity") |
| 94 | + self.lock_path.parent.mkdir(parents=True, exist_ok=True) |
| 95 | + with self._thread_lock, self.lock_path.open("a+b") as lock_file: |
| 96 | + fcntl.flock(lock_file.fileno(), fcntl.LOCK_EX) |
| 97 | + reasoner_was_active = False |
| 98 | + body_error: BaseException | None = None |
| 99 | + try: |
| 100 | + reasoner_was_active = self._is_reasoner_active() |
| 101 | + if reasoner_was_active: |
| 102 | + self._run(["systemctl", "--user", "stop", self.service]) |
| 103 | + self._wait_for_memory(required_bytes) |
| 104 | + yield |
| 105 | + except BaseException as error: |
| 106 | + body_error = error |
| 107 | + raise |
| 108 | + finally: |
| 109 | + if reasoner_was_active: |
| 110 | + try: |
| 111 | + self._run(["systemctl", "--user", "start", self.service]) |
| 112 | + except BaseException: |
| 113 | + if body_error is None: |
| 114 | + raise |
| 115 | + # Preserve the scientific execution failure as the primary |
| 116 | + # exception; systemd records the independent restart failure. |
| 117 | + fcntl.flock(lock_file.fileno(), fcntl.LOCK_UN) |
| 118 | + |
| 119 | + |
| 120 | +def coordinator_from_environment(repository: Path) -> SharedGpuCoordinator | None: |
| 121 | + """Build only under the operator's explicit same-GPU deployment grant.""" |
| 122 | + if os.environ.get("DIRAC_ALLOW_SHARED_GPU_AI") != "1": |
| 123 | + return None |
| 124 | + return SharedGpuCoordinator( |
| 125 | + lock_path=Path(os.environ.get( |
| 126 | + "DIRAC_SHARED_GPU_LOCK", |
| 127 | + repository / ".runtime/shared-gpu/scientific-execution.lock")), |
| 128 | + service=os.environ.get("DIRAC_QWEN_SYSTEMD_SERVICE", "dirac-qwen.service"), |
| 129 | + timeout_seconds=float(os.environ.get( |
| 130 | + "DIRAC_SHARED_GPU_RELEASE_TIMEOUT_SECONDS", "120")), |
| 131 | + ) |
0 commit comments