Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion examples/qrmi-workload/sampler.py
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,7 @@ def print_environment():
def backend_from_fluence():
backend = os.environ.get("FLUXION_BACKEND")
if not backend:
sys.exit("no FLUXION_BACKEND, this pod was not placed by fluence")
sys.exit("no FLUXION_BACKEND in the environment")
rtype = os.environ.get("FLUXION_QRMI_TYPE", "qiskit-runtime-service")
if rtype not in RESOURCE_TYPES:
sys.exit("unsupported QRMI type {} for {}".format(rtype, backend))
Expand Down
13 changes: 7 additions & 6 deletions pkg/webhook/handlers/handoff_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -53,14 +53,15 @@ func opsMountHandoffWritable(ops []spec.Op) bool {
return false
}

// sidecarMountsHandoffReadOnly finds the sidecar container in the ops and checks its mount
func sidecarMountsHandoffReadOnly(ops []spec.Op) bool {
// sidecarMountsHandoffWritable finds the sidecar container in the ops and checks
// its mount is writable, since it writes the workers done marker
func sidecarMountsHandoffWritable(ops []spec.Op) bool {
check := func(c corev1.Container) bool {
if c.Name != SidecarContainerName {
return false
}
for _, m := range c.VolumeMounts {
if m.Name == HandoffVolumeName && m.ReadOnly {
if m.Name == HandoffVolumeName && !m.ReadOnly {
return true
}
}
Expand Down Expand Up @@ -133,7 +134,7 @@ func TestWorkerGetsNoHandoff(t *testing.T) {
}
}

func TestScoutSidecarMountsHandoffReadOnly(t *testing.T) {
func TestScoutSidecarMountsHandoffWritable(t *testing.T) {
ns, group, job := "default", "qgang2", "qgang2-job"
m := handoffJobMutator(ns, job, 2)
ops := m.Mutate(context.Background(), sharedQPUPod(ns, group, "qgang2-0", job, "0"))
Expand All @@ -143,8 +144,8 @@ func TestScoutSidecarMountsHandoffReadOnly(t *testing.T) {
if !opsMountHandoffWritable(ops) {
t.Error("the scout workload should mount the handoff volume writable")
}
if !sidecarMountsHandoffReadOnly(ops) {
t.Error("the scout sidecar should mount the handoff volume read only")
if !sidecarMountsHandoffWritable(ops) {
t.Error("the scout sidecar should mount the handoff volume writable, it writes the workers done marker")
}
if !contains(opEnvNames(ops), HandoffDirEnv) {
t.Errorf("the scout should get %s, got %v", HandoffDirEnv, opEnvNames(ops))
Expand Down
5 changes: 3 additions & 2 deletions pkg/webhook/handlers/quantum.go
Original file line number Diff line number Diff line change
Expand Up @@ -514,7 +514,8 @@ const (

// HandoffVolumeName / HandoffMountPath / HandoffDirEnv. The shared emptyDir
// a submitting pod's interceptor writes its task id and session to, and
// its sidecar reads (python/fluence/handoff.py). Workers never get it.
// its sidecar reads, then writes the workers done marker to
// (python/fluence/handoff.py). Workers never get it.
HandoffVolumeName = "fluence-handoff"
HandoffMountPath = "/opt/fluence-handoff"
HandoffDirEnv = "FLUENCE_HANDOFF_DIR"
Expand Down Expand Up @@ -702,7 +703,7 @@ func sidecarContainerOps(m webhook.MutatorAPI, pod *corev1.Pod, observe bool, ex
var mounts []corev1.VolumeMount
for _, v := range pod.Spec.Volumes {
if v.Name == HandoffVolumeName {
mounts = append(mounts, corev1.VolumeMount{Name: HandoffVolumeName, MountPath: HandoffMountPath, ReadOnly: true})
mounts = append(mounts, corev1.VolumeMount{Name: HandoffVolumeName, MountPath: HandoffMountPath})
}
}
sidecar := corev1.Container{
Expand Down
17 changes: 16 additions & 1 deletion python/fluence/sidecar.py
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@
from fluence.providers.base import log, ungate_position
from fluence import handoff
from fluence.ungate import (
placement_from_env,
ungate_pods,
gated_pods_from_env,
namespace_from_env,
Expand Down Expand Up @@ -94,6 +95,7 @@ def main():
f"gang_group={gang_group} backend={backend} observe={observe}"
)

placed_backend, placed_attrs = placement_from_env()
provider = resolve_from_env()
if provider is None:
log("ERROR: could not resolve a quantum provider from the backend")
Expand All @@ -113,6 +115,8 @@ def main():
),
"",
namespace,
backend=placed_backend,
attributes=placed_attrs,
)
_hold_for_workers(namespace, gang_group, pod_name)
sys.exit(0)
Expand All @@ -126,6 +130,8 @@ def main():
),
"",
namespace,
backend=placed_backend,
attributes=placed_attrs,
)
_hold_for_workers(namespace, gang_group, pod_name)
sys.exit(1)
Expand All @@ -152,6 +158,8 @@ def main():
),
job_id,
namespace,
backend=placed_backend,
attributes=placed_attrs,
)
_hold_for_workers(namespace, gang_group, pod_name)
sys.exit(1)
Expand All @@ -162,7 +170,14 @@ def main():
)
log(f"ungating {len(gated_pods)} gang pod(s): {gated_pods}")
session_id = provider.session_id(task)
n_ok = ungate_pods(gated_pods, job_id, namespace, session_id=session_id)
n_ok = ungate_pods(
gated_pods,
job_id,
namespace,
session_id=session_id,
backend=placed_backend,
attributes=placed_attrs,
)
if n_ok == len(gated_pods):
log(f"done, {n_ok} gang pod(s) ungated")
else:
Expand Down
33 changes: 28 additions & 5 deletions python/fluence/ungate.py
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,11 @@

JOB_ID_ANNOTATION = "fluence.flux-framework.org/quantum-job-id"
SESSION_ID_ANNOTATION = "fluence.flux-framework.org/quantum-session-id"
# the backend fluence placed the scout on, and its attributes. The worker's
# FLUXION_* variables read these through the downward API
BACKEND_ANNOTATION = "fluence.flux-framework.org/backend"
ATTR_ANNOTATION_PREFIX = "fluence.flux-framework.org/attr-"
ENV_PREFIX = "FLUXION_"
QUANTUM_GATE_NAME = "quantum.braket/ready"
PRIORITY_CLASS = "fluence-quantum-classical"

Expand All @@ -29,12 +34,26 @@ def kubectl(args):
return result.stdout.strip()


def ungate_pods(gated_pods, job_id, namespace, session_id=None):
def placement_from_env():
"""The backend and attributes the webhook gave this pod, from FLUXION_*."""
backend = os.environ.get(ENV_PREFIX + "BACKEND", "")
attrs = {
k[len(ENV_PREFIX) :].lower(): v
for k, v in os.environ.items()
if k.startswith(ENV_PREFIX) and k != ENV_PREFIX + "BACKEND" and v
}
return backend, attrs


def ungate_pods(
gated_pods, job_id, namespace, session_id=None, backend=None, attributes=None
):
"""
For each gated worker pod.
1. Stamp the job id annotation so the worker can locate the quantum
result, and the session id when the scout has one, so the worker
can submit into the same session.
1. Stamp the backend and its attributes, so the worker's FLUXION_*
variables read the same values the scout got. Then the job id, so
the worker can locate the quantum result, and the session id when
the scout has one, so the worker can submit into the same session.
2. Remove the scheduling gate so the pod can be scheduled.

Note that priorityClassName is NOT set here, it is immutable after pod creation
Expand All @@ -53,6 +72,10 @@ def ungate_pods(gated_pods, job_id, namespace, session_id=None):
log(f"ungating pod: {pod_name}")

notes = []
if backend:
notes.append(f"{BACKEND_ANNOTATION}={backend}")
for key, value in sorted((attributes or {}).items()):
notes.append(f"{ATTR_ANNOTATION_PREFIX}{key}={value}")
if job_id:
notes.append(f"{JOB_ID_ANNOTATION}={job_id}")
if session_id:
Expand All @@ -74,7 +97,7 @@ def ungate_pods(gated_pods, job_id, namespace, session_id=None):
except RuntimeError as e:
log(f" WARNING: could not annotate {pod_name}: {e}")
else:
log(f" WARNING: no job id to patch onto {pod_name}")
log(f" nothing to stamp onto {pod_name}")

patch = json.dumps(
[
Expand Down
44 changes: 44 additions & 0 deletions python/tests/test_hold.py
Original file line number Diff line number Diff line change
Expand Up @@ -89,6 +89,50 @@ def test_ungate_stamps_job_id_and_session():
assert any(c[0] == "patch" for c in calls)


def test_ungate_stamps_backend_and_attributes():
from fluence import ungate

calls = []
orig = ungate.kubectl
ungate.kubectl = lambda args: calls.append(args) or ""
try:
ungate.ungate_pods(
["w-1"],
"",
"ns",
backend="ibm_fez",
attributes={"vendor": "ibm", "qrmi_type": "qiskit-runtime-service"},
)
finally:
ungate.kubectl = orig
annotate = [c for c in calls if c[0] == "annotate"][0]
assert f"{ungate.BACKEND_ANNOTATION}=ibm_fez" in annotate
assert f"{ungate.ATTR_ANNOTATION_PREFIX}vendor=ibm" in annotate
assert (
f"{ungate.ATTR_ANNOTATION_PREFIX}qrmi_type=qiskit-runtime-service" in annotate
)
assert not any(ungate.JOB_ID_ANNOTATION in a for a in annotate)


def test_placement_from_env_reads_fluxion_variables():
from fluence import ungate

os.environ.update(
{
"FLUXION_BACKEND": "ibm_fez",
"FLUXION_VENDOR": "ibm",
"FLUXION_QRMI_TYPE": "qiskit-runtime-service",
}
)
try:
backend, attrs = ungate.placement_from_env()
finally:
for k in ("FLUXION_BACKEND", "FLUXION_VENDOR", "FLUXION_QRMI_TYPE"):
os.environ.pop(k, None)
assert backend == "ibm_fez"
assert attrs == {"vendor": "ibm", "qrmi_type": "qiskit-runtime-service"}


def test_ungate_without_session_stamps_only_job_id():
from fluence import ungate

Expand Down
Loading