From ae03dd1883fc1c188bdfe9c97649f9d130e938ee Mon Sep 17 00:00:00 2001 From: vsoch Date: Sat, 17 Oct 2026 12:12:34 -0700 Subject: [PATCH 1/2] qrmi: add sidecar and tests Signed-off-by: vsoch --- .devcontainer/Dockerfile | 6 +- .devcontainer/Dockerfile.old | 4 +- .devcontainer/devcontainer.json | 2 +- .github/workflows/e2e-suite.yaml | 2 +- .github/workflows/e2e-tests.yaml | 2 +- .../workflows/qrmi-workload-build-deploy.yaml | 74 +++ .github/workflows/sidecar-build-deploy.yaml | 5 +- .gitignore | 2 + .pre-commit-config.yaml | 19 + Dockerfile | 2 +- README.md | 77 ++- cmd/webhook/main.go | 15 +- deploy/fluence-pull-test.yaml | 2 +- deploy/fluence-resources-test.yaml | 2 +- deploy/kind-config.yaml | 2 +- docker/Dockerfile | 2 +- docs/coordination-handler-design.md | 194 +++--- docs/handlers.md | 4 +- docs/qrmi-coordination.md | 157 +++++ docs/quantum-scheduling.md | 608 +++++------------- examples/qrmi-pod.yaml | 35 + examples/qrmi-workload/Dockerfile | 13 + examples/qrmi-workload/sampler.py | 287 +++++++++ examples/quantum-pod.yaml | 4 +- .../test/e2e/gang/multi-gang-requeue.yaml | 2 +- examples/test/e2e/quantum/qrmi-gang-pods.yaml | 48 ++ examples/test/e2e/quantum/qrmi-pod-mock.yaml | 33 + .../test/e2e/quantum/quantum-gang-pods.yaml | 14 +- examples/test/jobspec-cpu.yaml | 2 +- examples/test/jobspec-qpu.yaml | 2 +- pkg/fluence/fluence.go | 26 +- pkg/placement/placement.go | 4 + pkg/webhook/admission.go | 205 ++++++ pkg/webhook/handler.go | 15 + pkg/webhook/handlers/admission_test.go | 209 ++++++ pkg/webhook/handlers/dependency.go | 28 +- pkg/webhook/handlers/gang.go | 4 +- pkg/webhook/handlers/gang_test.go | 6 - pkg/webhook/handlers/handlers_test.go | 2 +- pkg/webhook/handlers/handoff_test.go | 180 ++++++ pkg/webhook/handlers/quantum.go | 322 ++++++---- pkg/webhook/handlers/quantum_test.go | 137 ++-- pkg/webhook/handlers/registry_test.go | 6 - pkg/webhook/webhook.go | 47 +- python/fluence/handoff.py | 127 ++++ python/fluence/interceptor.py | 12 +- python/fluence/providers/__init__.py | 2 +- python/fluence/providers/base.py | 12 +- python/fluence/providers/braket.py | 98 ++- python/fluence/providers/qrmi.py | 274 ++++++++ python/fluence/sidecar.py | 128 ++-- python/fluence/sitecustomize.py | 1 + python/fluence/stage.py | 7 +- python/fluence/ungate.py | 107 ++- python/pyproject.toml | 4 +- python/tests/test_fluence.py | 61 +- python/tests/test_hold.py | 215 +++++++ python/tests/test_qrmi.py | 397 ++++++++++++ test/e2e/quantum/setup.sh | 2 +- 59 files changed, 3310 insertions(+), 949 deletions(-) create mode 100644 .github/workflows/qrmi-workload-build-deploy.yaml create mode 100644 .pre-commit-config.yaml create mode 100644 docs/qrmi-coordination.md create mode 100644 examples/qrmi-pod.yaml create mode 100644 examples/qrmi-workload/Dockerfile create mode 100644 examples/qrmi-workload/sampler.py create mode 100644 examples/test/e2e/quantum/qrmi-gang-pods.yaml create mode 100644 examples/test/e2e/quantum/qrmi-pod-mock.yaml create mode 100644 pkg/webhook/admission.go create mode 100644 pkg/webhook/handlers/admission_test.go create mode 100644 pkg/webhook/handlers/handoff_test.go create mode 100644 python/fluence/handoff.py create mode 100644 python/fluence/providers/qrmi.py create mode 100644 python/tests/test_hold.py create mode 100644 python/tests/test_qrmi.py diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile index 0f730db..e0cea68 100644 --- a/.devcontainer/Dockerfile +++ b/.devcontainer/Dockerfile @@ -76,16 +76,16 @@ RUN groupadd -g ${USER_GID} ${USERNAME} && \ # This branch of flux-sched has the bindings we need! RUN ln -s /usr/local/go/bin/go /usr/bin/go && \ export FLUX_SCHED_VERSION=0.53.0 && \ - git clone -b implement-reapi-cli-update-allocate https://github.com/vsoch/flux-sched /opt/flux-sched && \ + git clone -b implement-reapi-cli-update-allocate https://github.com/vsoch/flux-sched /opt/flux-sched && \ # git clone -b https://github.com/flux-framework/flux-sched /opt/flux-sched && \ cd /opt/flux-sched && \ export WITH_GO=yes && \ ./configure --prefix=/usr && \ mkdir build && \ cd build && \ - cmake ../ && \ + cmake ../ && \ cd ../ && \ make && make install ENV LD_LIBRARY_PATH=/usr/lib:/opt/flux-sched/resource:/opt/flux-sched/resource/reapi/bindings:/opt/flux-sched/resource/libjobspec -USER $USERNAME \ No newline at end of file +USER $USERNAME diff --git a/.devcontainer/Dockerfile.old b/.devcontainer/Dockerfile.old index c976e45..e5c5701 100644 --- a/.devcontainer/Dockerfile.old +++ b/.devcontainer/Dockerfile.old @@ -56,8 +56,8 @@ RUN git clone -b lib-updates https://github.com/milroy/flux-sched /opt/flux-sche ./configure --prefix=/usr && \ mkdir build && \ cd build && \ - cmake ../ && \ + cmake ../ && \ cd ../ && \ make && make install -USER $USERNAME \ No newline at end of file +USER $USERNAME diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index c3f3f03..87d7d44 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -15,4 +15,4 @@ } }, "postStartCommand": "git config --global --add safe.directory /workspaces/fluence" -} \ No newline at end of file +} diff --git a/.github/workflows/e2e-suite.yaml b/.github/workflows/e2e-suite.yaml index dce5b8a..deeb828 100644 --- a/.github/workflows/e2e-suite.yaml +++ b/.github/workflows/e2e-suite.yaml @@ -106,4 +106,4 @@ jobs: run: | kubectl get pods -A -o wide kubectl logs -n kube-system deployment/fluence || true - kubectl logs -n kube-system deployment/fluence-webhook || true \ No newline at end of file + kubectl logs -n kube-system deployment/fluence-webhook || true diff --git a/.github/workflows/e2e-tests.yaml b/.github/workflows/e2e-tests.yaml index 4b405f6..021e611 100644 --- a/.github/workflows/e2e-tests.yaml +++ b/.github/workflows/e2e-tests.yaml @@ -19,4 +19,4 @@ jobs: suite: [gang, quantum] uses: ./.github/workflows/e2e-suite.yaml with: - suite: ${{ matrix.suite }} \ No newline at end of file + suite: ${{ matrix.suite }} diff --git a/.github/workflows/qrmi-workload-build-deploy.yaml b/.github/workflows/qrmi-workload-build-deploy.yaml new file mode 100644 index 0000000..690e7b7 --- /dev/null +++ b/.github/workflows/qrmi-workload-build-deploy.yaml @@ -0,0 +1,74 @@ +name: qrmi-workload-build-deploy + +# Builds the example QRMI workload image for amd64 and arm64. It is python +# and wheels, so QEMU is fine here. + +on: + push: + branches: [main] + tags: ["v*"] + paths: + - "examples/qrmi-workload/**" + - ".github/workflows/qrmi-workload-build-deploy.yaml" + pull_request: + branches: [main] + paths: + - "examples/qrmi-workload/**" + - ".github/workflows/qrmi-workload-build-deploy.yaml" + workflow_dispatch: + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +env: + REGISTRY: ghcr.io + IMAGE_NAME: ${{ github.repository }}-qrmi-workload + +jobs: + build-deploy: + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Log in to GHCR + if: github.event_name != 'pull_request' + uses: docker/login-action@v3 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Image metadata (tags, labels) + id: meta + uses: docker/metadata-action@v5 + with: + images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + tags: | + type=ref,event=branch + type=ref,event=pr + type=semver,pattern={{version}} + type=sha + type=raw,value=latest,enable=${{ github.ref == 'refs/heads/main' }} + + - name: Build and push + uses: docker/build-push-action@v6 + with: + context: ./examples/qrmi-workload + file: ./examples/qrmi-workload/Dockerfile + platforms: linux/amd64,linux/arm64 + push: ${{ github.event_name != 'pull_request' }} + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + cache-from: type=gha,scope=qrmi-workload + cache-to: type=gha,mode=max,scope=qrmi-workload diff --git a/.github/workflows/sidecar-build-deploy.yaml b/.github/workflows/sidecar-build-deploy.yaml index 90a418f..16b55a4 100644 --- a/.github/workflows/sidecar-build-deploy.yaml +++ b/.github/workflows/sidecar-build-deploy.yaml @@ -47,7 +47,10 @@ jobs: - name: Run tests working-directory: python - run: python tests/test_fluence.py + run: | + python tests/test_fluence.py + python tests/test_qrmi.py + python tests/test_hold.py build-deploy: runs-on: ubuntu-latest diff --git a/.gitignore b/.gitignore index b873546..2eb5785 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,4 @@ vendor bin +__pycache__/ +*.pyc diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..e364049 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,19 @@ +exclude: "python/dist|python/fluence_hpc.egg-info" +repos: + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v4.3.0 + hooks: + - id: check-added-large-files + - id: check-case-conflict + - id: end-of-file-fixer + - id: trailing-whitespace + - id: mixed-line-ending + + - repo: local + hooks: + - id: black + name: black + language: python + types: [python] + entry: black + args: [--target-version, py311] diff --git a/Dockerfile b/Dockerfile index 1e2f153..1c56f67 100644 --- a/Dockerfile +++ b/Dockerfile @@ -7,7 +7,7 @@ COPY . . RUN CGO_ENABLED=1 \ CGO_CFLAGS="-I/opt/flux-sched" \ CGO_LDFLAGS="-L/opt/flux-sched/resource -L/opt/flux-sched/resource/libjobspec -L/opt/flux-sched/resource/reapi/bindings -lresource -ljobspec_conv -lreapi_cli -lflux-idset -lstdc++ -lczmq -ljansson -lhwloc -lboost_system -lflux-hostlist -lboost_graph -lyaml-cpp" \ - go build -ldflags '-w' -o /bin/fluence ./cmd/fluence + go build -ldflags '-w' -o /bin/fluence ./cmd/fluence RUN CGO_ENABLED=0 go build -ldflags '-w' -o /bin/fluence-deviceplugin ./cmd/deviceplugin RUN CGO_ENABLED=0 go build -ldflags '-w' -o /bin/fluence-webhook ./cmd/webhook diff --git a/README.md b/README.md index d299757..aeee0ae 100644 --- a/README.md +++ b/README.md @@ -22,7 +22,7 @@ proven out first in ## How does it work? ```console - resources.yaml --+ + resources.yaml --+ +--------------+-------------------------------+ v v v scheduler device plugin webhook @@ -194,9 +194,9 @@ ceiling. Types come from the same config as the graph, so they can't drift. ### `sidecars/` — quantum coordination sidecars -Vendor-specific sidecar containers injected by the webhook into the producer pod +Vendor-specific sidecar containers injected by the webhook into the scout pod of a shared quantum workflow group. Each sidecar discovers the QPU task submitted -by the producer, polls the vendor queue, and ungates the consumer pods when the +by the scout, polls the vendor queue, and ungates the worker pods when the task reaches position==1. ```console @@ -221,7 +221,7 @@ spec: ``` Fluence creates the PodGroup, injects the sidecar, creates per-namespace -RBAC, and gates the consumer pods. See `sidecars/braket/design.md` for +RBAC, and gates the worker pods. See `sidecars/braket/design.md` for the full design including the SDK interceptor, queue position polling, and the two-queue problem motivation. @@ -242,8 +242,8 @@ workload reads these normalized names regardless of which backend it matched. - `cmd/deviceplugin` — the extended-resource DaemonSet. - `cmd/webhook` — the env-injection webhook. - `cmd/recovery-probe` — verifies allocation replay survives a graph rebuild - (what a restart does); see `make test-restore`. - + (what a restart does); see `make test-restore`. + Note that the recovery probe (and graph restore) was implemented but removed because the code in fluxion is only part of a PR branch, and I feel nervous about depending on it. ## Configuration @@ -255,6 +255,29 @@ These are environment variables for fluence. | `FLUENCE_RESOURCES` | scheduler, device plugin, webhook | path to `resources.yaml`; absent = classical-only | | `FLUENCE_MATCH_POLICY` | scheduler | Fluxion match policy (default `first`) | | `FLUENCE_RESOURCE_CAPACITY` | device plugin | per-node ceiling per type (default 1000) | +| `FLUENCE_ADMISSION` | webhook | `on` turns admission control on (default off) | +| `FLUENCE_RESERVE_CORES` | webhook | cores held back from the admission budget (default 0) | +| `FLUENCE_WORKER_TIMEOUT` | sidecar | seconds to hold for the worker group after ungating, 0 waits without limit (default 0) | +| `FLUENCE_SESSION_HOLD_TIMEOUT` | scout container | seconds a scout's release waits for the worker group, 0 waits without limit (default 0) | + +## Admission control + +A quantum group costs money once it holds a backend, so with FLUENCE_ADMISSION=on +the webhook only admits one that fits. It is off by default. This is the rule the flux-quantum jobtap plugin applies at submit. +The cores the whole group will need, plus what is already promised to admitted +groups that have not started, must fit in what is reachable. Reachable is the +free cores plus the cores held by running pods that could be preempted, meaning +pods that are unprotected and below the priority workers are ungated with. + +A group that does not fit is refused when its first pod is created, so kubectl +apply fails or the owning Job records the reason. It is not left pending with a +backend it cannot use. A standalone or independent quantum pod is a group of +one. A shared gang is one group, scout and workers together, admitted once +by whichever member arrives first and recorded on the gang PodGroup as +fluence.flux-framework.org/admitted-cores. Every quantum pod is marked +fluence.flux-framework.org/protected so it is never counted as preemptible, and +a pod that arrives already marked is refused. The code is in +pkg/webhook/admission.go. ## Observability @@ -346,7 +369,13 @@ kubectl create secret generic ibm-quantum -n default \ ``` Run a single quantum pod — it just requests `fluxion.flux-framework.org/qpu`, with -no hard-coded backend (the webhook + PreBind supply `FLUXION_BACKEND`): +no hard-coded backend (the webhook + PreBind supply `FLUXION_BACKEND`). The +example workload in examples/qrmi-workload does the QRMI side. It acquires a +session on the backend fluence chose, runs a Bell circuit through the QRMI +sampler, and releases the session. Without a Secret it prints what it would +need and exits. The image is ghcr.io/converged-computing/fluence-qrmi-workload, +built for amd64 and arm64, and examples/qrmi-pod.yaml runs it. +examples/quantum-pod.yaml runs the older qrmi-sampler: ```bash kubectl apply -f examples/quantum-pod.yaml @@ -369,13 +398,13 @@ Submission is **not** done by the scheduler — the workload container holds the user's credentials and submits via qrmi-go. Fluence only schedules and hands off the backend. (When we control local quantum devices this will change.) -### 3. Quantum workflow groups (producer + consumers) +### 3. Quantum workflow groups (scout + workers) A quantum workflow group is a gang whose members share **one** quantum task: -one pod **submits** the work (the producer) and N−1 pods **wait** for the result -(the consumers). All pods share a group label and run the *same* image; Fluence -co-schedules them, gives the producer a sidecar that watches the vendor queue, and -gates the consumers so they consume no node resources during the (long, variable) +one pod **submits** the work (the scout) and N−1 pods **wait** for the result +(the workers). All pods share a group label and run the *same* image; Fluence +co-schedules them, gives the scout a sidecar that watches the vendor queue, and +gates the workers so they consume no node resources during the (long, variable) QPU queue wait — releasing them only when the task reaches `queue_position == 1`. ```yaml @@ -396,7 +425,7 @@ spec: defaults to `independent`. - **`shared`** — the gang shares ONE quantum task. Fluence promotes one member to - producer and gates the rest as consumers (see below). Use this for a coordinated + scout and gates the rest as workers (see below). Use this for a coordinated workflow where the classical post-processing should start together as the single result lands. - **`independent`** (default) — every member does its own quantum work: its own @@ -404,9 +433,9 @@ defaults to `independent`. honest default; Fluence never invents coordination you did not ask for, and never dedups tasks meant to be distinct. -#### How the producer is chosen +#### How the scout is chosen -In `shared` mode the producer is the member the Job controller stamps with +In `shared` mode the scout is the member the Job controller stamps with `batch.kubernetes.io/job-completion-index: "0"` — so an **indexed Job** gives deterministic, race-free election from a single identical template (every pod has the same image and group label; only the index differs). This serves two contracts @@ -414,24 +443,24 @@ with no extra configuration: - an **explicit-role script** that branches on the completion index (index 0 submits; others wait and consume the result), and -- an **identical script** where every pod calls submit — the producer's submit is - real, and each consumer's submit is transparently returned the producer's task +- an **identical script** where every pod calls submit — the scout's submit is + real, and each worker's submit is transparently returned the scout's task (the shared-result dedup), so the code need not branch at all. -For loose pods with no completion index, the first pod admitted claims the producer +For loose pods with no completion index, the first pod admitted claims the scout slot; an indexed Job is recommended when you need determinism. #### What Fluence does -In `shared` mode the producer gets the interceptor (real mode) + sidecar and its -own group-of-one PodGroup `-producer` (`minCount: 1`), so it schedules -alone and runs the single real submit; it is never gated. The consumers join the +In `shared` mode the scout gets the interceptor (real mode) + sidecar and its +own group-of-one PodGroup `-scout` (`minCount: 1`), so it schedules +alone and runs the single real submit; it is never gated. The workers join the `` gang (`minCount: N−1`), get a `quantum.braket/ready` scheduling gate, and consume no node resources during the QPU queue wait. When the sidecar observes -`queue_position == 1`, it stamps the producer's task id onto each consumer +`queue_position == 1`, it stamps the scout's task id onto each worker (surfaced as `FLUENCE_QUANTUM_JOB_ID`) and removes their gates atomically with setting the `fluence-quantum-classical` priority class so they reschedule promptly. -The producer is one of the N members, so the application runs exactly N times — +The scout is one of the N members, so the application runs exactly N times — never N+1, and there is no separate submitter pod. Per-namespace RBAC (`fluence-sidecar` ServiceAccount/Role/RoleBinding) and the @@ -464,7 +493,7 @@ one result, use `coordination: shared` above. ## License -Distributed under the MIT license; all contributions must be made under it. +Distributed under the MIT license; all contributions must be made under it. See [LICENSE](LICENSE), [COPYRIGHT](COPYRIGHT), and [NOTICE](NOTICE). diff --git a/cmd/webhook/main.go b/cmd/webhook/main.go index 1a6709d..8510aa1 100644 --- a/cmd/webhook/main.go +++ b/cmd/webhook/main.go @@ -16,6 +16,7 @@ import ( "log" "net/http" "os" + "strconv" "strings" "time" @@ -109,10 +110,20 @@ func main() { log.Printf("no resources config at %s (%v); injecting FLUXION_BACKEND only", path, rerr) } } + // admission control is off unless FLUENCE_ADMISSION=on, and + // FLUENCE_RESERVE_CORES is held back from its budget + admission := env("FLUENCE_ADMISSION", "off") == "on" + reserve, err := strconv.Atoi(env("FLUENCE_RESERVE_CORES", "0")) + if err != nil || reserve < 0 { + log.Fatalf("FLUENCE_RESERVE_CORES must be a non-negative integer, got %q", os.Getenv("FLUENCE_RESERVE_CORES")) + } mutator := &webhook.Mutator{ - AttributeKeys: attrKeys, - Clientset: client, + AttributeKeys: attrKeys, + Clientset: client, + AdmissionControl: admission, + ReserveCores: reserve, } + log.Printf("[fluence-webhook] admission control on=%v reserve_cores=%d", admission, reserve) log.Printf("[fluence-webhook] env contract injected into fluxion pods: %v", mutator.EnvVarNames()) mux := http.NewServeMux() diff --git a/deploy/fluence-pull-test.yaml b/deploy/fluence-pull-test.yaml index ec819ae..aadef20 100644 --- a/deploy/fluence-pull-test.yaml +++ b/deploy/fluence-pull-test.yaml @@ -141,7 +141,7 @@ spec: containers: - name: fluence image: vanessa/fluence:test - imagePullPolicy: Always + imagePullPolicy: Always command: - /bin/fluence - --config=/etc/fluence/scheduler-config.yaml diff --git a/deploy/fluence-resources-test.yaml b/deploy/fluence-resources-test.yaml index b584bfb..8c1f6a8 100644 --- a/deploy/fluence-resources-test.yaml +++ b/deploy/fluence-resources-test.yaml @@ -90,4 +90,4 @@ spec: path: /var/lib/kubelet/device-plugins - name: resources configMap: - name: fluence-resources \ No newline at end of file + name: fluence-resources diff --git a/deploy/kind-config.yaml b/deploy/kind-config.yaml index ec310bc..c94e070 100644 --- a/deploy/kind-config.yaml +++ b/deploy/kind-config.yaml @@ -32,4 +32,4 @@ nodes: - name: feature-gates value: "GenericWorkload=true" - role: worker - - role: worker \ No newline at end of file + - role: worker diff --git a/docker/Dockerfile b/docker/Dockerfile index 4ec6a49..57b53c1 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -18,7 +18,7 @@ RUN wget -q https://go.dev/dl/go1.26.0.linux-amd64.tar.gz \ && tar -C /usr/local -xzf go1.26.0.linux-amd64.tar.gz && rm go1.26.0.linux-amd64.tar.gz ENV PATH=$PATH:/usr/local/go/bin -RUN wget https://github.com/flux-framework/flux-sched/releases/download/v0.52.0/flux-sched-0.52.0.tar.gz && \ +RUN wget https://github.com/flux-framework/flux-sched/releases/download/v0.52.0/flux-sched-0.52.0.tar.gz && \ tar -xzvf flux-sched-0.52.0.tar.gz && mv flux-sched-0.52.0 /opt/flux-sched \ && cd /opt/flux-sched && export WITH_GO=yes && ./configure --prefix=/usr \ && mkdir build && cd build && cmake ../ && cd ../ && make -j"$(nproc)" && make install diff --git a/docs/coordination-handler-design.md b/docs/coordination-handler-design.md index cdcfd38..0f6631b 100644 --- a/docs/coordination-handler-design.md +++ b/docs/coordination-handler-design.md @@ -1,15 +1,15 @@ -# Coordination handlers: producer/consumer gang split (no separate submitter) +# Coordination handlers: scout/worker gang split (no separate submitter) > **Status: implemented.** This design is live in `pkg/webhook/handlers/quantum.go` -> (the coordination router + `mutateProducer`/`mutateConsumer`/`coordinationMode`/ -> `isProducer`), `pkg/webhook/handlers/gang.go` (classical gangs defer quantum +> (the coordination router + `mutateScout`/`mutateWorker`/`coordinationMode`/ +> `isScout`), `pkg/webhook/handlers/gang.go` (classical gangs defer quantum > pods to the quantum handler), and `pkg/fluence/fluence.go` (reconcile reaps the -> `-producer` PodGroup, never the producer pod — it is a real member). +> `-scout` PodGroup, never the scout pod — it is a real member). > Unit tests are in `pkg/webhook/handlers/quantum_test.go`; structural e2e in > `test/e2e/quantum/02–04`. Coordination is **role-aware**: the webhook stamps -> `FLUENCE_COORDINATION_ROLE` (producer/consumer) and hands consumers the -> producer's task id (`FLUENCE_QUANTUM_JOB_ID`); the workload branches on the role -> (producer submits, consumer fetches the shared result by id). No submit +> `FLUENCE_COORDINATION_ROLE` (scout/worker) and hands workers the +> scout's task id (`FLUENCE_QUANTUM_JOB_ID`); the workload branches on the role +> (scout submits, worker fetches the shared result by id). No submit > interception, no faux flag — that earlier mechanism has been removed. ## Why this replaces the submitter-pod model @@ -20,11 +20,11 @@ application image to do the real submit, then ungates a gang of N faux-submittin members. That works, but it runs the user's application **N+1 times** for an N-gang: once in the submitter (a full run whose post-processing nobody consumes) plus once in each of the N members. The redundant run is not an implementation -wart — it is a symptom of modeling quantum work as a producer/consumer split +wart — it is a symptom of modeling quantum work as a scout/worker split while pretending one image plays both roles, selected at runtime by a faux flag. This design keeps the split (it is correct) but removes the separate pod: the -**producer is one of the N members**, promoted at admission, so the application +**scout is one of the N members**, promoted at admission, so the application runs exactly **N times** — the needed number — with exactly **one real submit**. The core thesis is unchanged: Fluence is a generic gang scheduler (native gangs @@ -41,10 +41,10 @@ pods that consume a result someone else produced. That partitions a quantum gang into two kinds, decided per pod: -- **producer** — runs its code, submits its own task, holds a node through the +- **scout** — runs its code, submits its own task, holds a node through the queue wait. Not gateable, ever. -- **consumer** — never submits; reads the producer's result. Fully gateable until - that result is ready. +- **worker** — does not submit its own task. It reads the scout's result, or + submits into the scout's session. Fully gateable until that result is ready. ## Coordination modes (user-facing contract) @@ -60,8 +60,8 @@ metadata: | mode | meaning | who submits | gating | app runs | real submits | |------|---------|-------------|--------|----------|--------------| -| `independent` (default) | N pods each do their own quantum work | every pod | none possible (all are producers) | N | N | -| `shared` | one task; N pods consume the result | producer only | consumers gated until task ready | N | 1 | +| `independent` (default) | N pods each do their own quantum work | every pod | none possible (all are scouts) | N | N | +| `shared` | one task; N pods consume the result | scout only | workers gated until task ready | N | 1 | `coordination` is an open enum so future designs (e.g. `scatter` — index-paired task↔pod, §6.2 of the quantum doc) slot in as new modes without changing the @@ -70,78 +70,78 @@ ask for, and never dedup tasks that were meant to be distinct. ### What each mode does to resources, honestly -- **shared**: the producer (1 node) holds its node through the queue wait; - consumers (N−1) consume **zero** node resources while gated, then start at +- **shared**: the scout (1 node) holds its node through the queue wait; + workers (N−1) consume **zero** node resources while gated, then start at position==1. Idle cost during the wait ≈ 1 node, vs N for a traditional gang. -- **independent**: every pod is a producer, so every pod holds its node through +- **independent**: every pod is a scout, so every pod holds its node through its own queue wait — N nodes idle. There is nothing to coordinate (no shared result), so this is not a Fluence deficiency; it is the physics of "N independent tasks," and it is the user's explicit choice. The only way to - reclaim even the producer's node in either mode is a resumable `.result()` + reclaim even the scout's node in either mode is a resumable `.result()` (replay), and is deliberately **out of scope for v1** (one idle node is cheap; replay imposes a replay-safe-code contract). -## Producer election +## Scout election -Exactly one member must be the producer. Election is deterministic for the +Exactly one member must be the scout. Election is deterministic for the recommended workload and best-effort otherwise: - **Indexed Job (recommended):** the pod carries - `batch.kubernetes.io/job-completion-index`. **Index `0` is the producer**; - every other index is a consumer. Deterministic, race-free, no recorded state — + `batch.kubernetes.io/job-completion-index`. **Index `0` is the scout**; + every other index is a worker. Deterministic, race-free, no recorded state — the controller already stamped the index, and identical templates yield differentiated behavior purely from it. This is why an indexed Job is the right shape and is what the experiments use. - **Non-indexed gang (Deployment / raw grouped pods):** first arrival claims the - producer slot by creating the producer PodGroup (create-if-absent); later pods - find it present and become consumers. Best-effort (racy under simultaneous + scout slot by creating the scout PodGroup (create-if-absent); later pods + find it present and become workers. Best-effort (racy under simultaneous admission); documented, with indexed Job recommended for determinism. ## The two-group split -| | producer (index 0) | consumers (indices 1..N−1) | +| | scout (index 0) | workers (indices 1..N−1) | |---|---|---| -| PodGroup | `-producer`, `minCount=1` | ``, `minCount=N−1` | +| PodGroup | `-scout`, `minCount=1` | ``, `minCount=N−1` | | schedules | immediately, alone | atomically as a gang, **after ungate** | | gate | none | `quantum.braket/ready` + preempting priority | -| interceptor | staged (tags the real submit) | **not staged** (a consumer never submits) | +| interceptor | staged (tags the real submit) | **not staged** (a worker's submits go into the scout session) | | sidecar | yes — polls the task, ungates `` at position==1 | no | -| app run | full; submits the one real task | full; reads role=consumer and fetches the shared result by id (no submit) | +| app run | full; submits the one real task | full; reads role=worker and fetches the shared result by id, or submits into the scout session | -`minCount=1` on the producer group is what removes the deadlock that forced a -separate submitter: a single-member group schedules alone, so the producer runs -during the wait while the `minCount=N−1` consumer group sits gated. The two -groups have independent minCounts; neither blocks the other. The consumer group +`minCount=1` on the scout group is what removes the deadlock that forced a +separate submitter: a single-member group schedules alone, so the scout runs +during the wait while the `minCount=N−1` worker group sits gated. The two +groups have independent minCounts; neither blocks the other. The worker group keeps a real gang `minCount` (N−1), so **gang scheduling is preserved and demonstrable** (experiment requirement 1). -Coordination is role-aware rather than interception-based: the consumer is told -`FLUENCE_COORDINATION_ROLE=consumer` and handed the producer's task id +Coordination is role-aware rather than interception-based: the worker is told +`FLUENCE_COORDINATION_ROLE=worker` and handed the scout's task id (`FLUENCE_QUANTUM_JOB_ID`, stamped by the sidecar at ungate), and the workload fetches the shared result by that id instead of submitting. One real task, N -consumers, each app run once, in full — and no SDK submit-interception. +workers, each app run once, in full — and no SDK submit-interception. ## Gate / ungate flow (shared mode) ``` -1. Producer (index 0) admitted -> own group-of-one, ungated, sidecar attached +1. Scout (index 0) admitted -> own group-of-one, ungated, sidecar attached (FLUENCE_GANG_GROUP=), interceptor in REAL mode. - Consumers (1..N-1) admitted -> group (minCount N-1), GATED, - role=consumer, depends-on producer=-producer. + Workers (1..N-1) admitted -> group (minCount N-1), GATED, + role=worker, depends-on scout=-scout. -2. Scheduler places the producer immediately (minCount=1). It runs the user app, +2. Scheduler places the scout immediately (minCount=1). It runs the user app, .run() submits the ONE real task (tagged fluence-pod-uid). -3. Producer sidecar discovers the task by tag, polls queue position. +3. Scout sidecar discovers the task by tag, polls queue position. 4. At position==1 (or RUNNING): for each gated pod in : annotate fluence.flux-framework.org/quantum-job-id= remove the quantum.braket/ready gate (priority already set at admission) -5. Consumer group (now ungated, minCount N-1) gang-schedules atomically and - starts as the quantum result arrives. Each consumer reads role=consumer and - fetches the producer's task by FLUENCE_QUANTUM_JOB_ID (.result() returns the - shared result); app post-processes. No consumer submits. +5. Worker group (now ungated, minCount N-1) gang-schedules atomically and + starts as the quantum result arrives. Each worker reads role=worker and + fetches the scout's task by FLUENCE_QUANTUM_JOB_ID (.result() returns the + shared result); app post-processes. No worker submits. ``` `independent` mode skips all of this: each pod is its own group-of-one, ungated, @@ -160,14 +160,14 @@ handler, `dependency.go`, `sidecar.go`, and the Python interceptor/sidecar are The gang handler currently calls `EnsurePodGroup` unconditionally and relies on idempotency to coexist with the quantum handler. With the two-group split the -quantum handler owns *both* quantum PodGroups (and the producer's group differs +quantum handler owns *both* quantum PodGroups (and the scout's group differs from its admission-time label), so the gang handler must not also gang quantum pods. Make it skip them: ```go func (h *gangHandler) Applies(ctx context.Context, m webhook.MutatorAPI, pod *corev1.Pod) bool { // Classical gangs only. A pod that requests the quantum resource is gang- - // scheduled by the quantum handler (which owns the producer/consumer split); + // scheduled by the quantum handler (which owns the scout/worker split); // handling it here too would create a second, conflicting PodGroup. if spec.PodRequestsResource(pod, QuantumResource) { return false @@ -185,29 +185,29 @@ const ( // CoordinationAnnotation selects how a quantum gang is coordinated. Open enum // so new designs (e.g. "scatter") add a mode without changing the mechanism. CoordinationAnnotation = "fluence.flux-framework.org/coordination" - // CoordinationShared: one real task; the producer (index 0) submits, the - // other members are gated consumers that dedup to the producer's task. + // CoordinationShared: one real task; the scout (index 0) submits, the + // other members are gated workers that dedup to the scout's task. CoordinationShared = "shared" // CoordinationIndependent (default): every member does its own quantum work; // no coordination, no gating, each holds its node through its own queue wait. CoordinationIndependent = "independent" - // ProducerGroupSuffix names the producer's own group-of-one: -producer + // ScoutGroupSuffix names the scout's own group-of-one: -scout // (minCount 1) so it schedules alone and never deadlocks against the gated - // consumer gang. - ProducerGroupSuffix = "-producer" + // worker gang. + ScoutGroupSuffix = "-scout" // CompletionIndexAnnotation is the indexed-Job completion index the Job - // controller stamps on each pod; index "0" is the producer (deterministic + // controller stamps on each pod; index "0" is the scout (deterministic // election, no recorded state). CompletionIndexAnnotation = "batch.kubernetes.io/job-completion-index" - // ProducerIndex is the completion index promoted to producer. - ProducerIndex = "0" + // ScoutIndex is the completion index promoted to scout. + ScoutIndex = "0" ) ``` -Keep `GangGroupEnv` (`FLUENCE_GANG_GROUP`) — it now tells the **producer's** -sidecar which consumer group to ungate. **Delete** the separate-submitter +Keep `GangGroupEnv` (`FLUENCE_GANG_GROUP`) — it now tells the **scout's** +sidecar which worker group to ungate. **Delete** the separate-submitter constants and helpers: `SubmitterAnnotation`, `GangGroupAnnotation`, `SubmitterGroupSuffix`, `SubmitterPodSuffix`, and the functions `mutateSubmitter` and `ensureSubmitterPod`. Everything else in the file @@ -226,7 +226,7 @@ func (h *quantumHandler) Mutate(ctx context.Context, m webhook.MutatorAPI, pod * // No coordination: a standalone quantum pod, or an explicitly independent // member. The REAL submit happens in THIS pod; sidecar only for observe-only // telemetry. (independent mode routes every member here -> N standalone - // producers, each owning its task and its own queue wait.) + // scouts, each owning its task and its own queue wait.) if mode != CoordinationShared || g == "" || n <= 1 { ops := interceptorOps(pod) if observe { @@ -239,44 +239,44 @@ func (h *quantumHandler) Mutate(ctx context.Context, m webhook.MutatorAPI, pod * return ops } - // shared mode: promote one member to producer; the rest are gated consumers. - if isProducer(ctx, m, pod, g) { - return h.mutateProducer(ctx, m, pod, g) + // shared mode: promote one member to scout; the rest are gated workers. + if isScout(ctx, m, pod, g) { + return h.mutateScout(ctx, m, pod, g) } - return h.mutateConsumer(ctx, m, pod, g, n) + return h.mutateWorker(ctx, m, pod, g, n) } -// mutateProducer: index-0 member. Its own group-of-one (minCount 1) so it +// mutateScout: index-0 member. Its own group-of-one (minCount 1) so it // schedules alone and runs the REAL submit; sidecar polls the task and ungates -// the consumer group. NOT gated, no faux. The producer is one of the N members, +// the worker group. NOT gated, no faux. The scout is one of the N members, // so the application is NOT run an extra time. -func (h *quantumHandler) mutateProducer(ctx context.Context, m webhook.MutatorAPI, pod *corev1.Pod, group string) []spec.Op { - pg := group + ProducerGroupSuffix +func (h *quantumHandler) mutateScout(ctx context.Context, m webhook.MutatorAPI, pod *corev1.Pod, group string) []spec.Op { + pg := group + ScoutGroupSuffix m.EnsurePodGroup(ctx, pod.Namespace, pg, pod.Name, 1) ops := linkGroupOps(pod, pg) ops = append(ops, interceptorOps(pod)...) // tags the real submit - ops = append(ops, roleEnvOps(pod, RoleProducer)...) // FLUENCE_COORDINATION_ROLE=producer + ops = append(ops, roleEnvOps(pod, RoleScout)...) // FLUENCE_COORDINATION_ROLE=scout sc := sidecarFor(m) sc.EnsureRBAC(ctx, pod.Namespace) - // Tell the sidecar which consumer group (the base group) to list + ungate. + // Tell the sidecar which worker group (the base group) to list + ungate. ops = append(ops, sc.ContainerOps(pod, false, []corev1.EnvVar{{Name: GangGroupEnv, Value: group}})...) - log.Printf("[fluence-webhook] quantum producer %s/%s — group %s (ungates %q)", + log.Printf("[fluence-webhook] quantum scout %s/%s — group %s (ungates %q)", pod.Namespace, pod.Name, pg, group) return ops } -// mutateConsumer: a non-producer member. Joins the consumer gang -// (minCount N-1), is gated until the producer's task is ready, and is told its -// role (FLUENCE_COORDINATION_ROLE=consumer) + the producer's task id -// (FLUENCE_QUANTUM_JOB_ID). A consumer fetches the shared result by id; it never +// mutateWorker: a non-scout member. Joins the worker gang +// (minCount N-1), is gated until the scout's task is ready, and is told its +// role (FLUENCE_COORDINATION_ROLE=worker) + the scout's task id +// (FLUENCE_QUANTUM_JOB_ID). A worker fetches the shared result by id; it never // submits, so it gets neither the interceptor nor a faux flag. -func (h *quantumHandler) mutateConsumer(ctx context.Context, m webhook.MutatorAPI, pod *corev1.Pod, group string, n int32) []spec.Op { +func (h *quantumHandler) mutateWorker(ctx context.Context, m webhook.MutatorAPI, pod *corev1.Pod, group string, n int32) []spec.Op { m.EnsurePodGroup(ctx, pod.Namespace, group, pod.Name, n-1) ops := linkGroupOps(pod, group) - dep := Dependency{Kind: DependencyKindQuantumSubmit, Producer: group + ProducerGroupSuffix, Gate: QuantumGate} + dep := Dependency{Kind: DependencyKindQuantumSubmit, Scout: group + ScoutGroupSuffix, Gate: QuantumGate} ops = append(ops, dep.applyOps(pod)...) // gate + preempting priority + depends-on - ops = append(ops, consumerEnvOps(pod)...) // role=consumer + FLUENCE_QUANTUM_JOB_ID - log.Printf("[fluence-webhook] quantum consumer %s/%s — group %s minCount=%d, gated (role=consumer)", + ops = append(ops, workerEnvOps(pod)...) // role=worker + FLUENCE_QUANTUM_JOB_ID + log.Printf("[fluence-webhook] quantum worker %s/%s — group %s minCount=%d, gated (role=worker)", pod.Namespace, pod.Name, group, n-1) return ops } @@ -289,19 +289,19 @@ func coordinationMode(pod *corev1.Pod) string { return CoordinationIndependent } -// isProducer decides whether THIS pod is the gang's single producer. Indexed Job -// (recommended): completion index 0 is the producer — deterministic, race-free. -// Otherwise: first arrival claims the producer slot by the absence of the -// producer PodGroup (best-effort; prefer an indexed Job). -func isProducer(ctx context.Context, m webhook.MutatorAPI, pod *corev1.Pod, group string) bool { +// isScout decides whether THIS pod is the gang's single scout. Indexed Job +// (recommended): completion index 0 is the scout — deterministic, race-free. +// Otherwise: first arrival claims the scout slot by the absence of the +// scout PodGroup (best-effort; prefer an indexed Job). +func isScout(ctx context.Context, m webhook.MutatorAPI, pod *corev1.Pod, group string) bool { if idx, ok := pod.Annotations[CompletionIndexAnnotation]; ok { - return idx == ProducerIndex + return idx == ScoutIndex } c := m.Client() if c == nil { - return true // tests / no client: treat as producer + return true // tests / no client: treat as scout } - pg := group + ProducerGroupSuffix + pg := group + ScoutGroupSuffix if _, err := c.SchedulingV1alpha2().PodGroups(pod.Namespace).Get(ctx, pg, metav1.GetOptions{}); err == nil { return false // already claimed by an earlier arrival } @@ -314,11 +314,11 @@ nil-safe in Go (indexing a nil map yields the zero value, `ok=false`). ### Sidecar (Python) — no change -The producer's sidecar already resolves the vendor at runtime from +The scout's sidecar already resolves the vendor at runtime from `FLUXION_BACKEND`, discovers the task by the `fluence-pod-uid` tag, polls queue -position, and ungates the group named by `FLUENCE_GANG_GROUP` (now the consumer -group) at position==1, stamping `quantum-job-id` on each consumer before removing -its gate. That is exactly the existing flow with the producer in place of the +position, and ungates the group named by `FLUENCE_GANG_GROUP` (now the worker +group) at position==1, stamping `quantum-job-id` on each worker before removing +its gate. That is exactly the existing flow with the scout in place of the submitter pod. --- @@ -335,11 +335,11 @@ Unchanged classical-gang coverage plus a shared-mode assertion: - **Classical gang (regression):** keep `test/e2e/gang/*`. A `minCount=N` classical PodGroup schedules all-or-nothing. This proves the generic gang machinery is intact (the change only adds a quantum-pod deferral to `gang.Applies`). -- **Shared consumer gang (new assertion):** submit a `coordination: shared` - indexed Job of N. Assert: exactly one `-producer` PodGroup (minCount 1) and - one `` PodGroup (minCount N−1); the producer runs while the N−1 consumers +- **Shared worker gang (new assertion):** submit a `coordination: shared` + indexed Job of N. Assert: exactly one `-scout` PodGroup (minCount 1) and + one `` PodGroup (minCount N−1); the scout runs while the N−1 workers are `SchedulingGated`; after ungate the N−1 schedule **together** (gang), not - one-by-one. This proves gang scheduling still holds for the consumer group. + one-by-one. This proves gang scheduling still holds for the worker group. ### Requirement 2 — Both modes work and shared beats a traditional gang @@ -351,7 +351,7 @@ Three arms, same N, same workload (the QAOA sampler), same backend: | arm | how | expected node-seconds during queue wait | correctness | |-----|-----|------------------------------------------|-------------| | **traditional gang** (baseline) | N pods all running, each waits the full queue (no Fluence coordination — e.g. a plain native gang, or `independent` with N=N) | ≈ **N × T_queue** | N pods each run; if they each submit, N real tasks | -| **shared** (new) | `coordination: shared` indexed Job, N pods | ≈ **1 × T_queue** (producer only; consumers gated) | **1** real task; all N pods produce the **same** result; app runs N times, never N+1 | +| **shared** (new) | `coordination: shared` indexed Job, N pods | ≈ **1 × T_queue** (scout only; workers gated) | **1** real task; all N pods produce the **same** result; app runs N times, never N+1 | | **independent** (new) | `coordination: independent` indexed Job, N pods | ≈ **N × T_queue** (no coordination possible) | N distinct tasks/results; correct and the user's explicit choice (reported as the honest baseline, **not** claimed as an improvement) | Headline comparison is **shared vs traditional**: same observable result to the N @@ -362,7 +362,7 @@ pods, but shared idles ~1 node through the queue wait instead of N, saving Instrumentation (reuse the Experiment 2 harness): - per-pod `TIMING` lines → derive each pod's gated interval vs running interval; sum running-but-pre-result node-seconds per arm. -- producer's sidecar logs queue position over time → T_queue. +- scout's sidecar logs queue position over time → T_queue. - assert real-submit count: shared = 1 (one tagged task on the backend), independent/traditional = N (count tagged tasks). - assert shared correctness: all N pods log the **same** task id / result hash. @@ -375,11 +375,11 @@ traditional and independent rise ~linearly in N; shared stays ~flat at one node. ### Build/run notes -- The producer/consumer split needs no new image: producers and consumers run the +- The scout/worker split needs no new image: scouts and workers run the same role-aware sampler; the branch is `FLUENCE_COORDINATION_ROLE` - (producer submits; consumer fetches the shared result by `FLUENCE_QUANTUM_JOB_ID`). + (scout submits; worker fetches the shared result by `FLUENCE_QUANTUM_JOB_ID`). - Use an **indexed** Job (`completionMode: Indexed`, `parallelism == completions == N`) - so producer election is deterministic (index 0) and `resolveGangSize` reads N + so scout election is deterministic (index 0) and `resolveGangSize` reads N from the owner. Stamp `fluence.flux-framework.org/coordination` in the pod template's annotations. - kind/mock runs exercise the structural assertions (groups, gating, ungate diff --git a/docs/handlers.md b/docs/handlers.md index ee70519..17ea0d7 100644 --- a/docs/handlers.md +++ b/docs/handlers.md @@ -3,8 +3,8 @@ Fluence's value is not creating gangs (Kubernetes 1.36 native gang scheduling already does that). It is **customizing the gang on the fly based on the resources a pod requests** — e.g. a shared quantum gang becomes a size-1 -producer gang plus a size-(N-1) consumer gang, with the producer running a -sidecar that ungates its consumers when the quantum task is ready. +scout gang plus a size-(N-1) worker gang, with the scout running a +sidecar that ungates its workers when the quantum task is ready. ## Handlers diff --git a/docs/qrmi-coordination.md b/docs/qrmi-coordination.md new file mode 100644 index 0000000..0c87063 --- /dev/null +++ b/docs/qrmi-coordination.md @@ -0,0 +1,157 @@ +# QRMI in Fluence + +Fluence has a **QRMI provider** that provides co-scheduling using [QRMI](https://github.com/qiskit-community/qrmi) (the Quantum Resource Management Interface) to open a session by the scout, and then deliver the session ID to any classical workers to use. Please read [`quantum-scheduling.md`](quantum-scheduling.md) first for the general design we are using across environments and interfaces. + +## One provider, many vendors + +QRMI is deliberately vendor-agnostic: a thin middleware (Rust core, Python/C +bindings) that exposes **one** lifecycle API over any resource it supports — +IBM Quantum, Pasqal, IQM, Alice & Bob, and more: + +```python +from qrmi import QuantumResource, ResourceType, Payload, TaskStatus + +qrmi = QuantumResource("ibm_marrakesh", ResourceType.QiskitRuntimeService) +lock = qrmi.acquire() # reserve the device -> session token +job_id = qrmi.task_start(Payload.QiskitPrimitive(input=..., program_id=...)) +status = qrmi.task_status(job_id) # TaskStatus.{Queued,Running,Completed,...} +result = qrmi.task_result(job_id) +qrmi.task_stop(job_id); qrmi.release(lock) +``` + +So Fluence has **one** provider — `fluence.providers.qrmi` — that drives this +API. It contains no per-vendor branches. Every vendor detail comes from the +Fluence **resource attributes**, which the scheduler stamps onto the match and +the webhook injects as `FLUXION_*` env into the sidecar and workload containers: + +| resource attribute | env | role | +|---|---|---| +| `qrmi_type` | `FLUXION_QRMI_TYPE` | the QRMI resource type — the single vendor-selecting knob (`qiskit-runtime-service`, `ibm-quantum-system`, `pasqal-cloud`, …). Maps to the QRMI `ResourceType` enum member. | +| `transport` (optional) | `FLUXION_TRANSPORT` | explicit `qrmi` marker for routing (see below). | +| `backend` (the vertex name) | `FLUXION_BACKEND` | the backend/resource id (`ibm_marrakesh`, `FRESNEL`, …). | +| `vendor` (optional) | `FLUXION_VENDOR` | informational / per-vendor sidecar-image routing. | + +Adding a new QRMI vendor is therefore a **config** change (a `qrmi_type` in +`resources.yaml`), not a code change. The `qrmi_type → ResourceType` mapping is +derived generically (kebab-case → the enum's PascalCase member, acronym-aware so +`ibm-quantum-system → IBMQuantumSystem`), with a tiny alias table for legacy +names (`direct-access → IBMQuantumSystem`) and a version-defensive fallback. + +## Routing (which provider handles a device) + +The sidecar resolves the provider at runtime from the injected attributes +(`resolve_from_env`). The QRMI provider matches when: + +1. `transport == qrmi` (explicit and future-proof — recommended in the resource + config), **or** +2. no `transport` is set but `qrmi_type` is present and is **not** a `braket-*` + type. Fluence uses `braket-*` qrmi_types for Braket-reached devices and the + QRMI resource-type strings for QRMI-reached ones, so "a qrmi_type that isn't + braket" cleanly means QRMI. This is data-driven: whatever the config sets + decides it, with no per-vendor list in the code. + +Braket and QRMI never collide. + +## Discovery: the one thing QRMI changes vs. Braket + +Braket's `SearchQuantumTasks` lets the sidecar find the scout's task by a tag +the interceptor stamped. QRMI has no tag-able submit and no searchable tag — +`task_start(payload)` takes no tags and returns an opaque id. So instead of a +vendor-side search, the provider uses an **in-pod handoff** (`fluence.handoff`): + +- the scout's interceptor patches `QuantumResource.task_start`, captures the + returned id, and writes `{job_id, session_id?, tier}` to + `$FLUENCE_HANDOFF_DIR/task.json`; +- the sidecar (always the same pod) reads that file, then polls + `task_status(job_id)` for readiness. + +The webhook mounts one writable `emptyDir` for this channel into the submitting +container(s) and the sidecar. It is RBAC-free because the scout and its +sidecar are co-located. This is a property of QRMI, not of any vendor, and the +handoff module is reusable by any future provider whose SDK can't be searched. + +## Free vs. paid: the acquire nuance (also vendor-neutral) + +QRMI separates *acquisition* (`acquire`/`release`, which reserves the device and, +where supported, grants a priority/session queue) from *submission*. + +| | acquire() | queue | how Fluence coordinates | +|---|---|---|---| +| **free / open** | not permitted (raises) | fair-share | submit-only; ungate when the task starts RUNNING | +| **paid / premium** | returns a session token | priority | same wait; session token captured and handed to workers | + +The provider **auto-detects** the tier — no config, no change to the user's code +path: + +- The interceptor patches `acquire()` so a **success** records the session token + + `tier=premium`, and a **failure** records `tier=free` and **re-raises + unchanged** so a free-tier app sees the real error and proceeds submit-only. + Fluence never forces `acquire()`. +- `task_start()` publishes `{job_id, session_id?, tier}` to the handoff. + +The **ungate decision is identical in both tiers**: ready once `task_status` is +`RUNNING` or terminal — the QRMI-visible analogue of Braket's +`queue_position == 1`. The tier only affects how long the wait is and whether a +session token exists. When present, the sidecar stamps it onto each worker +(`fluence.flux-framework.org/quantum-session-id` → `FLUENCE_QUANTUM_SESSION_ID`) +so a role-aware worker can submit into the same priority session. + +## Holding the session for the workers + +The scout acquired the session, so it would close when the scout +application calls release, long before the workers run. The interceptor +patches release on a scout so it waits for a marker the sidecar writes. +After the sidecar ungates the workers it waits for every pod in the gang +other than itself to reach Succeeded or Failed, then writes the marker to the +handoff volume and exits. The scout application then releases and exits, +the pod finishes, and fluence frees its allocation. The scout keeps its +session and its qpu the whole time, the way the scout does in flux-quantum. + +Two timeouts bound the hold. FLUENCE_WORKER_TIMEOUT on the sidecar is how +long it waits for the group, and FLUENCE_SESSION_HOLD_TIMEOUT in the scout +container is how long release waits for the marker. Both default to zero, +which waits without limit. + +> QRMI exposes no numeric queue position, so `queue_position()` is best-effort +> telemetry (1 once RUNNING, else unknown) and never required for the ungate. + +## The sidecar image (one image, per-vendor escape hatch) + +The default sidecar is a single provider-agnostic image (`pip install ".[all]"` += braket + `qrmi[ibm]`) that resolves the vendor at runtime. QRMI's `ibm` and +`iqm` extras pin incompatible Qiskit versions, so `all` includes `ibm` (the +default reachable QRMI vendor) but not `iqm`. When a vendor's SDK can't share the +combined image, build a slim single-vendor image and route to it: + +```bash +docker build --build-arg EXTRAS=iqm -t ghcr.io/you/fluence-sidecar:iqm python/ +``` + +Point the webhook at it globally (`FLUENCE_SIDECAR_IMAGE_`, e.g. +`FLUENCE_SIDECAR_IMAGE_IQM`) or per-pod +(`fluence.flux-framework.org/sidecar-image`). A pod that hints a vendor +(`require-vendor: iqm`, or a `require-qrmi_type` the webhook maps to a vendor) +then picks up `FLUENCE_SIDECAR_IMAGE_IQM` automatically. + +## Credentials + +The scheduler never needs vendor credentials. The workload container holds them +(e.g. IBM's `IBM_CLOUD_TOKEN`/`IBM_CLOUD_CRN` or QRMI's `QRMI_*` env), and the +webhook copies the workload's secret/configMap-sourced env onto the sidecar so +it can build its own `QuantumResource` handle and poll `task_status`. Polling +status needs no `acquire()`, so this works for both tiers. + +## Files + +- `python/fluence/providers/qrmi.py` — the provider (interceptor + sidecar). +- `python/fluence/handoff.py` — the shared in-pod job-id channel. +- `pkg/webhook/handlers/quantum.go` — mounts the writable handoff volume, sets + `FLUENCE_HANDOFF_DIR`, surfaces `FLUENCE_QUANTUM_SESSION_ID`, and resolves the + per-vendor sidecar image. + +## Tests + +```bash +PYTHONPATH=python python3 python/tests/test_qrmi.py # provider + handoff + tiers + multi-vendor routing +go test ./pkg/webhook/... # handoff wiring + per-vendor image selection +``` diff --git a/docs/quantum-scheduling.md b/docs/quantum-scheduling.md index de32220..5270106 100644 --- a/docs/quantum-scheduling.md +++ b/docs/quantum-scheduling.md @@ -1,455 +1,193 @@ -# Quantum-Classical Scheduling Coordination in Fluence - -## Abstract - -Hybrid quantum-classical workflows submit work to two independent queues: -the Kubernetes scheduler (classical compute) and a QPU vendor API (quantum -execution). Classical pods waste node resources while waiting for QPU queue -results. Fluence's coordination system thus gates classical consumer pods until -the QPU task is one position from executing, then releases them with high -priority so they preempt lower-priority work and start immediately as the -QPU result arrives. Yes, it could be the case the one task in the queue before -it takes a long time, but I think this is an improved approach than having consumer -pods running (and waiting) for a much longer queue. This only is important -given that you have gangs, or producer/consumer designs where one member is -launching the quantum work and otherwise the other members would be waiting and -doing nothing (and wasting resources). - -## 1. The Two-Queue Problem - -When a hybrid quantum-classical job runs on Kubernetes, the classical pod -starts immediately and blocks waiting for the QPU result. The QPU task -enters a vendor-managed queue shared across all users. The classical pod -consumes node resources — CPU, memory, potentially GPU — for the entire -duration of the QPU queue wait, which may be minutes to hours on real -hardware. - -This waste scales with concurrency. With N concurrent hybrid jobs, each pod -idles for the full QPU queue wait. On real QPU backends (IQM Garnet, IQM -Emerald) we measure 15–30% classical idle fraction at N=10, rising to over -70% for individual pods at N=20. Wall time scales linearly with concurrency -on real QPUs — submitting 20 jobs takes 5–8× longer than 1 job due to -self-imposed queue depth. - -## 2. Why Existing Mechanisms Are Insufficient - -### 2.1 Fluxion reservations - -Fluxion's backfill reservation policies (EASY, Conservative, Hybrid) compute -a future `time_at` from the internal resource graph — when currently running -classical jobs will finish. They cannot accept an externally-supplied time -derived from a vendor queue. Without a reliable `time_at`, a reservation -degenerates to a pending job. All reservations are cancelled and recomputed -from scratch at the start of every scheduling loop. - -QPU queue time is unknowable in advance. It depends on other -users' submissions, hardware calibration windows, and network latency. -Average task time per QPU cannot be estimated reliably. Therefore Fluxion -reservations cannot help with the two-queue problem. I learned that we are -working on "advanced reservations" that are more like a hold, but it is -not clear if that can be merged soon. - -### 2.2 Scheduling gates alone - -A scheduling gate holds a pod out of the scheduling queue entirely, consuming -no node resources. But ungating N pods simultaneously on a busy cluster -creates a race — resources may not be available when ungating occurs, and -the Fluxion graph allocation happens after ungating, not before. Without -priority, ungated pods compete equally with all pending work. - -### 2.3 Preemption alone - -Submitting classical pods with a high PriorityClass causes Kubernetes to -evict lower-priority pods immediately at submit time — during the entire QPU -queue wait — which is worse than the original problem. - -## 3. Design - -The design combines four mechanisms: - -1. **SDK interceptor** — tags every submitted QPU task with the pod UID so the - sidecar can find it (staged only on pods that submit) -2. **Fluence webhook** — splits a shared quantum gang into one producer and N-1 - gated consumers; injects the sidecar into the producer -3. **Sidecar controller** — discovers the QPU task, polls queue position, - ungates the consumers when position==1 -4. **High-priority ungating** — consumers preempt lower-priority work at the - last responsible moment - -### 3.0 When Fluence acts: the decision matrix - -Two orthogonal properties of a pod admitted with `schedulerName: fluence` -determine what Fluence does: - -- **Q (quantum?)** — does any container request a quantum resource - (`fluxion.flux-framework.org/qpu`)? If so, Fluence is scheduling the quantum - work and there is a vendor backend behind it. -- **G (gang?)** — does the pod carry `fluence.flux-framework.org/group`? - -A third property applies only to quantum gangs: the **coordination mode** -(`fluence.flux-framework.org/coordination`, default `independent`). In `shared` -mode the gang produces ONE quantum task that all members share; in `independent` -mode every member does its own quantum work. - -| | not quantum | quantum | -|--------------|------------------------|--------------------------------------------------------------------------------| -| **not gang** | group of 1 (nothing) | inject provider interceptor + env; **sidecar only in observe-only mode if telemetry requested** (nothing to ungate) | -| **gang** (independent) | gang-schedule only | every member is a standalone producer: interceptor + env, real submit, no gate | -| **gang** (shared) | — | producer (index 0): interceptor + env + sidecar, real submit, not gated, group-of-one `-producer`, role=producer; consumers: gate + role=consumer + producer's task id, gang `` (minCount N-1) | - -The crucial rule: **sidecar/interceptor injection is triggered by the quantum -resource request, not the group label.** The group label only controls gang -scheduling and (in shared mode) the producer/consumer split. A grouped pod that -requests no quantum resource (e.g. a classical pod that happens to set -`BRAKET_DEVICE` itself) is just gang-scheduled — Fluence injects no sidecar, -because there is no quantum work for it to coordinate. `BRAKET_DEVICE` (or any -direct device selection by the user) is the signal that Fluence is *not* -scheduling the quantum resource; `fluxion.flux-framework.org/qpu` is the signal -that it is. - -### 3.1 User interface - -The user labels all pods in a workflow group with: +# Quantum and classical scheduling in Fluence + +## The problem + +A hybrid quantum and classical job submits to two queues, the Kubernetes +scheduler for the classical pods and the vendor queue for the quantum task. The +classical pods start right away and then sit on their nodes waiting for a result +that may be minutes or hours away. That waste grows with concurrency. On real +QPUs we measured 15 to 30 percent classical idle time at ten concurrent jobs and +over 70 percent for single pods at twenty. + +Fluence holds the classical pods out of the cluster until the quantum task is +about to run, then releases them with high priority so they start as the result +lands. I think this beats having the pods run and wait, even when the task +ahead in the queue turns out to be slow. Better vendor estimates would help +everyone. This only matters when a gang of pods is waiting on one quantum task, +or a loop keeps resubmitting one. + +## The design + +A shared quantum gang is split into one scout and N minus one workers. + +- The scout is one of the N pods. It runs the application, submits the one real + quantum task, and is never gated. It carries a sidecar. +- The workers are the other pods. They start with a scheduling gate, so they + hold no node resources while the task waits in the vendor queue. They are + told their role and, once ungated, the job id and the session. +- The sidecar runs in the scout pod. It finds the task the scout submitted, + polls the vendor queue, and when the task is next in line it ungates the + workers. It then waits for the workers to finish, so the scout keeps its + session until they are done. +- An interceptor is staged into the scout container. It patches the vendor SDK + so the sidecar can find the task. For Braket it tags the task with the pod + uid. For QRMI, which has no tag search, it writes the job id and the session + to a small file the sidecar reads. + +The workers are released with a high priority class, so if the cluster filled +up while they waited they preempt lower priority work rather than queue behind +it. This is what makes the hold useful. Fluence frees the preempted pods' graph +allocations as they go. + +## What fluence looks at + +Two properties of a pod with schedulerName fluence decide what happens. + +- Does any container request a quantum resource, fluxion.flux-framework.org/qpu? + Then there is a vendor backend behind the pod and fluence places it against + the graph. +- Does the pod carry the group label fluence.flux-framework.org/group? Then it + is part of a gang, scheduled through the native PodGroup API in Kubernetes + 1.36 and later. + +A quantum gang also has a coordination mode, the annotation +fluence.flux-framework.org/coordination. The default is independent, every pod +does its own quantum work. In shared mode the gang shares one task and gets the +scout and worker split. The quantum resource request is what triggers the +interceptor and sidecar. The group label only controls gang scheduling and the +split. + +## What the user writes ```yaml metadata: labels: fluence.flux-framework.org/group: my-workflow annotations: - # only for a quantum gang that shares ONE task across members: fluence.flux-framework.org/coordination: shared spec: schedulerName: fluence ``` -`coordination` defaults to `independent` (every member does its own quantum -work). Set it to `shared` when the members should share a single quantum task — -then Fluence promotes one member (the indexed-Job completion index 0) to producer -and gates the rest as consumers. The user authors no roles and no submitter pod; -the split is derived from the completion index the Job controller already stamps. - -I initially started with having the user create a PodGroup object, and I found -that annoying. I do not want to require a PodGroup object when an annotation is easier, -and then I have fine-grained control of what the groups looks like. Fluence can handle -everything else automatically. - -The namespace distinction: -- `fluence.flux-framework.org/*` — Fluence scheduler-plugin concerns - (group label, coordination mode, gate name) -- `fluxion.flux-framework.org/*` — Fluxion resource-graph concerns - (extended resource types, backend attribute env vars) - -### 3.2 Webhook behavior (handler architecture) - -The webhook core is domain-agnostic: it owns the Mutator, a handler dispatcher, -per-namespace PodGroup/RBAC provisioning, the Model C package staging, the HTTP -entrypoint, and TLS. It knows nothing about quantum. Behavior is expressed as a -set of **handlers** that self-register (`webhook.Register` from each handler -package's `init()`); the core never names a handler. Each handler declares its -own trigger via `Applies(ctx, MutatorAPI, pod)` and contributes patch ops via -`Mutate`. A pod flows through every handler that applies, and their ops are -concatenated. This keeps quantum entirely encapsulated in one handler — adding -or removing behavior never touches the core. - -The three handlers (`pkg/webhook/handlers/`): - -**`fluxion` (`fluxion.go`)** — applies when any container requests a -`fluxion.flux-framework.org/*` resource. Injects the `FLUXION_*` env contract -(backend + attributes) sourced from the annotations the scheduler writes in -PreBind. Generic to all Fluxion resources. - -**`gang` (`gang.go`)** — applies when the pod carries the group label **and does -not request the quantum resource** (a quantum pod is gang-scheduled by the quantum -handler instead, which owns the producer/consumer split). Creates a Fluence-owned -PodGroup on first admission and stamps `spec.schedulingGroup.podGroupName` on -every pod in the group so the scheduler gangs them. The user only ever sets the -LABEL; the webhook translates it into the native field, so the user never creates -a PodGroup or knows it exists. Knows nothing about quantum — a purely classical -gang is fully handled here, with no sidecar. - -**`quantum` (`quantum.go`)** — the only handler that knows about quantum -resources, gates, coordination, and observe semantics. A quantum task's circuit -comes from user code, so the pod that defines a task must RUN to submit it: submit -and gate are mutually exclusive per pod, and gating only helps pods that do not -submit. The handler therefore routes each quantum pod to one of three roles: -- **standalone / independent** (a lone quantum pod, or any member of a gang in - the default `independent` mode): gets the interceptor staged (real mode) so its - own task is tagged, performs its own real submit, is never gated, and gets the - sidecar only when observe-only telemetry is requested. Independent mode means N - members run N tasks and hold N node-waits — honest physics, the user's explicit - default. -- **producer** (in `shared` mode, the completion index 0 member): its own - group-of-one `-producer` (minCount 1) so it schedules alone and runs the - SINGLE real submit; interceptor in real mode; gets the **sidecar**, told which - consumer group to ungate (`FLUENCE_GANG_GROUP`); never gated. The producer is - one of the N members, so the application runs exactly N times — never N+1. -- **consumer** (in `shared` mode, the other N-1 members): joins the `` - gang (minCount N-1), gets the `quantum.braket/ready` scheduling gate (entering - `SchedulingGated` — invisible to Fluxion, consuming no resources — until the - producer's sidecar ungates it), and is told its role - (`FLUENCE_COORDINATION_ROLE=consumer`) and the producer's task id - (`FLUENCE_QUANTUM_JOB_ID`, stamped at ungate). A consumer does **not** submit — - it fetches the shared result by that id — so it gets neither the interceptor nor - any faux flag. - -Role is decided by the **completion index**, not resource request or admission -order. In an indexed Job every pod is identical — same group label, same image, -every pod requests the quantum resource — so the producer is simply the pod the -Job controller stamps with `batch.kubernetes.io/job-completion-index: "0"`; every -other index is a consumer. (For loose pods with no completion index, the first -arrival claims the producer slot by the absence of the `-producer` -PodGroup; an indexed Job is recommended for deterministic election.) The two -groups carry independent minCounts (producer=1, consumers=N-1), which is what lets -the producer schedule and submit while the consumers stay gated — no deadlock, and -no separate submitter pod. - -The workload is **role-aware**: every shared-mode pod is told its role positively -via `FLUENCE_COORDINATION_ROLE` (the webhook's election is the single source of -truth), and the application branches on it — the producer submits, a consumer -fetches the shared result by `FLUENCE_QUANTUM_JOB_ID`. The same image plays both -roles with one cheap branch; there is no submit-interception magic and no faux -flag. - -### 3.3 Interceptor and Model C delivery - -The interceptor tags each submitted quantum task with the pod UID so the sidecar -can discover it. It must run inside the **user's** application container — which -does not have the `fluence` package installed. Rather than require a user -install or mount a hand-assembled text file, Fluence uses **Model C**: the -`fluence` Python package is built into the sidecar image, and the webhook stages -it into the user container at admission. - -The quantum handler's `InterceptorOps`: -1. adds a shared `emptyDir` volume; -2. injects an **init container** (the sidecar image) running - `python -m fluence.stage `, which copies the pure-Python `fluence` - package plus a `sitecustomize.py` into that volume; -3. mounts the volume into the quantum container and prepends `` to - `PYTHONPATH`, and sets `FLUENCE_POD_UID`. - -Python imports `sitecustomize` automatically on every interpreter start — -including non-interactive `python app.py`, unlike `PYTHONSTARTUP`, which fires -only for interactive sessions. The staged `sitecustomize.py` does a guarded -`import fluence.interceptor`, which asks every registered provider to install -its tag hook. Each provider fail-soft skips if its vendor SDK is not importable -in the user container, so the one staged package works for any vendor and never -breaks the user app. - -For Braket the hook monkey-patches `AwsDevice.run()` to add a -`fluence-pod-uid` tag to every task submission: - -```python -def patched_run(self, task_specification, *args, **kwargs): - pod_uid = os.environ.get("FLUENCE_POD_UID", "") - if pod_uid: - tags = kwargs.get("tags", {}) - tags["fluence-pod-uid"] = pod_uid - kwargs["tags"] = tags - return original_run(self, task_specification, *args, **kwargs) -``` - -This is completely transparent to the user application — no code changes, no -package install, no vendor SDK added to the user image (the hook patches -whatever SDK the user already has). - -### 3.4 Sidecar controller - -The sidecar runs as a container alongside the producer pod, sharing its AWS -credentials and network namespace. - -```console -1. READ FLUXION_ARN, FLUENCE_POD_UID from env - -2. DISCOVER task by tag: - search_quantum_tasks(filters=[ - deviceArn == FLUXION_ARN, - tags:fluence-pod-uid == FLUENCE_POD_UID - ]) - Poll every 10s, timeout after 300s. - On timeout: fall back to time-window heuristic (tasks submitted - after pod start time on the same device). - -3. DISCOVER consumer pods: - List pods in namespace with fluence.flux-framework.org/group label - matching this pod's group, having quantum.braket/ready gate present. - -4. POLL task.queue_position() every 30s. - Log position for experiment instrumentation. - -5. WHEN is_ready_to_ungate(task) (position == 1 OR state == RUNNING): - For each consumer pod: - kubectl annotate pod fluence.flux-framework.org/quantum-job-id= - kubectl patch pod --type=json \ - -p='[{"op":"add","path":"/spec/priorityClassName", - "value":"fluence-quantum-classical"}, - {"op":"remove","path":"/spec/schedulingGates/0"}]' +That is all. The scout is the pod with completion index 0 in an indexed Job, +which the Job controller stamps, so an identical template yields the roles +with no state and no race. I started out requiring a PodGroup object and found +it annoying, so the webhook creates it from the label. + +The two prefixes are deliberate. fluence.flux-framework.org is the scheduler +plugin, the group, the mode, the gate. fluxion.flux-framework.org is the +resource graph, the extended resources and the backend attributes. + +## The webhook + +The webhook core knows nothing about quantum. It dispatches to handlers that +register themselves, and a pod flows through every handler that applies. There +are three. + +- fluxion applies to any pod requesting a fluxion resource and injects the + FLUXION_* environment from the annotations the scheduler writes. +- gang applies to grouped pods that request no quantum resource. It creates the + PodGroup and links the pod to it. +- quantum applies to any pod requesting the quantum resource. A standalone or + independent pod gets the interceptor and submits for itself. In shared mode + the scout gets its own PodGroup of one, named group-scout, the interceptor, + the handoff volume, and the sidecar. A worker joins the gang PodGroup of + N minus one, is gated, gets the high priority class, and is given its role + plus downward API variables for the job id and the session, read from + annotations the sidecar stamps at ungate. + +The interceptor is delivered by an init container that copies the fluence +package into a shared volume on the application container's PYTHONPATH. Python +imports sitecustomize on every start, so the interceptor loads before user code +and patches whichever vendor SDK is present. No user install and no change to +the user image. + +The quantum handler also enforces admission control when it is turned on. See +pkg/webhook/admission.go. A quantum group is only admitted when the cores all +of its pods will need fit in what is reachable, free cores plus cores held by +preemptible pods, less what is promised to groups that have not started. + +## The sidecar + +1. Resolve the provider from the FLUXION_* environment. +2. Find the task. Braket searches by the pod uid tag. QRMI reads the handoff file. +3. Find the workers, the pods in the group that still carry the gate. +4. Poll the task. Log the position for experiments. +5. When the task is next in line, or already running, annotate each worker + with the job id and the session, then remove the gate. +6. Wait for the workers to reach Succeeded or Failed, write the done marker so + the scout's release proceeds, and exit. + +If the task is never found or fails, the sidecar ungates anyway so the workers +are not stranded, and exits non zero so it shows. + +## Priority and preemption + +Workers get the fluence-quantum-classical priority class, value one million, +at creation, since priorityClassName cannot change after a pod exists. The +gate keeps the priority from mattering until ungate. Once ungated, Kubernetes +preemption evicts lower priority pods to make room, and fluence's pod deletion +handler cancels their Fluxion allocations so the graph is free for the workers. + +Because each gang is released when its own task is next in line, classical +resources follow quantum execution order rather than submission order. A gang +on a quiet backend gets its nodes before an earlier gang on a busy one. + +## Providers + +The webhook cannot know the vendor at admission, since the scheduler picks the +backend later. So the interceptor is staged into every quantum container and +asks every provider to install its hook, each skipping if its SDK is absent. +The sidecar resolves the one provider that matches at runtime. + +A provider implements both halves in one module under +python/fluence/providers and calls register at import. -6. EXIT ``` - -The priority class and gate removal are applied atomically in one patch. -This ensures consumers enter the scheduling queue with high priority -immediately, without a window where they are ungated but low-priority. - -### 3.5 Priority and preemption - -The `fluence-quantum-classical` PriorityClass (value: 1,000,000) is applied -by the sidecar at ungate time, not by the webhook at pod creation. Setting -it at creation time causes an admission controller conflict (priority integer -already defaulted to 0). - -When consumers are ungated with high priority, Kubernetes preemption evicts -lower-priority pods to make room. Fluence's pod deletion informer catches -these evictions, calls `Cancel(jobid)` in Fluxion, and frees the graph -vertices so Fluxion can allocate them to the incoming high-priority consumers. - -### 3.6 Classical allocation follows quantum execution order - -Because each workflow's gate is removed independently when its QPU task -reaches position==1, workflows whose QPU tasks execute earlier get classical -resources earlier — regardless of submission order. A workflow submitted to -a quiet backend gets its classical resources before one submitted earlier to -a busy backend. This aligns classical resource allocation with actual quantum -execution order across heterogeneous backends. - -### 3.7 Provider interface - -The webhook is provider-agnostic. It cannot know the vendor at admission time, -because the scheduler may choose a backend from a mixed set of vendors — the -vendor is only fixed once the scheduler writes the backend annotation in -PreBind, after the webhook has run. The design therefore splits by which -artifact needs the vendor and when: - -- **Interceptor** — runs in the user's application container and is staged there - at admission, before the vendor is known. It is the `fluence` package's - `interceptor` module, which on import asks *every* registered provider to - install its tag hook; each provider fail-soft skips if its vendor SDK is not - importable in that container. So the one staged package works in any quantum - container regardless of which SDK is present. This is *forced* by mixed-vendor - scheduling: the webhook cannot pick a single provider at stage time. Delivery - is Model C (§3.3): an init container stages the package into a shared volume on - the user container's PYTHONPATH — no ConfigMap, no user install. - -- **Sidecar** — runs in the Fluence sidecar image and resolves the vendor at - *runtime* from the backend annotation (`FLUXION_BACKEND`). It loads only the - one provider that matches, so an unrelated provider's SDK failure never affects - the run. - -Every provider implements a common interface (`python/fluence/providers/base.py`), -holding both halves of its mechanism: - +install_interceptor(pod_uid) patch the SDK submit call +matches(attrs) does this provider own this backend +find_my_task(pod_uid, ...) the task, as an opaque object +is_ready_to_ungate(task) next in line or already running +queue_position(task) optional, for telemetry +job_id(task) the id handed to workers +session_id(task) the session handed to workers, if any ``` -Provider: - name # "braket", "ibm", ... - install_interceptor(pod_uid) # interceptor half: patch the SDK submit call - matches(vendor, backend) # runtime resolution from backend annotation - find_my_task(pod_uid, ...) # search by the fluence-pod-uid tag → opaque Task - is_ready_to_ungate(task) # decision primitive: position==1 OR running - queue_position(task) # optional richer telemetry; None if unavailable - job_id(task) # cross-vendor id handed to consumers (NOT the ARN) -``` - -Vendor-specific identifiers (a Braket task ARN, an IBM job id, a GCP operation -name) are never named in the interface — they live behind the opaque `Task` and -are surfaced generically through `job_id()`. The interceptor hook and -`find_my_task` are joined only by a shared tag convention (`fluence-pod-uid`): -the hook stamps the tag at submission, `find_my_task` searches for it. -`is_ready_to_ungate` is the decision primitive and is always implementable, even -for vendors that do not expose a numeric queue position (it can key off the -QUEUED→RUNNING transition). `queue_position` is the optional richer signal used -for observe-only telemetry and position-series logging. +Braket and QRMI are implemented. QRMI covers IBM, Pasqal, IQM, and others +through one API, with the vendor chosen by the qrmi_type attribute. -Adding a vendor is one module under `python/fluence/providers/.py`: a -`Provider` subclass implementing both halves that calls `register(PROVIDER)` at -import, plus one import line in `providers/__init__.py`. Registration wires it in -for both the interceptor (all providers) and runtime sidecar resolution (the -matching provider). Nothing else changes — no build script, no concatenation. +A single quantum pod with no workers gets the interceptor and environment but +no sidecar. The label fluence.flux-framework.org/observe set to true adds the +sidecar in observe mode, which logs the queue position and ungates nothing. +Experiments use that to measure singletons and gangs the same way. -#### Observe-only (telemetry) mode - -A quantum pod that is *not* a gang (a single quantum pod, no consumers to ungate) -gets the interceptor and env only — no sidecar — by default, so no surprise -machinery is injected. Telemetry is opt-in via the label -`fluence.flux-framework.org/observe: "true"`, surfaced to the sidecar as -`FLUENCE_OBSERVE=true`. In observe-only mode the sidecar discovers the task and -polls `queue_position`, logging the series for measurement, but ungates nothing. -Experiments use this to get a uniform queue-position measurement path across -singleton and gang runs. - -## 4. Properties +## Properties | Property | Value | |---|---| -| User code changes required | None | -| User manifest changes required | Add group label + schedulerName | -| Classical resources during QPU wait | Zero (SchedulingGated) | -| QPU queue time estimation needed | No — position==1 is observable | -| Works across heterogeneous backends | Yes — any backend in Fluxion graph | -| Multi-vendor | Yes — provider interface, vendor resolved at runtime | -| Vendor API cooperation needed | No — SDK interceptor handles tagging | - -## 5. Limitations - -### 5.1 Preemption disrupts lower-priority work - -At position==1, consumers preempt running lower-priority pods. This work is -re-queued and eventually runs, but there is a disruption cost. A future -design using a `MatchReserveAt(time_at, spec)` Fluxion primitive — where -`time_at` is supplied by the QPU vendor via an ETA or task-start event — -would allow graceful node draining instead of preemption. No current QPU -vendor exposes such an API. - -### 5.2 Provider coverage - -The provider interface (§3.7) makes adding a vendor a matter of implementing -`find_my_task`/`is_ready_to_ungate`/`job_id` and an interceptor block. Braket is -implemented. IBM Qiskit Runtime is a natural next provider — it supports -submit-time `job_tags` and tag-based filtering via `QiskitRuntimeService.jobs`, -so both halves of the mechanism are expressible. Vendors whose APIs expose -neither tag-search nor a queue position would need a fallback discovery -heuristic (e.g. a time window) rather than the tag mechanism. - -### 5.3 Single task per workflow - -The sidecar tracks one QPU task ARN per producer pod. Parameter-shift gradient -estimation and other multi-circuit workflows require tracking a set of ARNs. -See the scatter design issue for the proposed extension. - -### 5.4 Namespace-scoped RBAC - -The webhook creates `fluence-sidecar` RBAC in each namespace on first use. -This is correct behavior — the sidecar only needs permissions in its own -namespace. A Helm chart or operator would manage this more cleanly. - -## 6. Future Work - -### 6.1 MatchReserveAt Fluxion primitive - -A new `MatchReserveAt(time_at, spec)` function in the Fluxion Go bindings -would allow an externally-supplied reservation time. The sidecar would feed -live QPU queue position into this estimate, enabling graceful node draining -rather than preemption. This requires the C++ reapi `match_allocate_multi` -function to be exposed through the Go bindings with a `starttime` parameter. - -### 6.2 Scatter design - -For workflows with N independent QPU tasks each paired with one classical -pod, an index-based pairing mechanism (`fluence.flux-framework.org/index`) -would allow the sidecar to ungate specific consumer pods when their specific -task reaches position==1. See the open scatter design issue. - -### 6.3 Vendor task-start events - -If QPU vendors exposed SNS/EventBridge notifications when a task transitions -from QUEUED to RUNNING, the sidecar could react to events rather than -polling. This would eliminate the 30s polling latency and enable more -precise ungating. - -### 6.4 PostFilter topology-aware preemption - -A custom Fluence `PostFilter` plugin would ask Fluxion which graph vertices -are blocking a high-priority consumer pod, then target preemption at exactly -those pods — rather than the default Kubernetes preemption which picks -lowest-priority pods regardless of graph topology. This ensures preemption -always produces a valid Fluxion allocation. +| User code changes | none | +| User manifest changes | the group label, the mode, and schedulerName | +| Classical resources during the wait | none, the workers are gated | +| Queue time estimate needed | no, next in line is observable | +| Heterogeneous backends | yes, anything in the graph | +| Vendors | Braket and everything QRMI reaches | + +## Limitations + +- Ungating preempts lower priority work. That work requeues and runs later, + but it is a disruption. A Fluxion reservation at a vendor supplied start + time would allow draining instead, and no vendor supplies one today. +- One task per gang. Parameter shift gradients and other multi circuit loops + need a set of tasks. See the scatter design below. +- The sidecar RBAC is created per namespace on first use. A chart or operator + would manage it more cleanly. + +## Future work + +- A MatchReserveAt primitive in the Fluxion bindings, fed by the live queue + position, to drain nodes rather than preempt. +- A scatter design, where N independent tasks each pair with one worker and the + sidecar ungates each worker as its own task comes up. +- Vendor task start events in place of polling. +- Topology aware preemption in PostFilter, so eviction targets exactly the pods + blocking a worker. diff --git a/examples/qrmi-pod.yaml b/examples/qrmi-pod.yaml new file mode 100644 index 0000000..298f0ce --- /dev/null +++ b/examples/qrmi-pod.yaml @@ -0,0 +1,35 @@ +# A QRMI workload scheduled by fluence. The pod requests a qpu and names no +# backend. fluence picks one, the webhook injects FLUXION_BACKEND and +# FLUXION_QRMI_TYPE, and examples/qrmi-workload/sampler.py acquires a QRMI +# session on that backend with the credentials from the Secret (in the user's space). +# This then runs a Bell circuit, and releases the session. + +# Usage: +# kubectl create secret generic ibm-quantum --from-literal=token="$IBM_CLOUD_TOKEN" --from-literal=crn="$IBM_CLOUD_CRN" +# kubectl apply -f examples/qrmi-pod.yaml +# kubectl logs -f qrmi-sampler +apiVersion: v1 +kind: Pod +metadata: + name: qrmi-sampler + annotations: + # the workload speaks qiskit-runtime-service, so ask for an IBM backend + fluence.flux-framework.org/require-qrmi_type: qiskit-runtime-service +spec: + schedulerName: fluence + restartPolicy: Never + containers: + - name: sampler + image: ghcr.io/converged-computing/fluence-qrmi-workload:latest + args: ["--shots", "100"] + env: + - name: IBM_CLOUD_TOKEN + valueFrom: {secretKeyRef: {name: ibm-quantum, key: token, optional: true}} + - name: IBM_CLOUD_CRN + valueFrom: {secretKeyRef: {name: ibm-quantum, key: crn, optional: true}} + resources: + requests: + cpu: "1" + fluxion.flux-framework.org/qpu: "1" + limits: + fluxion.flux-framework.org/qpu: "1" diff --git a/examples/qrmi-workload/Dockerfile b/examples/qrmi-workload/Dockerfile new file mode 100644 index 0000000..859baab --- /dev/null +++ b/examples/qrmi-workload/Dockerfile @@ -0,0 +1,13 @@ +# The QRMI workload for pods that fluence places. Built for amd64 and arm64 +# as ghcr.io/converged-computing/fluence-qrmi-workload. +FROM python:3.12-slim + +LABEL org.opencontainers.image.source="https://github.com/converged-computing/fluence" +LABEL org.opencontainers.image.description="QRMI workload for fluence" + +RUN pip install --no-cache-dir "qrmi[ibm]" qiskit qiskit-ibm-runtime \ + && python3 -c "import qrmi, qiskit, qiskit_ibm_runtime; from qrmi.primitives.ibm import SamplerV2" + +COPY sampler.py /app/sampler.py +WORKDIR /app +ENTRYPOINT ["python3", "/app/sampler.py"] diff --git a/examples/qrmi-workload/sampler.py b/examples/qrmi-workload/sampler.py new file mode 100644 index 0000000..6f290b7 --- /dev/null +++ b/examples/qrmi-workload/sampler.py @@ -0,0 +1,287 @@ +#!/usr/bin/env python3 +"""A QRMI workload for a pod that fluence placed. + +fluence picked a backend for this pod and the webhook put its name in the +environment as FLUXION_BACKEND, with the backend attributes as FLUXION_. +This script takes it from there with QRMI. It acquires a session on that +backend with the credentials mounted on the pod, runs a Bell circuit through +the QRMI sampler, prints the counts, and releases the session. fluence holds +the graph allocation and this pod holds the session. A worker in a shared gang +is handed the scout session instead and submits into it without acquiring. + +For an IBM backend the credentials are the four _QRMI_IBM_QRS_* +variables, or IBM_CLOUD_TOKEN and IBM_CLOUD_CRN from which they are derived. +Without credentials this is a rehearsal. It prints what it would need and +exits cleanly, so the same pod proves the placement and the handoff on a +cluster with no account. +""" + +import argparse +import os +import sys +import time + +from qiskit import QuantumCircuit +from qiskit.transpiler.preset_passmanagers import generate_preset_pass_manager +from qrmi import QuantumResource, ResourceType +from qrmi.primitives.ibm import SamplerV2, get_target + +# QRMI type strings to ResourceType names +RESOURCE_TYPES = { + "qiskit-runtime-service": "IBMQiskitRuntimeService", + "ibm-quantum-system": "IBMQuantumSystem", + "pasqal-cloud": "PasqalCloud", + "pasqal-local": "PasqalLocal", + "iqm-server": "IQMServer", + "alice-bob-felis": "AliceBobFelis", +} + +# what QRMI reads for each type, looked up as +REQUIRED_ENV = { + "qiskit-runtime-service": ( + "_QRMI_IBM_QRS_ENDPOINT", + "_QRMI_IBM_QRS_IAM_ENDPOINT", + "_QRMI_IBM_QRS_IAM_APIKEY", + "_QRMI_IBM_QRS_SERVICE_CRN", + ), + "ibm-quantum-system": ( + "_QRMI_IBM_QS_ENDPOINT", + "_QRMI_IBM_QS_IAM_ENDPOINT", + "_QRMI_IBM_QS_IAM_APIKEY", + "_QRMI_IBM_QS_SERVICE_CRN", + ), + "iqm-server": ("_QRMI_IQM_ISA_ENDPOINT", "_QRMI_IQM_ISA_TOKEN"), +} + +# the IBM endpoints are fixed, so the credentials reduce to a key and a CRN +IBM_QRS_DERIVED = { + "_QRMI_IBM_QRS_ENDPOINT": "https://quantum.cloud.ibm.com/api/v1", + "_QRMI_IBM_QRS_IAM_ENDPOINT": "https://iam.cloud.ibm.com", +} + + +def section(title): + print() + print("== {} ==".format(title), flush=True) + + +def print_environment(): + """What fluence injected, and which credentials are set, by name only.""" + section("environment from fluence") + for name in sorted(os.environ): + if name.startswith("FLUXION_") or name.startswith("FLUENCE_"): + print("{:<40} {}".format(name, os.environ[name])) + section("QRMI environment") + shown = False + for name in sorted(os.environ): + if "QRMI" in name or name in ("IBM_CLOUD_TOKEN", "IBM_CLOUD_CRN"): + print("{:<40} (set, value not shown)".format(name)) + shown = True + if not shown: + print("(none)") + + +def backend_from_fluence(): + """The backend fluence chose and its QRMI type, from the injected env.""" + backend = os.environ.get("FLUXION_BACKEND") + if not backend: + sys.exit( + "no FLUXION_BACKEND: this pod was not placed by fluence, or the " + "webhook did not inject it (does the pod request a qpu?)" + ) + rtype = os.environ.get("FLUXION_QRMI_TYPE", "qiskit-runtime-service") + if rtype not in RESOURCE_TYPES: + sys.exit("unknown QRMI type {!r} for backend {}".format(rtype, backend)) + section("backend") + print("backend: ", backend) + print("type: ", rtype) + return backend, rtype + + +def credentials(backend, rtype): + """Derive the QRMI variables from the IBM Secret and return what is still missing.""" + if rtype == "qiskit-runtime-service": + token = os.environ.get("IBM_CLOUD_TOKEN") + crn = os.environ.get("IBM_CLOUD_CRN") + derived = dict(IBM_QRS_DERIVED) + if token: + derived["_QRMI_IBM_QRS_IAM_APIKEY"] = token + if crn: + derived["_QRMI_IBM_QRS_SERVICE_CRN"] = crn + for suffix, value in derived.items(): + os.environ.setdefault(backend + suffix, value) + missing = [ + backend + s + for s in REQUIRED_ENV.get(rtype, ()) + if not os.environ.get(backend + s) + ] + if missing: + print("missing: ", ", ".join(missing)) + else: + print("credentials present for", backend) + return missing + + +def acquire(backend, rtype, ready_timeout): + """Acquire a QRMI session on the backend and wait until it accepts work.""" + qr = QuantumResource(backend, getattr(ResourceType, RESOURCE_TYPES[rtype])) + lock = qr.acquire() + print("session: ", lock) + # so anything else in this pod that constructs the resource joins this session + os.environ[backend + "_QRMI_JOB_ACQUISITION_TOKEN"] = str(lock) + deadline = time.time() + ready_timeout + while ready_timeout > 0: + try: + if qr.is_accessible(): + print("backend: ", backend, "is accepting work") + break + why = "not accepting work" + except Exception as e: # keep polling until the deadline + why = str(e) + if time.time() >= deadline: + qr.release(lock) + sys.exit( + "{} did not become usable within {}s: {}".format( + backend, ready_timeout, why + ) + ) + time.sleep(5) + return qr, lock + + +def build_circuit(qubits): + qc = QuantumCircuit(qubits) + qc.h(0) + for q in range(1, qubits): + qc.cx(q - 1, q) + qc.measure_all() + return qc + + +def transpile_for_device(qc, qr): + target = get_target(qr) + isa = generate_preset_pass_manager(optimization_level=1, target=target).run(qc) + print("circuit: ", dict(isa.count_ops())) + return isa + + +def run_sampler(qr, isa, shots): + sampler = SamplerV2(qr, options={"default_shots": shots}) + job = sampler.run([isa]) + print("task: ", job.job_id(), "submitted into the session", flush=True) + return job + + +def wait_for_task(job, interval=2.0): + started = time.time() + last = None + while not job.in_final_state(): + status = job.status() + if status != last: + print( + "status: ", + status, + "at {:.0f}s".format(time.time() - started), + flush=True, + ) + last = status + time.sleep(interval) + print("status: ", job.status(), "after {:.0f}s".format(time.time() - started)) + if job.errored(): + print("logs:") + print(job.logs()) + sys.exit("the task failed") + + +def report_counts(job, qubits, shots): + counts = job.result()[0].data.meas.get_counts() + print("counts: ", dict(sorted(counts.items(), key=lambda kv: -kv[1]))) + ideal = {"0" * qubits, "1" * qubits} + good = sum(v for k, v in counts.items() if k in ideal) + print("fidelity:", "{}/{} shots in {}".format(good, shots, sorted(ideal))) + + +def release(qr, lock): + qr.release(lock) + print("released:", lock) + + +def linger(seconds, step=5): + if seconds <= 0: + return + section("lingering for {}s so the pod can be watched".format(seconds)) + left = seconds + while left > 0: + print("working, {}s left".format(left), flush=True) + time.sleep(min(step, left)) + left -= step + + +def main(): + ap = argparse.ArgumentParser(description=__doc__.split("\n")[0]) + ap.add_argument("--shots", type=int, default=100) + ap.add_argument( + "--qubits", type=int, default=2, help="width of the GHZ/Bell circuit, 2" + ) + ap.add_argument( + "--linger", + type=int, + default=int(os.environ.get("LINGER", "0")), + help="seconds to stay alive after the work, 0", + ) + ap.add_argument( + "--ready-timeout", + type=float, + default=float(os.environ.get("READY_TIMEOUT", "120")), + help="seconds to wait for the backend to accept work after acquiring, 120", + ) + args = ap.parse_args() + + print_environment() + backend, rtype = backend_from_fluence() + + section("credentials, names only") + if credentials(backend, rtype): + print() + print("rehearsal: no credentials, so nothing is acquired or submitted. The") + print("placement and the handoff from fluence worked. A Secret with the") + print("variables above would make this a real run.") + linger(args.linger) + return + + role = os.environ.get("FLUENCE_COORDINATION_ROLE") + session = os.environ.get("FLUENCE_QUANTUM_SESSION_ID") + if role == "worker" and session: + # a worker joins the scout session rather than acquiring its own + section("circuit in the scout session, through QRMI") + print("session: ", session, "from the scout") + os.environ[backend + "_QRMI_JOB_ACQUISITION_TOKEN"] = session + qr = QuantumResource(backend, getattr(ResourceType, RESOURCE_TYPES[rtype])) + isa = transpile_for_device(build_circuit(args.qubits), qr) + job = run_sampler(qr, isa, args.shots) + wait_for_task(job) + report_counts(job, args.qubits, args.shots) + linger(args.linger) + return + if role == "worker": + print( + "role: worker with no session, task", + os.environ.get("FLUENCE_QUANTUM_JOB_ID", "?"), + ) + linger(args.linger) + return + + section("session and circuit, through QRMI") + qr, lock = acquire(backend, rtype, args.ready_timeout) + try: + isa = transpile_for_device(build_circuit(args.qubits), qr) + job = run_sampler(qr, isa, args.shots) + wait_for_task(job) + report_counts(job, args.qubits, args.shots) + finally: + release(qr, lock) + linger(args.linger) + + +if __name__ == "__main__": + main() diff --git a/examples/quantum-pod.yaml b/examples/quantum-pod.yaml index b5dfbc9..cacbe6c 100644 --- a/examples/quantum-pod.yaml +++ b/examples/quantum-pod.yaml @@ -4,10 +4,10 @@ # the request against the resource graph and picks a backend, the webhook injects # FLUXION_BACKEND (the allocated backend) automatically, and note we can add other # envars here in the future. I chose a webhook because I think this is going to -# be a requirement, and the pod is immutable after creation. +# be a requirement, and the pod is immutable after creation. # Then the container submits via qrmi-go (the separate qrmi-sampler image). # the credentials live on the level of the application container NOT in -# a shared space. +# a shared space. apiVersion: v1 kind: Pod metadata: diff --git a/examples/test/e2e/gang/multi-gang-requeue.yaml b/examples/test/e2e/gang/multi-gang-requeue.yaml index a8e8636..a5d34f6 100644 --- a/examples/test/e2e/gang/multi-gang-requeue.yaml +++ b/examples/test/e2e/gang/multi-gang-requeue.yaml @@ -45,4 +45,4 @@ spec: - name: w image: busybox command: ["sh","-c","sleep 10"] - resources: {requests: {cpu: "1"}} \ No newline at end of file + resources: {requests: {cpu: "1"}} diff --git a/examples/test/e2e/quantum/qrmi-gang-pods.yaml b/examples/test/e2e/quantum/qrmi-gang-pods.yaml new file mode 100644 index 0000000..8ecc91f --- /dev/null +++ b/examples/test/e2e/quantum/qrmi-gang-pods.yaml @@ -0,0 +1,48 @@ +# Shared-coordination quantum gang for IBM Quantum via QRMI +apiVersion: v1 +kind: Pod +metadata: + name: qrmi-gang-0 + labels: + app: qrmi-gang + fluence.flux-framework.org/group: qrmi-gang + annotations: + fluence.flux-framework.org/group-size: "2" + fluence.flux-framework.org/coordination: shared + fluence.flux-framework.org/require-qrmi_type: qiskit-runtime-service + batch.kubernetes.io/job-completion-index: "0" # -> scout +spec: + schedulerName: fluence + restartPolicy: Never + containers: + - name: app + image: busybox + command: ["sh", "-c", "echo qrmi gang member (role=$FLUENCE_COORDINATION_ROLE); sleep 600"] + resources: + requests: {fluxion.flux-framework.org/qpu: "1"} + limits: {fluxion.flux-framework.org/qpu: "1"} +--- +apiVersion: v1 +kind: Pod +metadata: + name: qrmi-gang-1 + labels: + app: qrmi-gang + fluence.flux-framework.org/group: qrmi-gang + annotations: + fluence.flux-framework.org/group-size: "2" + fluence.flux-framework.org/coordination: shared + fluence.flux-framework.org/require-qrmi_type: qiskit-runtime-service + batch.kubernetes.io/job-completion-index: "1" # -> worker +spec: + schedulerName: fluence + restartPolicy: Never + containers: + - name: app + image: busybox + # A worker fetches the shared result by id (FLUENCE_QUANTUM_JOB_ID) and, + # for a paid session, may reuse FLUENCE_QUANTUM_SESSION_ID for priority. + command: ["sh", "-c", "echo qrmi gang member (role=$FLUENCE_COORDINATION_ROLE job=$FLUENCE_QUANTUM_JOB_ID); sleep 600"] + resources: + requests: {fluxion.flux-framework.org/qpu: "1"} + limits: {fluxion.flux-framework.org/qpu: "1"} diff --git a/examples/test/e2e/quantum/qrmi-pod-mock.yaml b/examples/test/e2e/quantum/qrmi-pod-mock.yaml new file mode 100644 index 0000000..d3bf07e --- /dev/null +++ b/examples/test/e2e/quantum/qrmi-pod-mock.yaml @@ -0,0 +1,33 @@ +# CI mock for the QRMI/IBM standalone path (cf. examples/quantum-pod.yaml, which +# runs the real qrmi-sampler). Requests the same fluxion.flux-framework.org/qpu +# resource so fluence matches an IBM backend and the webhook injects +# FLUXION_BACKEND + FLUXION_QRMI_TYPE, plus the writable handoff volume and +# FLUENCE_HANDOFF_DIR. +apiVersion: v1 +kind: Pod +metadata: + name: qrmi-sampler-mock + annotations: + fluence.flux-framework.org/require-qrmi_type: qiskit-runtime-service +spec: + schedulerName: fluence + restartPolicy: Never + containers: + - name: sampler + image: busybox + command: + - sh + - -c + - | + echo "BACKEND=$FLUXION_BACKEND QRMI_TYPE=$FLUXION_QRMI_TYPE" + echo "HANDOFF_DIR=$FLUENCE_HANDOFF_DIR" + # Prove the handoff volume is mounted writable (the real interceptor + # writes the job id here for the sidecar). + echo '{"job_id":"mock-job-123","tier":"free"}' > "$FLUENCE_HANDOFF_DIR/task.json" + echo "HANDOFF_WROTE=$(cat $FLUENCE_HANDOFF_DIR/task.json)" + sleep 3600 + resources: + requests: + fluxion.flux-framework.org/qpu: "1" + limits: + fluxion.flux-framework.org/qpu: "1" diff --git a/examples/test/e2e/quantum/quantum-gang-pods.yaml b/examples/test/e2e/quantum/quantum-gang-pods.yaml index aacce44..eaa0978 100644 --- a/examples/test/e2e/quantum/quantum-gang-pods.yaml +++ b/examples/test/e2e/quantum/quantum-gang-pods.yaml @@ -1,17 +1,17 @@ -# Shared-coordination quantum gang for the e2e (producer/consumer, no submitter). +# Shared-coordination quantum gang for the e2e (scout/worker, no submitter). # # Two identical pods, both requesting the quantum resource, in group "qgang" with # coordination=shared. The user authors NO roles and NO submitter pod. The webhook # splits the gang by completion index: -# qgang-0 (index 0) -> PRODUCER: its own group-of-one "qgang-producer" +# qgang-0 (index 0) -> SCOUT: its own group-of-one "qgang-scout" # (minCount 1), real submit, sidecar, NOT gated. It is a # real member, so the app runs N times, never N+1. -# qgang-1 (index 1+) -> CONSUMER: the "qgang" gang (minCount N-1=1), gated on +# qgang-1 (index 1+) -> WORKER: the "qgang" gang (minCount N-1=1), gated on # quantum.braket/ready + preempting priority, interceptor -# told role=consumer; it fetches the producer's result by id. +# told role=worker; it fetches the scout's result by id. # # These are raw pods (not a Job) so the e2e can reference stable names; the -# completion-index annotation is set manually to make producer election +# completion-index annotation is set manually to make scout election # deterministic (a real workload uses an indexed Job, which the controller stamps # with batch.kubernetes.io/job-completion-index automatically). group-size makes N # deterministic for raw pods, which have no owning Job to derive it from. busybox @@ -27,7 +27,7 @@ metadata: annotations: fluence.flux-framework.org/group-size: "2" fluence.flux-framework.org/coordination: shared - batch.kubernetes.io/job-completion-index: "0" # -> producer + batch.kubernetes.io/job-completion-index: "0" # -> scout spec: schedulerName: fluence restartPolicy: Never @@ -49,7 +49,7 @@ metadata: annotations: fluence.flux-framework.org/group-size: "2" fluence.flux-framework.org/coordination: shared - batch.kubernetes.io/job-completion-index: "1" # -> consumer + batch.kubernetes.io/job-completion-index: "1" # -> worker spec: schedulerName: fluence restartPolicy: Never diff --git a/examples/test/jobspec-cpu.yaml b/examples/test/jobspec-cpu.yaml index 5677c33..164fa54 100644 --- a/examples/test/jobspec-cpu.yaml +++ b/examples/test/jobspec-cpu.yaml @@ -16,4 +16,4 @@ tasks: - command: [""] slot: default count: - per_slot: 1 \ No newline at end of file + per_slot: 1 diff --git a/examples/test/jobspec-qpu.yaml b/examples/test/jobspec-qpu.yaml index 5802579..d10fd15 100644 --- a/examples/test/jobspec-qpu.yaml +++ b/examples/test/jobspec-qpu.yaml @@ -15,4 +15,4 @@ tasks: - command: [""] slot: default count: - per_slot: 1 \ No newline at end of file + per_slot: 1 diff --git a/pkg/fluence/fluence.go b/pkg/fluence/fluence.go index 6c3dc13..235725c 100644 --- a/pkg/fluence/fluence.go +++ b/pkg/fluence/fluence.go @@ -828,18 +828,18 @@ func (f *Fluence) reconcileGroup(ctx context.Context, namespace, group string) { log.Printf("fluence: reconciled completed gang %s/%s — deleted Fluence-created PodGroup, allocation freed", namespace, group) - // Producer-group cleanup: in shared coordination the gang is split into the - // consumer group (this group) and the producer's group-of-one - // -producer (a Fluence-created PodGroup, minCount 1). The producer POD + // Scout-group cleanup: in shared coordination the gang is split into the + // worker group (this group) and the scout's group-of-one + // -scout (a Fluence-created PodGroup, minCount 1). The scout POD // is a real member of the user's workload (indexed-Job index 0), so we must // NOT delete it — only its Fluence-created PodGroup, as a backstop to free its - // allocation (its own reconcile pass also reaps it once the producer pod is - // terminal). Skip when this group is itself a producer group, to avoid - // recursing on -producer-producer. - if !strings.HasSuffix(group, producerGroupSuffix) { - pg := group + producerGroupSuffix + // allocation (its own reconcile pass also reaps it once the scout pod is + // terminal). Skip when this group is itself a scout group, to avoid + // recursing on -scout-scout. + if !strings.HasSuffix(group, scoutGroupSuffix) { + pg := group + scoutGroupSuffix _ = f.handle.ClientSet().SchedulingV1alpha2().PodGroups(namespace).Delete(ctx, pg, metav1.DeleteOptions{}) - log.Printf("fluence: reaped producer PodGroup %s/%s for gang %s", namespace, pg, group) + log.Printf("fluence: reaped scout PodGroup %s/%s for gang %s", namespace, pg, group) } } @@ -852,11 +852,11 @@ const reconcileGraceForEmpty = 2 * time.Minute // package (the scheduler must not depend on the webhook). Kept in sync with it. const webhookGroupLabel = "fluence.flux-framework.org/group" -// producerGroupSuffix mirrors handlers.ProducerGroupSuffix: in shared -// coordination the producer (indexed-Job index 0) is its own group-of-one named -// -producer. Duplicated here to avoid importing the webhook handlers package +// scoutGroupSuffix mirrors handlers.ScoutGroupSuffix: in shared +// coordination the scout (indexed-Job index 0) is its own group-of-one named +// -scout. Duplicated here to avoid importing the webhook handlers package // into the scheduler plugin; keep the two in sync. -const producerGroupSuffix = "-producer" +const scoutGroupSuffix = "-scout" // onPodGroupDeleted frees the gang's allocation when its PodGroup is deleted. func (f *Fluence) onPodGroupDeleted(obj interface{}) { diff --git a/pkg/placement/placement.go b/pkg/placement/placement.go index c7f76de..0a13571 100644 --- a/pkg/placement/placement.go +++ b/pkg/placement/placement.go @@ -175,6 +175,10 @@ func podResources(p *corev1.Pod) map[string]int { return counts } +// PodCores is the whole core request of a pod as the graph counts it, cpu +// rounded up to cores and never less than one. Admission control uses it. +func PodCores(p *corev1.Pod) int { return podResources(p)["core"] } + // splitResources separates a pod's resource counts into the compute resources // (core/memory/gpu, satisfied by a virtual=false node) and the virtual device // resources (everything requested via the fluxion prefix, satisfied by a diff --git a/pkg/webhook/admission.go b/pkg/webhook/admission.go new file mode 100644 index 0000000..b1c910e --- /dev/null +++ b/pkg/webhook/admission.go @@ -0,0 +1,205 @@ +// Admission control, the rule the flux-quantum jobtap plugin applies at submit. +// +// A quantum group is admitted only if the cores all of its members will need, +// plus what is promised to admitted groups that have not started, fit in what +// is reachable. Reachable is the free cores plus the cores held by running +// pods that are unprotected and below the priority workers are ungated with. +// The budget is the allocatable cores of schedulable nodes less ReserveCores. +// A refusal is an admission denial, so it lands at kubectl apply or on the +// owning Job, not as a pod left pending with a backend it cannot use. +// +// Every quantum pod is marked protected, which keeps it out of preemptible, +// and a pod that arrives already marked is refused. +package webhook + +import ( + "context" + "fmt" + "log" + "strconv" + + "github.com/converged-computing/fluence/pkg/placement" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" +) + +const ( + // ProtectedAnnotation marks a pod as one of ours and not preemptible. Set by + // the webhook on every quantum pod; a submission carrying it is refused. + ProtectedAnnotation = "fluence.flux-framework.org/protected" + ProtectedValue = "quantum" + + // AdmittedCoresAnnotation records, on the group's PodGroup (or on a + // standalone pod), the core total the group was admitted with. + AdmittedCoresAnnotation = "fluence.flux-framework.org/admitted-cores" + + // AdmittedGroupAnnotation names, on every member pod, the record it belongs + // to as namespace/name, so running members are subtracted from the promise. + AdmittedGroupAnnotation = "fluence.flux-framework.org/admitted-group" + + // PreemptiblePriorityCeiling is the fluence-quantum-classical priority that + // workers are ungated with. Running work below it is what they could + // preempt, so that is what counts as preemptible. + PreemptiblePriorityCeiling int32 = 1000000 +) + +// Capacity is one reading of the budget, in whole cores. +type Capacity struct { + Budget int // allocatable cores of schedulable nodes, less the reserve + Running int // cores of every bound, non-terminal pod on those nodes + Free int // Budget - Running, never negative + Preemptible int // running cores that are unprotected and below the ceiling + Promised int // admitted to quantum groups whose members are not all running +} + +// Reachable is what a new group could have: free cores plus what preemption +// could free. +func (c Capacity) Reachable() int { return c.Free + c.Preemptible } + +// Admit refuses the group named key when its cores do not fit. With admission +// control off, or no client, every group is admitted. An API error admits too +// with a log line, since a webhook that cannot read the cluster must not block +// pods. +func (m *Mutator) Admit(ctx context.Context, pod *corev1.Pod, key string, cores int) error { + if !m.AdmissionControl || m.Clientset == nil || cores <= 0 { + return nil + } + c, err := m.Capacity(ctx) + if err != nil { + log.Printf("[fluence-webhook] admission: cannot read capacity, admitting %s: %v", key, err) + return nil + } + if cores+c.Promised > c.Reachable() { + return fmt.Errorf("quantum: no room for another pair. %s needs %d cores, %d are already "+ + "promised to pairs that have not started, and only %d are reachable (%d free, %d that "+ + "could be preempted). Wait for one to finish or ask for fewer cores", + key, cores, c.Promised, c.Reachable(), c.Free, c.Preemptible) + } + log.Printf("[fluence-webhook] admission: %s admitted with %d cores (free=%d preemptible=%d promised=%d budget=%d)", + key, cores, c.Free, c.Preemptible, c.Promised, c.Budget) + return nil +} + +// RecordAdmission writes the admitted total on the PodGroup once. Later +// members of the group skip the check because of it, and later groups count +// it as promised. +func (m *Mutator) RecordAdmission(ctx context.Context, namespace, podGroup string, cores int) { + if m.Clientset == nil || cores <= 0 { + return + } + pg, err := m.Clientset.SchedulingV1alpha2().PodGroups(namespace).Get(ctx, podGroup, metav1.GetOptions{}) + if err != nil { + log.Printf("[fluence-webhook] admission: no PodGroup %s/%s to record %d cores on: %v", namespace, podGroup, cores, err) + return + } + if pg.Annotations[AdmittedCoresAnnotation] != "" { + return + } + patch := fmt.Sprintf(`{"metadata":{"annotations":{%q:%q}}}`, AdmittedCoresAnnotation, strconv.Itoa(cores)) + if _, err := m.Clientset.SchedulingV1alpha2().PodGroups(namespace).Patch( + ctx, podGroup, types.MergePatchType, []byte(patch), metav1.PatchOptions{}); err != nil { + log.Printf("[fluence-webhook] admission: recording %d cores on PodGroup %s/%s failed: %v", cores, namespace, podGroup, err) + } +} + +// Capacity reads the cluster and computes the budget. +func (m *Mutator) Capacity(ctx context.Context) (Capacity, error) { + nodes, err := m.Clientset.CoreV1().Nodes().List(ctx, metav1.ListOptions{}) + if err != nil { + return Capacity{}, fmt.Errorf("list nodes: %w", err) + } + pods, err := m.Clientset.CoreV1().Pods(metav1.NamespaceAll).List(ctx, metav1.ListOptions{}) + if err != nil { + return Capacity{}, fmt.Errorf("list pods: %w", err) + } + groups, err := m.Clientset.SchedulingV1alpha2().PodGroups(metav1.NamespaceAll).List(ctx, metav1.ListOptions{}) + if err != nil { + return Capacity{}, fmt.Errorf("list podgroups: %w", err) + } + + var c Capacity + schedulable := map[string]bool{} + for _, n := range nodes.Items { + if !nodeSchedulable(n) { + continue + } + schedulable[n.Name] = true + c.Budget += nodeCores(n) + } + c.Budget -= m.ReserveCores + if c.Budget < 0 { + c.Budget = 0 + } + + // cores running per admission record, keyed namespace/name + runningOf := map[string]int{} + // the records themselves: PodGroups, and standalone pods that carry their own + records := map[string]int{} + for _, pg := range groups.Items { + if n, err := strconv.Atoi(pg.Annotations[AdmittedCoresAnnotation]); err == nil && n > 0 { + records[pg.Namespace+"/"+pg.Name] = n + } + } + for i := range pods.Items { + p := &pods.Items[i] + if n, err := strconv.Atoi(p.Annotations[AdmittedCoresAnnotation]); err == nil && n > 0 { + records[p.Namespace+"/"+p.Name] = n + } + if !podBound(p) || !schedulable[p.Spec.NodeName] { + continue + } + cores := placement.PodCores(p) + c.Running += cores + if p.Annotations[ProtectedAnnotation] == "" && podPriority(p) < PreemptiblePriorityCeiling { + c.Preemptible += cores + } + if g := p.Annotations[AdmittedGroupAnnotation]; g != "" { + runningOf[g] += cores + } + } + for key, total := range records { + if left := total - runningOf[key]; left > 0 { + c.Promised += left + } + } + c.Free = c.Budget - c.Running + if c.Free < 0 { + c.Free = 0 + } + return c, nil +} + +// nodeSchedulable mirrors the scheduler. Cordoned nodes and nodes with a +// NoSchedule or NoExecute taint are not part of the budget. +func nodeSchedulable(n corev1.Node) bool { + if n.Spec.Unschedulable { + return false + } + for _, t := range n.Spec.Taints { + if t.Effect == corev1.TaintEffectNoSchedule || t.Effect == corev1.TaintEffectNoExecute { + return false + } + } + return true +} + +// nodeCores is the node's allocatable cpu in whole cores, rounded down so a +// fractional core is never promised. +func nodeCores(n corev1.Node) int { + q := n.Status.Allocatable[corev1.ResourceCPU] + return int(q.MilliValue() / 1000) +} + +// podBound reports a pod that holds its node, bound and not finished. +func podBound(p *corev1.Pod) bool { + return p.Spec.NodeName != "" && p.Status.Phase != corev1.PodSucceeded && p.Status.Phase != corev1.PodFailed +} + +func podPriority(p *corev1.Pod) int32 { + if p.Spec.Priority != nil { + return *p.Spec.Priority + } + return 0 +} diff --git a/pkg/webhook/handler.go b/pkg/webhook/handler.go index 61b97b1..1cbcdc2 100644 --- a/pkg/webhook/handler.go +++ b/pkg/webhook/handler.go @@ -30,6 +30,21 @@ type MutatorAPI interface { // of the group label. creatorPod is recorded only as the PodGroup's creator // reference; the core ascribes no role semantics to it. EnsurePodGroup(ctx context.Context, namespace, group, creatorPod string, minCount int32) + + // Admit applies admission control, see admission.go. key names the group + // in the refusal. + Admit(ctx context.Context, pod *corev1.Pod, key string, cores int) error + + // RecordAdmission stamps the admitted core total on the PodGroup once. + RecordAdmission(ctx context.Context, namespace, podGroup string, cores int) +} + +// Admitter is an optional handler capability. A handler that implements it +// may refuse a pod, and the webhook answers the review with allowed false and +// the error as the message. Admit runs before any Mutate, and a refusal +// discards every patch. +type Admitter interface { + Admit(ctx context.Context, m MutatorAPI, pod *corev1.Pod) error } // Handler inspects a pod and, when it applies, contributes JSON patch ops. A pod diff --git a/pkg/webhook/handlers/admission_test.go b/pkg/webhook/handlers/admission_test.go new file mode 100644 index 0000000..feefa86 --- /dev/null +++ b/pkg/webhook/handlers/admission_test.go @@ -0,0 +1,209 @@ +// Admission control tests. +package handlers + +import ( + "context" + "strings" + "testing" + + "github.com/converged-computing/fluence/pkg/webhook" + "github.com/converged-computing/fluence/pkg/webhook/spec" + + corev1 "k8s.io/api/core/v1" + schedulingv1alpha2 "k8s.io/api/scheduling/v1alpha2" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/client-go/kubernetes/fake" +) + +func cpuQty(cores int64) resource.Quantity { return *resource.NewQuantity(cores, resource.DecimalSI) } + +func runningPod(name, node string, cores int64, priority int32, protected bool) *corev1.Pod { + p := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: "default", Annotations: map[string]string{}}, + Spec: corev1.PodSpec{ + NodeName: node, + Priority: &priority, + Containers: []corev1.Container{{Name: "c", Resources: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{corev1.ResourceCPU: cpuQty(cores)}}}}, + }, + Status: corev1.PodStatus{Phase: corev1.PodRunning}, + } + if protected { + p.Annotations[webhook.ProtectedAnnotation] = webhook.ProtectedValue + } + return p +} + +func budgetCluster() []runtime.Object { + node := &corev1.Node{ + ObjectMeta: metav1.ObjectMeta{Name: "n1"}, + Status: corev1.NodeStatus{Allocatable: corev1.ResourceList{corev1.ResourceCPU: cpuQty(8)}}, + } + cordoned := &corev1.Node{ // not part of the budget + ObjectMeta: metav1.ObjectMeta{Name: "n2"}, + Spec: corev1.NodeSpec{Unschedulable: true}, + Status: corev1.NodeStatus{Allocatable: corev1.ResourceList{corev1.ResourceCPU: cpuQty(100)}}, + } + admitted := &schedulingv1alpha2.PodGroup{ObjectMeta: metav1.ObjectMeta{ + Name: "earlier", Namespace: "default", + Annotations: map[string]string{webhook.AdmittedCoresAnnotation: "3"}}} + return []runtime.Object{ + node, cordoned, admitted, + runningPod("batch", "n1", 2, 0, false), // preemptible + runningPod("pair", "n1", 2, webhook.PreemptiblePriorityCeiling, true), // protected, high priority + runningPod("elsewhere", "n2", 50, 0, false), // on the cordoned node, ignored + } +} + +func budgetMutator(objs ...runtime.Object) *webhook.Mutator { + return &webhook.Mutator{Clientset: fake.NewSimpleClientset(objs...), AdmissionControl: true} +} + +func standaloneQPU(cores int64) *corev1.Pod { + p := qpuPod("fluence") + p.Name, p.Namespace = "sampler", "default" + p.Spec.Containers[0].Resources.Requests[corev1.ResourceCPU] = cpuQty(cores) + return p +} + +func TestCapacityArithmetic(t *testing.T) { + m := budgetMutator(budgetCluster()...) + c, err := m.Capacity(context.Background()) + if err != nil { + t.Fatal(err) + } + want := webhook.Capacity{Budget: 8, Running: 4, Free: 4, Preemptible: 2, Promised: 3} + if c != want { + t.Fatalf("capacity = %+v, want %+v", c, want) + } + if c.Reachable() != 6 { + t.Fatalf("reachable = %d, want 6", c.Reachable()) + } +} + +func TestReserveComesOffTheBudget(t *testing.T) { + m := budgetMutator(budgetCluster()...) + m.ReserveCores = 3 + c, err := m.Capacity(context.Background()) + if err != nil { + t.Fatal(err) + } + if c.Budget != 5 || c.Free != 1 { + t.Fatalf("with 3 reserved: budget=%d free=%d, want 5 and 1", c.Budget, c.Free) + } +} + +func TestStandaloneAdmittedWhenItFits(t *testing.T) { + m := budgetMutator(budgetCluster()...) + ops, err := m.MutateOrDeny(context.Background(), standaloneQPU(3)) // 3 + 3 promised <= 6 reachable + if err != nil { + t.Fatalf("expected admission, got refusal: %v", err) + } + ann := annotationsAdded(ops) + if ann[webhook.ProtectedAnnotation] != webhook.ProtectedValue { + t.Errorf("admitted pod must be marked protected, got %v", ann) + } + if ann[webhook.AdmittedCoresAnnotation] != "3" { + t.Errorf("standalone pod must carry its own record, got %v", ann) + } + if ann[webhook.AdmittedGroupAnnotation] != "default/sampler" { + t.Errorf("standalone pod must point at its own record, got %v", ann) + } +} + +func TestStandaloneRefusedWhenItDoesNotFit(t *testing.T) { + m := budgetMutator(budgetCluster()...) + _, err := m.MutateOrDeny(context.Background(), standaloneQPU(4)) // 4 + 3 > 6 + if err == nil { + t.Fatal("expected refusal") + } + if !strings.Contains(err.Error(), "no room for another pair") || !strings.Contains(err.Error(), "needs 4 cores") { + t.Errorf("refusal should explain the budget, got: %v", err) + } +} + +func TestPromiseIsReleasedAsMembersRun(t *testing.T) { + // the earlier group's 3 promised cores shrink by what its members run + objs := budgetCluster() + member := runningPod("earlier-0", "n1", 2, webhook.PreemptiblePriorityCeiling, true) + member.Annotations[webhook.AdmittedGroupAnnotation] = "default/earlier" + m := budgetMutator(append(objs, member)...) + c, err := m.Capacity(context.Background()) + if err != nil { + t.Fatal(err) + } + if c.Promised != 1 || c.Running != 6 { + t.Fatalf("promised=%d running=%d, want 1 and 6", c.Promised, c.Running) + } +} + +func TestSelfProtectedPodRefused(t *testing.T) { + m := budgetMutator(budgetCluster()...) + p := standaloneQPU(1) + p.Annotations = map[string]string{webhook.ProtectedAnnotation: "please"} + if _, err := m.MutateOrDeny(context.Background(), p); err == nil { + t.Fatal("a pod that marks itself protected must be refused") + } +} + +func TestAdmissionOffAdmitsEverything(t *testing.T) { + m := budgetMutator(budgetCluster()...) + m.AdmissionControl = false + if _, err := m.MutateOrDeny(context.Background(), standaloneQPU(100)); err != nil { + t.Fatalf("admission control off must not refuse: %v", err) + } +} + +func TestNonQuantumPodIsNotBudgeted(t *testing.T) { + m := budgetMutator(budgetCluster()...) + p := cpuPod("fluence") + p.Name, p.Namespace = "classical", "default" + p.Spec.Containers[0].Resources.Requests[corev1.ResourceCPU] = cpuQty(100) + if _, err := m.MutateOrDeny(context.Background(), p); err != nil { + t.Fatalf("the budget is for quantum groups only: %v", err) + } +} + +func TestRecordAdmissionWritesOnce(t *testing.T) { + pg := &schedulingv1alpha2.PodGroup{ObjectMeta: metav1.ObjectMeta{Name: "qg", Namespace: "default"}} + m := budgetMutator(pg) + ctx := context.Background() + m.RecordAdmission(ctx, "default", "qg", 6) + m.RecordAdmission(ctx, "default", "qg", 99) // ignored, already recorded + got, err := m.Clientset.SchedulingV1alpha2().PodGroups("default").Get(ctx, "qg", metav1.GetOptions{}) + if err != nil { + t.Fatal(err) + } + if got.Annotations[webhook.AdmittedCoresAnnotation] != "6" { + t.Fatalf("record = %q, want 6", got.Annotations[webhook.AdmittedCoresAnnotation]) + } +} + +// annotationsAdded collects the annotations a patch adds, whether as one map op +// or per-key ops. +func annotationsAdded(ops []spec.Op) map[string]string { + out := map[string]string{} + for _, op := range ops { + if op.Op != "add" { + continue + } + if op.Path == "/metadata/annotations" { + if m, ok := op.Value.(map[string]string); ok { + for k, v := range m { + out[k] = v + } + } + continue + } + const prefix = "/metadata/annotations/" + if strings.HasPrefix(op.Path, prefix) { + key := strings.ReplaceAll(strings.ReplaceAll(strings.TrimPrefix(op.Path, prefix), "~1", "/"), "~0", "~") + if v, ok := op.Value.(string); ok { + out[key] = v + } + } + } + return out +} diff --git a/pkg/webhook/handlers/dependency.go b/pkg/webhook/handlers/dependency.go index d25d598..33f2980 100644 --- a/pkg/webhook/handlers/dependency.go +++ b/pkg/webhook/handlers/dependency.go @@ -6,7 +6,7 @@ import ( corev1 "k8s.io/api/core/v1" ) -// Dependency is Fluence's GENERAL "this set of pods must wait for a producer to +// Dependency is Fluence's GENERAL "this set of pods must wait for a scout to // be ready" primitive. It is deliberately NOT quantum-specific: quantum is the // first resource type to use it (a gang waits for a quantum submission to reach // the device queue), but the same primitive applies to any resource type whose @@ -19,26 +19,26 @@ import ( // (in its reconcile loop): // // - Kind: what KIND of readiness this is (the resource type's name). The -// producer side knows how to satisfy this kind; the consumer side +// scout side knows how to satisfy this kind; the worker side // only knows it must wait. Quantum's kind is "quantum-submit". -// - Producer: the identity of the thing that will signal ready. For quantum it +// - Scout: the identity of the thing that will signal ready. For quantum it // is the submitter's (base) group; generally it is whatever the // kind's handler records as the satisfier. -// - Gate: the scheduling gate held on the dependent (consumer) pods until -// the producer signals ready. Removing the gate is the "ungate" -// and is performed by whatever observes the producer's readiness +// - Gate: the scheduling gate held on the dependent (worker) pods until +// the scout signals ready. Removing the gate is the "ungate" +// and is performed by whatever observes the scout's readiness // (the quantum sidecar for kind=quantum-submit; the scheduler's // reconcile loop for kinds whose readiness is in-cluster, e.g. // "another gang is Running"). // -// The webhook PRODUCES a Dependency (gates the consumers, stamps the +// The webhook PRODUCES a Dependency (gates the workers, stamps the // annotations); REMOVING the gate is owned by the observer best placed to see -// the producer's readiness. That split — declare here, observe elsewhere — is +// the scout's readiness. That split — declare here, observe elsewhere — is // what keeps the primitive general: a new resource type adds a Kind and an // observer and reuses the gating/annotation machinery unchanged. type Dependency struct { Kind string // resource-type readiness kind, e.g. "quantum-submit" - Producer string // identity of the readiness producer (e.g. the base group) + Scout string // identity of the readiness scout (e.g. the base group) Gate string // scheduling gate held on dependents until ready } @@ -47,10 +47,10 @@ type Dependency struct { const ( // DependsOnKindAnnotation names the readiness kind the dependent waits for. DependsOnKindAnnotation = "fluence.flux-framework.org/depends-on-kind" - // DependsOnProducerAnnotation names the producer expected to signal ready. - DependsOnProducerAnnotation = "fluence.flux-framework.org/depends-on-producer" + // DependsOnScoutAnnotation names the scout expected to signal ready. + DependsOnScoutAnnotation = "fluence.flux-framework.org/depends-on-scout" // DependsOnGateAnnotation records which scheduling gate encodes the wait, so - // an observer knows exactly which gate to remove when the producer is ready. + // an observer knows exactly which gate to remove when the scout is ready. DependsOnGateAnnotation = "fluence.flux-framework.org/depends-on-gate" ) @@ -62,7 +62,7 @@ const ( func (d Dependency) applyOps(pod *corev1.Pod) []spec.Op { ops := gateWithName(pod, d.Gate) ops = append(ops, annotateOp(pod, DependsOnKindAnnotation, d.Kind)...) - ops = append(ops, annotateOp(pod, DependsOnProducerAnnotation, d.Producer)...) + ops = append(ops, annotateOp(pod, DependsOnScoutAnnotation, d.Scout)...) ops = append(ops, annotateOp(pod, DependsOnGateAnnotation, d.Gate)...) return ops } @@ -77,7 +77,7 @@ func DependencyOf(pod *corev1.Pod) (Dependency, bool) { } return Dependency{ Kind: kind, - Producer: spec.Annotation(pod, DependsOnProducerAnnotation), + Scout: spec.Annotation(pod, DependsOnScoutAnnotation), Gate: spec.Annotation(pod, DependsOnGateAnnotation), }, true } diff --git a/pkg/webhook/handlers/gang.go b/pkg/webhook/handlers/gang.go index 8ba83f3..b827d17 100644 --- a/pkg/webhook/handlers/gang.go +++ b/pkg/webhook/handlers/gang.go @@ -27,8 +27,8 @@ func (h *gangHandler) Name() string { return "gang" } func (h *gangHandler) Applies(ctx context.Context, m webhook.MutatorAPI, pod *corev1.Pod) bool { // Classical gangs only. A pod that requests the quantum resource is gang- - // scheduled by the quantum handler, which owns the producer/consumer split and - // creates both the -producer and PodGroups itself; handling it + // scheduled by the quantum handler, which owns the scout/worker split and + // creates both the -scout and PodGroups itself; handling it // here too would create a second, conflicting PodGroup for the group. if spec.PodRequestsResource(pod, QuantumResource) { return false diff --git a/pkg/webhook/handlers/gang_test.go b/pkg/webhook/handlers/gang_test.go index ac027f8..aeaa0c4 100644 --- a/pkg/webhook/handlers/gang_test.go +++ b/pkg/webhook/handlers/gang_test.go @@ -1,9 +1,3 @@ -/* -Copyright 2024 Lawrence Livermore National Security, LLC - (c.f. AUTHORS, NOTICE.LLNS, COPYING) -SPDX-License-Identifier: Apache-2.0 -*/ - // Tests for gang PodGroup minCount: the whole gang (full N) must schedule // atomically. Regression guard for the bug where every PodGroup was created // with minCount=1, so a multi-pod gang was "satisfied" by a single pod and the diff --git a/pkg/webhook/handlers/handlers_test.go b/pkg/webhook/handlers/handlers_test.go index dee0746..27b5983 100644 --- a/pkg/webhook/handlers/handlers_test.go +++ b/pkg/webhook/handlers/handlers_test.go @@ -82,7 +82,7 @@ func hasGateOp(ops []spec.Op) bool { } // hasDropQuantumResourceOp reports whether ops remove the Fluxion quantum -// resource from a container's requests or limits (the consumer qpu strip). +// resource from a container's requests or limits (the worker qpu strip). func hasDropQuantumResourceOp(ops []spec.Op) bool { for _, op := range ops { if op.Op == "remove" && strings.HasSuffix(op.Path, "qpu") && diff --git a/pkg/webhook/handlers/handoff_test.go b/pkg/webhook/handlers/handoff_test.go new file mode 100644 index 0000000..02fb827 --- /dev/null +++ b/pkg/webhook/handlers/handoff_test.go @@ -0,0 +1,180 @@ +// Tests for the handoff volume a submitting pod gets, and the session env a +// worker gets. Fixtures come from handlers_test.go and quantum_test.go. +package handlers + +import ( + "context" + "strings" + "testing" + + "github.com/converged-computing/fluence/pkg/webhook" + "github.com/converged-computing/fluence/pkg/webhook/spec" + + batchv1 "k8s.io/api/batch/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes/fake" +) + +func opsHaveVolume(ops []spec.Op, name string) bool { + for _, op := range ops { + switch v := op.Value.(type) { + case corev1.Volume: + if v.Name == name { + return true + } + case []corev1.Volume: + for _, vol := range v { + if vol.Name == name { + return true + } + } + } + } + return false +} + +// opsMountHandoffWritable reports a mount of the handoff volume with ReadOnly false +func opsMountHandoffWritable(ops []spec.Op) bool { + for _, op := range ops { + switch v := op.Value.(type) { + case corev1.VolumeMount: + if v.Name == HandoffVolumeName && !v.ReadOnly { + return true + } + case []corev1.VolumeMount: + for _, m := range v { + if m.Name == HandoffVolumeName && !m.ReadOnly { + return true + } + } + } + } + return false +} + +// sidecarMountsHandoffReadOnly finds the sidecar container in the ops and checks its mount +func sidecarMountsHandoffReadOnly(ops []spec.Op) bool { + check := func(c corev1.Container) bool { + if c.Name != SidecarContainerName { + return false + } + for _, m := range c.VolumeMounts { + if m.Name == HandoffVolumeName && m.ReadOnly { + return true + } + } + return false + } + for _, op := range ops { + switch v := op.Value.(type) { + case corev1.Container: + if check(v) { + return true + } + case []corev1.Container: + for _, c := range v { + if check(c) { + return true + } + } + } + } + return false +} + +func handoffJobMutator(ns, job string, n int32) *webhook.Mutator { + cs := fake.NewSimpleClientset(&batchv1.Job{ + ObjectMeta: metav1.ObjectMeta{Name: job, Namespace: ns}, + Spec: batchv1.JobSpec{Parallelism: &n, Completions: &n}, + }) + return &webhook.Mutator{Clientset: cs} +} + +func TestStandalonePodGetsWritableHandoff(t *testing.T) { + m := &webhook.Mutator{} + ops := m.Mutate(context.Background(), qpuPod("fluence")) + if !opsHaveVolume(ops, HandoffVolumeName) { + t.Fatalf("a submitting pod should get the %q volume", HandoffVolumeName) + } + if !opsMountHandoffWritable(ops) { + t.Error("the workload container should mount the handoff volume writable") + } + if !contains(opEnvNames(ops), HandoffDirEnv) { + t.Errorf("want %s on a submitting pod, got %v", HandoffDirEnv, opEnvNames(ops)) + } +} + +func TestStageVolumeSurvivesTheHandoffAdd(t *testing.T) { + // both volumes are added to a pod that had none, so the second must append + m := &webhook.Mutator{} + ops := m.Mutate(context.Background(), qpuPod("fluence")) + if !opsHaveVolume(ops, StageVolumeName) || !opsHaveVolume(ops, HandoffVolumeName) { + t.Fatal("both the stage and handoff volumes should be present") + } + for _, op := range ops { + if op.Path == "/spec/volumes" { + if vols, ok := op.Value.([]corev1.Volume); ok && len(vols) == 1 && vols[0].Name == HandoffVolumeName { + t.Error("the handoff volume replaced the whole volumes list instead of appending") + } + } + } +} + +func TestWorkerGetsNoHandoff(t *testing.T) { + ns, group, job := "default", "qgang", "qgang-job" + m := handoffJobMutator(ns, job, 2) + ops := m.Mutate(context.Background(), sharedQPUPod(ns, group, "qgang-1", job, "1")) + if opsHaveVolume(ops, HandoffVolumeName) { + t.Error("a worker should not get the handoff volume") + } + if contains(opEnvNames(ops), HandoffDirEnv) { + t.Error("a worker should not get FLUENCE_HANDOFF_DIR") + } +} + +func TestScoutSidecarMountsHandoffReadOnly(t *testing.T) { + ns, group, job := "default", "qgang2", "qgang2-job" + m := handoffJobMutator(ns, job, 2) + ops := m.Mutate(context.Background(), sharedQPUPod(ns, group, "qgang2-0", job, "0")) + if !opsHaveVolume(ops, HandoffVolumeName) { + t.Error("the scout should get the handoff volume") + } + if !opsMountHandoffWritable(ops) { + t.Error("the scout workload should mount the handoff volume writable") + } + if !sidecarMountsHandoffReadOnly(ops) { + t.Error("the scout sidecar should mount the handoff volume read only") + } + if !contains(opEnvNames(ops), HandoffDirEnv) { + t.Errorf("the scout should get %s, got %v", HandoffDirEnv, opEnvNames(ops)) + } +} + +func TestHandoffMountPathIsUnderOpt(t *testing.T) { + if !strings.HasPrefix(HandoffMountPath, "/opt/") { + t.Errorf("unexpected handoff mount path %q", HandoffMountPath) + } +} + +func TestWorkerGetsSessionEnvFromAnnotation(t *testing.T) { + ns, group, job := "default", "qgang3", "qgang3-job" + m := handoffJobMutator(ns, job, 2) + ops := m.Mutate(context.Background(), sharedQPUPod(ns, group, "qgang3-1", job, "1")) + e, ok := envOp(ops, QuantumSessionIDEnv) + if !ok { + t.Fatalf("worker missing %s", QuantumSessionIDEnv) + } + if e.ValueFrom == nil || e.ValueFrom.FieldRef == nil || !strings.Contains(e.ValueFrom.FieldRef.FieldPath, QuantumSessionIDAnnotation) { + t.Errorf("%s should read the %s annotation, got %+v", QuantumSessionIDEnv, QuantumSessionIDAnnotation, e) + } +} + +func TestScoutGetsNoSessionEnv(t *testing.T) { + ns, group, job := "default", "qgang4", "qgang4-job" + m := handoffJobMutator(ns, job, 2) + ops := m.Mutate(context.Background(), sharedQPUPod(ns, group, "qgang4-0", job, "0")) + if _, ok := envOp(ops, QuantumSessionIDEnv); ok { + t.Error("the scout acquires the session itself and must not get the worker env") + } +} diff --git a/pkg/webhook/handlers/quantum.go b/pkg/webhook/handlers/quantum.go index 26a09b0..a64c5d7 100644 --- a/pkg/webhook/handlers/quantum.go +++ b/pkg/webhook/handlers/quantum.go @@ -8,6 +8,7 @@ import ( "strconv" "strings" + "github.com/converged-computing/fluence/pkg/placement" "github.com/converged-computing/fluence/pkg/webhook" "github.com/converged-computing/fluence/pkg/webhook/spec" @@ -21,47 +22,23 @@ func init() { webhook.Register(&quantumHandler{}) } -// Quantum-specific policy. The webhook core knows NONE of these — they live only -// here, in the quantum handler. -// -// Model (producer/consumer split, no separate submitter pod). A quantum task's -// circuit comes from user code, so the pod that defines a task must RUN to submit -// it — submit and gate are mutually exclusive per pod. Gating therefore only -// helps pods that do NOT submit. A quantum gang in CoordinationShared mode is -// split, per pod, into two roles decided at admission: -// -// - PRODUCER (one member, the indexed-Job completion index 0): its own -// group-of-one -producer (minCount 1) so it schedules alone and runs -// the SINGLE real submit; staged with the interceptor in REAL (tag) mode and -// given the sidecar, which polls the task and ungates the consumers at -// position==1. NOT gated. The producer is one of the N members, so the -// application is run exactly N times — never N+1. -// - CONSUMERS (the other N-1 members): the gang (minCount N-1), each -// gated on quantum.braket/ready + preempting priority, told its role via -// FLUENCE_COORDINATION_ROLE=consumer and handed the producer's task id via -// FLUENCE_QUANTUM_JOB_ID. A consumer does NOT submit; it fetches the shared -// result by that id. Ungated together when the producer's task is ready. -// -// In CoordinationIndependent mode (the default) there is no shared result to -// coordinate: every member is its own standalone producer (real submit, no gate), -// each owning its task and its own queue wait. A lone quantum pod (no group) is -// always standalone. +// Quantum-specific policy. const ( // QuantumResource is the Fluxion resource a pod requests to ask Fluence to // schedule quantum work. Requesting it is the sole trigger for this handler. QuantumResource = "fluxion.flux-framework.org/qpu" - // QuantumGate holds a consumer pod unscheduled until the producer's task is - // ready (the producer's sidecar removes it). + // QuantumGate holds a worker pod unscheduled until the scout's task is + // ready (the scout's sidecar removes it). QuantumGate = "quantum.braket/ready" // ObserveLabel opts a STANDALONE quantum pod (a group of one) into - // observe-only telemetry: the sidecar is injected and polls queue position - // but ungates nothing. + // observe-only. The sidecar is injected and polls queue position + // does not ungate. ObserveLabel = "fluence.flux-framework.org/observe" // DependencyKindQuantumSubmit is the readiness Kind for the quantum resource - // type: consumer pods wait for a quantum submission to reach the device queue. + // type: worker pods wait for a quantum submission to reach the device queue. // First concrete instance of the general Dependency primitive (dependency.go). DependencyKindQuantumSubmit = "quantum-submit" @@ -70,10 +47,10 @@ const ( // without changing the mechanism. CoordinationAnnotation = "fluence.flux-framework.org/coordination" - // CoordinationShared: one real task; the producer (index 0) submits and the - // other members are gated consumers that fetch the producer's result. Each + // CoordinationShared: one real task; the scout (index 0) submits and the + // other members are gated workers that fetch the scout's result. Each // member is told its role via FLUENCE_COORDINATION_ROLE; a role-aware workload - // branches on it (producer submits, consumer fetches by FLUENCE_QUANTUM_JOB_ID). + // branches on it (scout submits, worker fetches by FLUENCE_QUANTUM_JOB_ID). CoordinationShared = "shared" // CoordinationIndependent (default): every member does its own quantum work; @@ -81,52 +58,118 @@ const ( // for, and never dedup tasks meant to be distinct. CoordinationIndependent = "independent" - // ProducerGroupSuffix names the producer's own group-of-one: -producer + // ScoutGroupSuffix names the scout's own group-of-one: -scout // (minCount 1) so it schedules alone and never deadlocks against the gated - // consumer gang. - ProducerGroupSuffix = "-producer" + // worker gang. + ScoutGroupSuffix = "-scout" // CompletionIndexAnnotation is the indexed-Job completion index the Job - // controller stamps on each pod; index "0" is the producer (deterministic + // controller stamps on each pod; index "0" is the scout (deterministic // election with no recorded state). CompletionIndexAnnotation = "batch.kubernetes.io/job-completion-index" - // ProducerIndex is the completion index promoted to producer. - ProducerIndex = "0" + // ScoutIndex is the completion index promoted to scout. + ScoutIndex = "0" - // GangGroupEnv tells the producer's sidecar which consumer group label to list + // GangGroupEnv tells the scout's sidecar which worker group label to list // and ungate when the task is ready. GangGroupEnv = "FLUENCE_GANG_GROUP" ) -// quantumHandler splits a shared quantum gang into a single producer (real -// submit + sidecar) and N-1 gated, role-aware consumers, or runs every member -// standalone in independent mode (see the package-level model comment). It is the -// only place in the webhook that knows about quantum resources, gates, -// coordination, or observe semantics. +// quantumHandler splits a shared quantum gang into a single scout (real +// submit + sidecar) and N-1 gated workers. type quantumHandler struct{} func (h *quantumHandler) Name() string { return "quantum" } -// Applies to any pod requesting the quantum resource. Producers, consumers, and -// standalone quantum pods all request it; nothing without the resource needs -// quantum handling, so this is the single, unambiguous trigger. +// Applies to any pod requesting the quantum resource. func (h *quantumHandler) Applies(ctx context.Context, m webhook.MutatorAPI, pod *corev1.Pod) bool { return spec.PodRequestsResource(pod, QuantumResource) } +var _ webhook.Admitter = (*quantumHandler)(nil) + +// Admit is admission control, see pkg/webhook/admission.go. +func (h *quantumHandler) Admit(ctx context.Context, m webhook.MutatorAPI, pod *corev1.Pod) error { + if spec.Annotation(pod, webhook.ProtectedAnnotation) != "" { + return fmt.Errorf("%s is set by fluence and must not be set on a submission. It marks a pod as "+ + "not preemptible, which is not yours to decide", webhook.ProtectedAnnotation) + } + g := resolveGroup(pod) + n := resolveGangSize(ctx, m, pod, g) + cores := placement.PodCores(pod) + key := pod.Namespace + "/" + pod.Name + want := cores + if coordinationMode(pod) == CoordinationShared && g != "" && n > 1 { + if groupAdmitted(ctx, m, pod.Namespace, g) { + return nil + } + key = pod.Namespace + "/" + g + want = cores * int(n) + } + return m.Admit(ctx, pod, key, want) +} + +// groupAdmitted reports whether the gang PodGroup already carries an admission record. +func groupAdmitted(ctx context.Context, m webhook.MutatorAPI, namespace, group string) bool { + c := m.Client() + if c == nil { + return false + } + pg, err := c.SchedulingV1alpha2().PodGroups(namespace).Get(ctx, group, metav1.GetOptions{}) + if err != nil { + return false + } + return pg.Annotations[webhook.AdmittedCoresAnnotation] != "" +} + +// admissionOps marks a quantum pod protected and names its admission record. +// A standalone pod is its own record and carries the total. +func admissionOps(pod *corev1.Pod, record string, standaloneCores int) []spec.Op { + ann := map[string]string{ + webhook.ProtectedAnnotation: webhook.ProtectedValue, + webhook.AdmittedGroupAnnotation: record, + } + if standaloneCores > 0 { + ann[webhook.AdmittedCoresAnnotation] = strconv.Itoa(standaloneCores) + } + return annotateOps(pod, ann) +} + +// annotateOps adds annotations with one patch when the pod has none yet, +// since repeated adds of the whole map would replace each other, and per key +// otherwise. The pod is updated in place so later helpers see the keys. +func annotateOps(pod *corev1.Pod, ann map[string]string) []spec.Op { + if pod.Annotations == nil { + pod.Annotations = map[string]string{} + for k, v := range ann { + pod.Annotations[k] = v + } + return []spec.Op{{Op: "add", Path: "/metadata/annotations", Value: ann}} + } + var ops []spec.Op + for k, v := range ann { + ops = append(ops, spec.Op{Op: "add", Path: "/metadata/annotations/" + escapeJSONPointer(k), Value: v}) + pod.Annotations[k] = v + } + return ops +} + func (h *quantumHandler) Mutate(ctx context.Context, m webhook.MutatorAPI, pod *corev1.Pod) []spec.Op { g := resolveGroup(pod) n := resolveGangSize(ctx, m, pod, g) mode := coordinationMode(pod) observe := spec.Label(pod, ObserveLabel) == "true" + cores := placement.PodCores(pod) // No coordination: a standalone quantum pod, or an explicitly independent // member. The REAL submit happens in THIS pod; the sidecar is added only for // observe-only telemetry. (independent mode routes every member here -> N - // standalone producers, each owning its task and its own queue wait.) + // standalone scouts, each owning its task and its own queue wait.) if mode != CoordinationShared || g == "" || n <= 1 { - ops := interceptorOps(pod) + ops := admissionOps(pod, pod.Namespace+"/"+pod.Name, cores) + ops = append(ops, interceptorOps(pod)...) + ops = append(ops, handoffOps(pod)...) if observe { sc := sidecarFor(m) sc.EnsureRBAC(ctx, pod.Namespace) @@ -137,65 +180,66 @@ func (h *quantumHandler) Mutate(ctx context.Context, m webhook.MutatorAPI, pod * return ops } - // shared mode: promote one member to producer; the rest are gated consumers. - if isProducer(ctx, m, pod, g) { - return h.mutateProducer(ctx, m, pod, g) + // shared mode: promote one member to scout; the rest are gated workers. + // the gang PodGroup is the admission record for scout and workers + // alike, so make sure it exists and stamp the total on it once + m.EnsurePodGroup(ctx, pod.Namespace, g, pod.Name, n-1) + m.RecordAdmission(ctx, pod.Namespace, g, cores*int(n)) + ops := admissionOps(pod, pod.Namespace+"/"+g, 0) + if isScout(ctx, m, pod, g) { + return append(ops, h.mutateScout(ctx, m, pod, g)...) } - return h.mutateConsumer(ctx, m, pod, g, n) + return append(ops, h.mutateWorker(ctx, m, pod, g, n)...) } -// mutateProducer wires the single producer member (indexed-Job completion index -// 0): its own group-of-one -producer (minCount 1) so it schedules alone +// mutateScout wires the single scout member (indexed-Job completion index +// 0): its own group-of-one -scout (minCount 1) so it schedules alone // and runs the REAL submit, the interceptor in tag mode, RBAC, and the sidecar -// told which consumer group to ungate (FLUENCE_GANG_GROUP). The producer is one +// told which worker group to ungate (FLUENCE_GANG_GROUP). The scout is one // of the N members, so the application is NOT run an extra time. Never gated. -func (h *quantumHandler) mutateProducer(ctx context.Context, m webhook.MutatorAPI, pod *corev1.Pod, group string) []spec.Op { - pg := group + ProducerGroupSuffix +func (h *quantumHandler) mutateScout(ctx context.Context, m webhook.MutatorAPI, pod *corev1.Pod, group string) []spec.Op { + pg := group + ScoutGroupSuffix m.EnsurePodGroup(ctx, pod.Namespace, pg, pod.Name, 1) ops := linkGroupOps(pod, pg) - ops = append(ops, interceptorOps(pod)...) // tag mode: the producer submits for real - ops = append(ops, roleEnvOps(pod, RoleProducer)...) // FLUENCE_COORDINATION_ROLE=producer + ops = append(ops, interceptorOps(pod)...) // tag mode: the scout submits for real + ops = append(ops, handoffOps(pod)...) // where the interceptor writes the task id and session + ops = append(ops, roleEnvOps(pod, RoleScout)...) // FLUENCE_COORDINATION_ROLE=scout sc := sidecarFor(m) sc.EnsureRBAC(ctx, pod.Namespace) extra := []corev1.EnvVar{{Name: GangGroupEnv, Value: group}} ops = append(ops, sc.ContainerOps(pod, false, extra)...) - log.Printf("[fluence-webhook] quantum producer %s/%s — group %s (ungates consumers %q)", + log.Printf("[fluence-webhook] quantum scout %s/%s — group %s (ungates workers %q)", pod.Namespace, pod.Name, pg, group) return ops } -// mutateConsumer wires a non-producer member: it joins the consumer gang -// (minCount N-1) and is gated until the producer's task is ready. It is told its -// role (FLUENCE_COORDINATION_ROLE=consumer) and handed the producer's task id +// mutateWorker wires a non-scout member: it joins the worker gang +// (minCount N-1) and is gated until the scout's task is ready. It is told its +// role (FLUENCE_COORDINATION_ROLE=worker) and handed the scout's task id // (FLUENCE_QUANTUM_JOB_ID, stamped on the pod by the sidecar at ungate). A -// role-aware consumer reads those and fetches the shared result instead of -// submitting — so the consumer never calls the vendor submit, and needs neither +// role-aware worker reads those and fetches the shared result instead of +// submitting — so the worker never calls the vendor submit, and needs neither // the interceptor nor a faux flag. -func (h *quantumHandler) mutateConsumer(ctx context.Context, m webhook.MutatorAPI, pod *corev1.Pod, group string, n int32) []spec.Op { +func (h *quantumHandler) mutateWorker(ctx context.Context, m webhook.MutatorAPI, pod *corev1.Pod, group string, n int32) []spec.Op { m.EnsurePodGroup(ctx, pod.Namespace, group, pod.Name, n-1) ops := linkGroupOps(pod, group) - // Express the wait as the GENERAL dependency primitive: this consumer depends - // on the quantum submission produced by -producer, held by the quantum - // gate. applyOps gates the pod, raises priority, and stamps depends-on-*. - dep := Dependency{Kind: DependencyKindQuantumSubmit, Producer: group + ProducerGroupSuffix, Gate: QuantumGate} + // the worker waits on the scout's submission, held by the quantum gate + dep := Dependency{Kind: DependencyKindQuantumSubmit, Scout: group + ScoutGroupSuffix, Gate: QuantumGate} ops = append(ops, dep.applyOps(pod)...) - ops = append(ops, consumerEnvOps(pod)...) - // A gated consumer never runs the QPU task — it only fetches the producer's - // shared result — so it must not hold the Fluxion quantum resource. Leaving it - // would make Fluxion allocate a qpu per consumer, capping the gang at the - // backend's graph qpu count and, on a single-slot real QPU, leaving the - // consumers unschedulable. Applies() already routed this pod on the request, so - // stripping it here is safe. + ops = append(ops, workerEnvOps(pod)...) + // A worker does not hold a qpu of its own. It uses the scout session, so + // its quantum resource request is dropped. Otherwise Fluxion would allocate + // a qpu per worker and a backend with one qpu could never seat the gang. ops = append(ops, dropQuantumResourceOps(pod)...) - log.Printf("[fluence-webhook] quantum consumer %s/%s — group %s minCount=%d, gated (role=consumer, qpu stripped)", + log.Printf("[fluence-webhook] quantum worker %s/%s — group %s minCount=%d, gated (role=worker, qpu stripped)", pod.Namespace, pod.Name, group, n-1) return ops } -// dropQuantumResourceOps removes the Fluxion quantum resource from a consumer's +// dropQuantumResourceOps removes the Fluxion quantum resource from a worker's // containers (requests and limits), returning the patch ops and mutating pod in // place. Only entries that are present are removed (a JSON-patch remove on a -// missing path would fail). The sidecar container is never a consumer concern. +// missing path would fail). The sidecar container is never a worker concern. func dropQuantumResourceOps(pod *corev1.Pod) []spec.Op { rn := corev1.ResourceName(QuantumResource) // JSON Pointer escaping for the resource key: '~' -> '~0', '/' -> '~1'. @@ -227,21 +271,21 @@ func coordinationMode(pod *corev1.Pod) string { return CoordinationIndependent } -// isProducer decides whether THIS pod is the gang's single producer. Indexed Job -// (recommended): completion index 0 is the producer — deterministic, race-free, -// no recorded state. Otherwise: first arrival claims the producer slot by the -// absence of the producer PodGroup (best-effort under concurrent admission; +// isScout decides whether THIS pod is the gang's single scout. Indexed Job +// (recommended): completion index 0 is the scout — deterministic, race-free, +// no recorded state. Otherwise: first arrival claims the scout slot by the +// absence of the scout PodGroup (best-effort under concurrent admission; // prefer an indexed Job for determinism). Indexing a nil annotations map yields // ok=false, so the indexed branch is nil-safe. -func isProducer(ctx context.Context, m webhook.MutatorAPI, pod *corev1.Pod, group string) bool { +func isScout(ctx context.Context, m webhook.MutatorAPI, pod *corev1.Pod, group string) bool { if idx, ok := pod.Annotations[CompletionIndexAnnotation]; ok { - return idx == ProducerIndex + return idx == ScoutIndex } c := m.Client() if c == nil { - return true // tests / no client: treat as producer + return true // tests / no client: treat as scout } - pg := group + ProducerGroupSuffix + pg := group + ScoutGroupSuffix if _, err := c.SchedulingV1alpha2().PodGroups(pod.Namespace).Get(ctx, pg, metav1.GetOptions{}); err == nil { return false // already claimed by an earlier arrival } @@ -330,7 +374,7 @@ func countGroupPods(ctx context.Context, m webhook.MutatorAPI, namespace, group return int32(len(list.Items)) } -// linkGroupOps ensures the gang pod carries the group label (so the producer's +// linkGroupOps ensures the gang pod carries the group label (so the scout's // sidecar can list it) and is linked to the gang PodGroup via // spec.schedulingGroup.podGroupName. Idempotent. func linkGroupOps(pod *corev1.Pod, group string) []spec.Op { @@ -362,35 +406,42 @@ func escapeJSONPointer(s string) string { const QuantumClassicalPriorityClass = "fluence-quantum-classical" -// ── coordination role (producer / consumer) ───────────────────────────────────── +// ── coordination role (scout / worker) ───────────────────────────────────── // // In a shared gang each member is told its role positively, so the application // branches on it instead of relying on any submit-interception magic: -// producer submits the one real task (and is tagged so the sidecar finds it); -// consumer does NOT submit — it reads the producer's task id and fetches the +// scout submits the one real task (and is tagged so the sidecar finds it); +// worker does NOT submit — it reads the scout's task id and fetches the // shared result (e.g. via the vendor's S3-backed result API). -// The role is decided at admission by isProducer (completion index 0, else the -// producer-group claim) and surfaced as FLUENCE_COORDINATION_ROLE. Because the +// The role is decided at admission by isScout (completion index 0, else the +// scout-group claim) and surfaced as FLUENCE_COORDINATION_ROLE. Because the // election is the webhook's, this env is the single source of truth — the // container never re-derives its role from the Job index (which loose, non-Job // pods don't even have). const ( // CoordinationRoleEnv carries the pod's role in a shared gang. A role-aware - // workload branches on it: RoleProducer submits, RoleConsumer fetches the + // workload branches on it: RoleScout submits, RoleWorker fetches the // shared result by id. Unset for standalone/independent pods (they all submit). CoordinationRoleEnv = "FLUENCE_COORDINATION_ROLE" - RoleProducer = "producer" - RoleConsumer = "consumer" + RoleScout = "scout" + RoleWorker = "worker" // QuantumJobIDAnnotation is the vendor-neutral task id the ungating sidecar - // stamps on each consumer (mirrors python/fluence/ungate.py JOB_ID_ANNOTATION), + // stamps on each worker (mirrors python/fluence/ungate.py JOB_ID_ANNOTATION), // BEFORE removing the gate. Surfaced into FLUENCE_QUANTUM_JOB_ID via the - // downward API so a consumer can fetch the producer's result by id. + // downward API so a worker can fetch the scout's result by id. QuantumJobIDAnnotation = "fluence.flux-framework.org/quantum-job-id" - // QuantumJobIDEnv is the env a consumer reads for the producer's task id. + // QuantumJobIDEnv is the env a worker reads for the scout's task id. QuantumJobIDEnv = "FLUENCE_QUANTUM_JOB_ID" + + // QuantumSessionIDAnnotation is the session the scout acquired, stamped + // on each worker at ungate when there is one, the same way as the job id + // (python/fluence/ungate.py SESSION_ID_ANNOTATION). A worker can submit + // into the same session with it. + QuantumSessionIDAnnotation = "fluence.flux-framework.org/quantum-session-id" + QuantumSessionIDEnv = "FLUENCE_QUANTUM_SESSION_ID" ) // roleEnvOps sets FLUENCE_COORDINATION_ROLE= on each non-sidecar container. @@ -398,13 +449,13 @@ func roleEnvOps(pod *corev1.Pod, role string) []spec.Op { return setContainerEnvOps(pod, corev1.EnvVar{Name: CoordinationRoleEnv, Value: role}) } -// consumerEnvOps tells a consumer its role and hands it the producer's task id -// (FLUENCE_QUANTUM_JOB_ID, downward API from the annotation the ungating sidecar -// stamps). A consumer never submits, so it gets neither the interceptor nor any -// faux flag — just its role and the id to fetch the shared result with. -func consumerEnvOps(pod *corev1.Pod) []spec.Op { - ops := roleEnvOps(pod, RoleConsumer) +// workerEnvOps tells a worker its role and hands it the scout's task id and +// session, read from the annotations the sidecar stamps at ungate. A worker +// may fetch the scout result by id or submit into the scout session. +func workerEnvOps(pod *corev1.Pod) []spec.Op { + ops := roleEnvOps(pod, RoleWorker) ops = append(ops, setContainerEnvOps(pod, spec.AnnotationEnv(QuantumJobIDEnv, QuantumJobIDAnnotation))...) + ops = append(ops, setContainerEnvOps(pod, spec.AnnotationEnv(QuantumSessionIDEnv, QuantumSessionIDAnnotation))...) return ops } @@ -460,6 +511,13 @@ const ( // and prepended to PYTHONPATH (Model C delivery). StageVolumeName = "fluence-pkg" StageMountPath = "/opt/fluence-staged" + + // HandoffVolumeName / HandoffMountPath / HandoffDirEnv: the shared emptyDir + // a submitting pod's interceptor writes its task id and session to, and + // its sidecar reads (python/fluence/handoff.py). Workers never get it. + HandoffVolumeName = "fluence-handoff" + HandoffMountPath = "/opt/fluence-handoff" + HandoffDirEnv = "FLUENCE_HANDOFF_DIR" ) // sidecarImage resolves the sidecar image: the FLUENCE_SIDECAR_IMAGE override @@ -515,8 +573,8 @@ func ensureSidecarRBAC(ctx context.Context, m webhook.MutatorAPI, namespace stri // copies it into a shared emptyDir, mounted into every workload container // (skipping the sidecar) with PYTHONPATH + FLUENCE_POD_UID, so Python auto-imports // the interceptor via sitecustomize, which tags the vendor submit so the sidecar -// can find the task. Added to producers and standalone/independent pods (the ones -// that actually submit); consumers don't submit, so they don't get it. +// can find the task. Added to scouts and standalone/independent pods (the ones +// that actually submit); workers don't submit, so they don't get it. func interceptorOps(pod *corev1.Pod) []spec.Op { var ops []spec.Op @@ -526,6 +584,7 @@ func interceptorOps(pod *corev1.Pod) []spec.Op { } else { ops = append(ops, spec.Op{Op: "add", Path: "/spec/volumes/-", Value: vol}) } + pod.Spec.Volumes = append(pod.Spec.Volumes, vol) initc := corev1.Container{ Name: "fluence-stage", @@ -553,6 +612,7 @@ func interceptorOps(pod *corev1.Pod) []spec.Op { } else { ops = append(ops, spec.Op{Op: "add", Path: fmt.Sprintf("/spec/containers/%d/volumeMounts/-", i), Value: mount}) } + pod.Spec.Containers[i].VolumeMounts = append(pod.Spec.Containers[i].VolumeMounts, mount) if !spec.HasEnv(c, "PYTHONPATH") { if len(c.Env) == 0 { ops = append(ops, spec.Op{Op: "add", Path: fmt.Sprintf("/spec/containers/%d/env", i), Value: []corev1.EnvVar{pythonpath}}) @@ -570,6 +630,40 @@ func interceptorOps(pod *corev1.Pod) []spec.Op { return ops } +// handoffOps gives a submitting pod the handoff volume. Workload containers +// mount it writable and get FLUENCE_HANDOFF_DIR. The sidecar mounts it read +// only in sidecarContainerOps. Idempotent. +func handoffOps(pod *corev1.Pod) []spec.Op { + for _, v := range pod.Spec.Volumes { + if v.Name == HandoffVolumeName { + return nil + } + } + var ops []spec.Op + vol := corev1.Volume{Name: HandoffVolumeName, VolumeSource: corev1.VolumeSource{EmptyDir: &corev1.EmptyDirVolumeSource{}}} + if len(pod.Spec.Volumes) == 0 { + ops = append(ops, spec.Op{Op: "add", Path: "/spec/volumes", Value: []corev1.Volume{vol}}) + } else { + ops = append(ops, spec.Op{Op: "add", Path: "/spec/volumes/-", Value: vol}) + } + pod.Spec.Volumes = append(pod.Spec.Volumes, vol) + + mount := corev1.VolumeMount{Name: HandoffVolumeName, MountPath: HandoffMountPath} + for i, c := range pod.Spec.Containers { + if c.Name == SidecarContainerName { + continue + } + if len(c.VolumeMounts) == 0 { + ops = append(ops, spec.Op{Op: "add", Path: fmt.Sprintf("/spec/containers/%d/volumeMounts", i), Value: []corev1.VolumeMount{mount}}) + } else { + ops = append(ops, spec.Op{Op: "add", Path: fmt.Sprintf("/spec/containers/%d/volumeMounts/-", i), Value: mount}) + } + pod.Spec.Containers[i].VolumeMounts = append(pod.Spec.Containers[i].VolumeMounts, mount) + } + ops = append(ops, setContainerEnvOps(pod, corev1.EnvVar{Name: HandoffDirEnv, Value: HandoffMountPath})...) + return ops +} + // sidecarContainerOps adds the fluence sidecar container (pod identity env, the // generic FLUXION_* contract from InjectedEnv, the observe flag, handler-supplied // extraEnv, and the workload's secret/configMap-sourced credentials) and sets the @@ -605,9 +699,15 @@ func sidecarContainerOps(m webhook.MutatorAPI, pod *corev1.Pod, observe bool, ex } } } + var mounts []corev1.VolumeMount + for _, v := range pod.Spec.Volumes { + if v.Name == HandoffVolumeName { + mounts = append(mounts, corev1.VolumeMount{Name: HandoffVolumeName, MountPath: HandoffMountPath, ReadOnly: true}) + } + } sidecar := corev1.Container{ Name: SidecarContainerName, Image: sidecarImage(), ImagePullPolicy: corev1.PullAlways, - Env: env, + Env: env, VolumeMounts: mounts, Resources: corev1.ResourceRequirements{Requests: corev1.ResourceList{ corev1.ResourceCPU: resource.MustParse("100m"), corev1.ResourceMemory: resource.MustParse("256Mi"), }}, diff --git a/pkg/webhook/handlers/quantum_test.go b/pkg/webhook/handlers/quantum_test.go index 10a000d..d4faf0d 100644 --- a/pkg/webhook/handlers/quantum_test.go +++ b/pkg/webhook/handlers/quantum_test.go @@ -1,10 +1,4 @@ -/* -Copyright 2024 Lawrence Livermore National Security, LLC - (c.f. AUTHORS, NOTICE.LLNS, COPYING) -SPDX-License-Identifier: Apache-2.0 -*/ - -// quantum_test.go — all tests for the quantum handler: the producer/consumer +// quantum_test.go — all tests for the quantum handler: the scout/worker // shared-coordination split (no separate submitter pod), independent mode, // the coordination role + job-id handoff, the sidecar wiring, the Dependency primitive, and the // standalone/observe paths. Shared fixtures (qpuPod, cpuPod, op helpers) live in @@ -57,11 +51,11 @@ func TestObserveLabelInjectsSidecar(t *testing.T) { } } -// ── shared coordination: producer / consumer split ────────────────────────────── +// ── shared coordination: scout / worker split ────────────────────────────── // sharedQPUPod is a quantum workload pod (requests the resource) in a group, // owned by a Job of parallelism N, with coordination=shared and a completion -// index. Index "0" is the producer; any other index is a consumer. This is the +// index. Index "0" is the scout; any other index is a worker. This is the // real shape: an indexed Job whose identical template yields differentiated // roles purely from the completion index. func sharedQPUPod(ns, group, name, job, index string) *corev1.Pod { @@ -98,11 +92,11 @@ func mincount(t *testing.T, cs *fake.Clientset, ns, group string) (int32, bool) return pg.Spec.SchedulingPolicy.Gang.MinCount, true } -// A shared-mode CONSUMER (completion index != 0, owned by Job parallelism=3) is -// gated, told its role (FLUENCE_COORDINATION_ROLE=consumer), joins the -// consumer gang at minCount N-1 (the split), and gets NO sidecar (it is gated). -// No separate submitter pod is ever created — the producer is one of the N members. -func TestSharedConsumerGatedRoleAndSplit(t *testing.T) { +// A shared-mode WORKER (completion index != 0, owned by Job parallelism=3) is +// gated, told its role (FLUENCE_COORDINATION_ROLE=worker), joins the +// worker gang at minCount N-1 (the split), and gets NO sidecar (it is gated). +// No separate submitter pod is ever created — the scout is one of the N members. +func TestSharedWorkerGatedRoleAndSplit(t *testing.T) { ns, group, job := "default", "qg", "qg-job" par := int32(3) cs := fake.NewSimpleClientset(&batchv1.Job{ @@ -113,20 +107,20 @@ func TestSharedConsumerGatedRoleAndSplit(t *testing.T) { ops := m.Mutate(context.Background(), sharedQPUPod(ns, group, "qg-1", job, "1")) if !hasGateOp(ops) { - t.Error("consumer must be gated") + t.Error("worker must be gated") } if hasSidecarOp(ops) { - t.Error("consumer (gated) must NOT get a sidecar") + t.Error("worker (gated) must NOT get a sidecar") } if !hasDropQuantumResourceOp(ops) { - t.Error("consumer (gated, never runs the QPU) must have its qpu resource stripped") + t.Error("worker (gated, uses the scout session) must have its qpu resource stripped") } - if e, ok := envOp(ops, CoordinationRoleEnv); !ok || e.Value != RoleConsumer { - t.Errorf("consumer must get %s=%s", CoordinationRoleEnv, RoleConsumer) + if e, ok := envOp(ops, CoordinationRoleEnv); !ok || e.Value != RoleWorker { + t.Errorf("worker must get %s=%s", CoordinationRoleEnv, RoleWorker) } - // Consumer gang is minCount N-1 (the producer/consumer split). + // Worker gang is minCount N-1 (the scout/worker split). if mc, ok := mincount(t, cs, ns, group); !ok || mc != 2 { - t.Errorf("consumer PodGroup minCount=%d (ok=%v), want 2 (N-1 split)", mc, ok) + t.Errorf("worker PodGroup minCount=%d (ok=%v), want 2 (N-1 split)", mc, ok) } // No separate submitter pod is created. pods, _ := cs.CoreV1().Pods(ns).List(context.Background(), metav1.ListOptions{}) @@ -135,11 +129,11 @@ func TestSharedConsumerGatedRoleAndSplit(t *testing.T) { } } -// The shared-mode PRODUCER (completion index 0) is wired as the real coordinator: -// its own group-of-one -producer at minCount 1, the real sidecar, not -// gated, role=producer, and told which consumer group to ungate via +// The shared-mode SCOUT (completion index 0) is wired as the real coordinator: +// its own group-of-one -scout at minCount 1, the real sidecar, not +// gated, role=scout, and told which worker group to ungate via // FLUENCE_GANG_GROUP. It is one of the N members — no extra pod is created. -func TestSharedProducerWiredAsRealSidecar(t *testing.T) { +func TestSharedScoutWiredAsRealSidecar(t *testing.T) { ns, group, job := "default", "qg2", "qg2-job" par := int32(2) cs := fake.NewSimpleClientset(&batchv1.Job{ @@ -150,21 +144,21 @@ func TestSharedProducerWiredAsRealSidecar(t *testing.T) { ops := m.Mutate(context.Background(), sharedQPUPod(ns, group, "qg2-0", job, "0")) if !hasSidecarOp(ops) { - t.Error("producer must get the real sidecar") + t.Error("scout must get the real sidecar") } if hasGateOp(ops) { - t.Error("producer must NOT be gated") + t.Error("scout must NOT be gated") } if hasDropQuantumResourceOp(ops) { - t.Error("producer must KEEP its qpu resource (it runs the real submit)") + t.Error("scout must KEEP its qpu resource (it runs the real submit)") } - if e, ok := envOp(ops, CoordinationRoleEnv); !ok || e.Value != RoleProducer { - t.Errorf("producer must get %s=%s", CoordinationRoleEnv, RoleProducer) + if e, ok := envOp(ops, CoordinationRoleEnv); !ok || e.Value != RoleScout { + t.Errorf("scout must get %s=%s", CoordinationRoleEnv, RoleScout) } if _, ok := envOp(ops, QuantumJobIDEnv); ok { - t.Error("producer must NOT get FLUENCE_QUANTUM_JOB_ID (it submits its own task)") + t.Error("scout must NOT get FLUENCE_QUANTUM_JOB_ID (it submits its own task)") } - // FLUENCE_GANG_GROUP (the consumer group to ungate) is on the sidecar. + // FLUENCE_GANG_GROUP (the worker group to ungate) is on the sidecar. var sidecar *corev1.Container for _, op := range ops { if c, ok := op.Value.(corev1.Container); ok && c.Name == SidecarContainerName { @@ -173,7 +167,7 @@ func TestSharedProducerWiredAsRealSidecar(t *testing.T) { } } if sidecar == nil { - t.Fatal("no sidecar container on producer") + t.Fatal("no sidecar container on scout") } var gotGang bool for _, e := range sidecar.Env { @@ -182,21 +176,21 @@ func TestSharedProducerWiredAsRealSidecar(t *testing.T) { } } if !gotGang { - t.Errorf("producer sidecar must get %s=%q", GangGroupEnv, group) + t.Errorf("scout sidecar must get %s=%q", GangGroupEnv, group) } - // Producer is its own group-of-one (minCount 1). - if mc, ok := mincount(t, cs, ns, group+ProducerGroupSuffix); !ok || mc != 1 { - t.Errorf("producer PodGroup %s minCount=%d (ok=%v), want 1", group+ProducerGroupSuffix, mc, ok) + // Scout is its own group-of-one (minCount 1). + if mc, ok := mincount(t, cs, ns, group+ScoutGroupSuffix); !ok || mc != 1 { + t.Errorf("scout PodGroup %s minCount=%d (ok=%v), want 1", group+ScoutGroupSuffix, mc, ok) } // No separate submitter pod. pods, _ := cs.CoreV1().Pods(ns).List(context.Background(), metav1.ListOptions{}) if len(pods.Items) != 0 { - t.Errorf("producer is a member, not a spawned pod; found %d pods", len(pods.Items)) + t.Errorf("scout is a member, not a spawned pod; found %d pods", len(pods.Items)) } } -// Shared mode never creates an extra pod: a full gang (producer index 0 + -// consumers) is N members, so the application runs exactly N times (not N+1 as +// Shared mode never creates an extra pod: a full gang (scout index 0 + +// workers) is N members, so the application runs exactly N times (not N+1 as // the old submitter-pod model did). func TestSharedGangNoSeparateSubmitterPod(t *testing.T) { ns, group, job := "default", "qauto", "qauto-job" @@ -206,19 +200,19 @@ func TestSharedGangNoSeparateSubmitterPod(t *testing.T) { Spec: batchv1.JobSpec{Parallelism: &par, Completions: &par}}) m := &webhook.Mutator{Clientset: cs} - m.Mutate(context.Background(), sharedQPUPod(ns, group, "qauto-0", job, "0")) // producer - m.Mutate(context.Background(), sharedQPUPod(ns, group, "qauto-1", job, "1")) // consumer + m.Mutate(context.Background(), sharedQPUPod(ns, group, "qauto-0", job, "0")) // scout + m.Mutate(context.Background(), sharedQPUPod(ns, group, "qauto-1", job, "1")) // worker pods, _ := cs.CoreV1().Pods(ns).List(context.Background(), metav1.ListOptions{}) if len(pods.Items) != 0 { t.Errorf("shared mode must not create any pods (no submitter); found %d", len(pods.Items)) } // Both groups exist with the right minCounts. - if mc, ok := mincount(t, cs, ns, group+ProducerGroupSuffix); !ok || mc != 1 { - t.Errorf("producer group minCount=%d (ok=%v), want 1", mc, ok) + if mc, ok := mincount(t, cs, ns, group+ScoutGroupSuffix); !ok || mc != 1 { + t.Errorf("scout group minCount=%d (ok=%v), want 1", mc, ok) } if mc, ok := mincount(t, cs, ns, group); !ok || mc != 1 { - t.Errorf("consumer group minCount=%d (ok=%v), want N-1=1", mc, ok) + t.Errorf("worker group minCount=%d (ok=%v), want N-1=1", mc, ok) } } @@ -243,8 +237,8 @@ func TestIndependentGroupedQuantumIsStandalone(t *testing.T) { if _, ok := envOp(ops, CoordinationRoleEnv); ok { t.Error("independent member must not get a coordination role env") } - if _, ok := mincount(t, cs, ns, group+ProducerGroupSuffix); ok { - t.Error("independent mode must not create a producer group") + if _, ok := mincount(t, cs, ns, group+ScoutGroupSuffix); ok { + t.Error("independent mode must not create a scout group") } pods, _ := cs.CoreV1().Pods(ns).List(context.Background(), metav1.ListOptions{}) if len(pods.Items) != 0 { @@ -277,7 +271,7 @@ func TestStandaloneQuantumIsReal(t *testing.T) { } // Even with coordination=shared, a group of one (Job parallelism 1) has no -// consumers to coordinate, so it falls through to the standalone real-submit path. +// workers to coordinate, so it falls through to the standalone real-submit path. func TestSharedGroupOfOneIsStandalone(t *testing.T) { ns, group, job := "default", "one", "one-job" par := int32(1) @@ -290,8 +284,8 @@ func TestSharedGroupOfOneIsStandalone(t *testing.T) { if hasGateOp(ops) { t.Error("shared group-of-one must not be gated") } - if _, ok := mincount(t, cs, ns, group+ProducerGroupSuffix); ok { - t.Error("shared group-of-one must not create a producer group") + if _, ok := mincount(t, cs, ns, group+ScoutGroupSuffix); ok { + t.Error("shared group-of-one must not create a scout group") } } @@ -364,10 +358,10 @@ func unescapeJSONPointer(s string) string { return out } -// A shared-mode consumer is expressed as a general Dependency: gated, stamped -// with depends-on-{kind,producer,gate}, and the producer is the -producer +// A shared-mode worker is expressed as a general Dependency: gated, stamped +// with depends-on-{kind,scout,gate}, and the scout is the -scout // group. -func TestQuantumConsumerIsGeneralDependency(t *testing.T) { +func TestQuantumWorkerIsGeneralDependency(t *testing.T) { ns, group, job := "default", "depq", "depq-job" par := int32(3) cs := fake.NewSimpleClientset(&batchv1.Job{ @@ -378,14 +372,14 @@ func TestQuantumConsumerIsGeneralDependency(t *testing.T) { ops := m.Mutate(context.Background(), sharedQPUPod(ns, group, "depq-1", job, "1")) if !hasGateOp(ops) { - t.Errorf("consumer not gated by the dependency (ops: %+v)", ops) + t.Errorf("worker not gated by the dependency (ops: %+v)", ops) } ann := annotationOps(ops) if ann[DependsOnKindAnnotation] != DependencyKindQuantumSubmit { t.Errorf("depends-on-kind=%q, want %q", ann[DependsOnKindAnnotation], DependencyKindQuantumSubmit) } - if ann[DependsOnProducerAnnotation] != group+ProducerGroupSuffix { - t.Errorf("depends-on-producer=%q, want %q (the producer group)", ann[DependsOnProducerAnnotation], group+ProducerGroupSuffix) + if ann[DependsOnScoutAnnotation] != group+ScoutGroupSuffix { + t.Errorf("depends-on-scout=%q, want %q (the scout group)", ann[DependsOnScoutAnnotation], group+ScoutGroupSuffix) } if ann[DependsOnGateAnnotation] != QuantumGate { t.Errorf("depends-on-gate=%q, want %q", ann[DependsOnGateAnnotation], QuantumGate) @@ -397,11 +391,11 @@ func TestQuantumConsumerIsGeneralDependency(t *testing.T) { func TestDependencyOfRoundTrip(t *testing.T) { pod := &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Annotations: map[string]string{ DependsOnKindAnnotation: DependencyKindQuantumSubmit, - DependsOnProducerAnnotation: "grp", + DependsOnScoutAnnotation: "grp", DependsOnGateAnnotation: QuantumGate, }}} d, ok := DependencyOf(pod) - if !ok || d.Kind != DependencyKindQuantumSubmit || d.Producer != "grp" || d.Gate != QuantumGate { + if !ok || d.Kind != DependencyKindQuantumSubmit || d.Scout != "grp" || d.Gate != QuantumGate { t.Errorf("DependencyOf=%+v ok=%v, want quantum-submit/grp/%s", d, ok, QuantumGate) } if _, ok := DependencyOf(&corev1.Pod{}); ok { @@ -409,11 +403,10 @@ func TestDependencyOfRoundTrip(t *testing.T) { } } -// The consumer is role-aware: it gets FLUENCE_COORDINATION_ROLE=consumer and the -// producer's task id via the FLUENCE_QUANTUM_JOB_ID downward-API env, and it is -// NOT staged with the interceptor (a consumer never submits, so it needs neither -// the interceptor nor any faux flag). The user's script branches on the role. -func TestQuantumConsumerStagedWithRole(t *testing.T) { +// A worker gets FLUENCE_COORDINATION_ROLE=worker and the scout's task id in +// FLUENCE_QUANTUM_JOB_ID. It is not staged with the interceptor, since any +// submit it makes goes into the scout session and needs no tagging. +func TestQuantumWorkerStagedWithRole(t *testing.T) { ns, group, job := "default", "roleq", "roleq-job" par := int32(2) cs := fake.NewSimpleClientset(&batchv1.Job{ @@ -424,19 +417,19 @@ func TestQuantumConsumerStagedWithRole(t *testing.T) { ops := m.Mutate(context.Background(), sharedQPUPod(ns, group, "roleq-1", job, "1")) // Role surfaced to the container. - if e, ok := envOp(ops, CoordinationRoleEnv); !ok || e.Value != RoleConsumer { - t.Errorf("consumer missing %s=%s (got %+v, ok=%v)", CoordinationRoleEnv, RoleConsumer, e, ok) + if e, ok := envOp(ops, CoordinationRoleEnv); !ok || e.Value != RoleWorker { + t.Errorf("worker missing %s=%s (got %+v, ok=%v)", CoordinationRoleEnv, RoleWorker, e, ok) } - // A consumer never submits, so it is NOT staged with the interceptor. + // no interceptor on a worker, its submits go into the scout session if _, ok := envOp(ops, "PYTHONPATH"); ok { - t.Error("consumer must NOT be staged with the interceptor (it does not submit)") + t.Error("worker must not be staged with the interceptor") } - // Producer's task id sourced from the annotation the ungating sidecar stamps. + // Scout's task id sourced from the annotation the ungating sidecar stamps. e, ok := envOp(ops, QuantumJobIDEnv) if !ok { - t.Fatalf("consumer missing %s env", QuantumJobIDEnv) + t.Fatalf("worker missing %s env", QuantumJobIDEnv) } if e.ValueFrom == nil || e.ValueFrom.FieldRef == nil || e.ValueFrom.FieldRef.FieldPath != "metadata.annotations['"+QuantumJobIDAnnotation+"']" { @@ -511,11 +504,11 @@ func TestSidecarInheritsWorkloadSecretEnv(t *testing.T) { } } -// The producer member of a shared gang requests the quantum resource (it runs the +// The scout member of a shared gang requests the quantum resource (it runs the // real submit). Sanity check that the helper builds a quantum pod. -func TestSharedProducerRequestsQuantumResource(t *testing.T) { +func TestSharedScoutRequestsQuantumResource(t *testing.T) { p := sharedQPUPod("default", "g", "g-0", "g-job", "0") if !spec.PodRequestsResource(p, QuantumResource) { - t.Error("producer must request the quantum resource (it runs the real submit)") + t.Error("scout must request the quantum resource (it runs the real submit)") } } diff --git a/pkg/webhook/handlers/registry_test.go b/pkg/webhook/handlers/registry_test.go index 346d786..08ed644 100644 --- a/pkg/webhook/handlers/registry_test.go +++ b/pkg/webhook/handlers/registry_test.go @@ -1,9 +1,3 @@ -/* -Copyright 2024 Lawrence Livermore National Security, LLC - (c.f. AUTHORS, NOTICE.LLNS, COPYING) -SPDX-License-Identifier: Apache-2.0 -*/ - // Registry behavior: dispatch order comes from the active handler list (not a // per-handler Order), and the list both selects and orders handlers. package handlers diff --git a/pkg/webhook/webhook.go b/pkg/webhook/webhook.go index b39bec1..b81fcba 100644 --- a/pkg/webhook/webhook.go +++ b/pkg/webhook/webhook.go @@ -63,6 +63,11 @@ const ( type Mutator struct { AttributeKeys []string Clientset kubernetes.Interface + + // AdmissionControl turns on admission control, see admission.go. + // ReserveCores is held back from its budget. + AdmissionControl bool + ReserveCores int } // compile-time check that *Mutator satisfies the handler capability interface. @@ -139,17 +144,35 @@ func (m *Mutator) EnsurePodGroup(ctx context.Context, namespace, group, leaderPo // patch operations from those that apply. Pods not scheduled by Fluence are left // untouched. func (m *Mutator) Mutate(ctx context.Context, pod *corev1.Pod) []spec.Op { - if pod.Spec.SchedulerName != SchedulerName { + ops, err := m.MutateOrDeny(ctx, pod) + if err != nil { + log.Printf("[fluence-webhook] %s/%s refused: %v", pod.Namespace, pod.Name, err) return nil } + return ops +} + +// MutateOrDeny runs the handlers that apply, in order. A handler that is also +// an Admitter may refuse the pod first. A refusal returns the error and no +// patches, and the HTTP handler turns it into a denied review. +func (m *Mutator) MutateOrDeny(ctx context.Context, pod *corev1.Pod) ([]spec.Op, error) { + if pod.Spec.SchedulerName != SchedulerName { + return nil, nil + } var ops []spec.Op for _, h := range registered() { - if h.Applies(ctx, m, pod) { - log.Printf("[fluence-webhook] handler %q applies to %s/%s", h.Name(), pod.Namespace, pod.Name) - ops = append(ops, h.Mutate(ctx, m, pod)...) + if !h.Applies(ctx, m, pod) { + continue + } + log.Printf("[fluence-webhook] handler %q applies to %s/%s", h.Name(), pod.Namespace, pod.Name) + if a, ok := h.(Admitter); ok { + if err := a.Admit(ctx, m, pod); err != nil { + return nil, err + } } + ops = append(ops, h.Mutate(ctx, m, pod)...) } - return ops + return ops, nil } // ── HTTP handler ─────────────────────────────────────────────────────────────── @@ -168,7 +191,19 @@ func (m *Mutator) Handler(w http.ResponseWriter, r *http.Request) { resp := &admissionv1.AdmissionResponse{UID: review.Request.UID, Allowed: true} var pod corev1.Pod if err := json.Unmarshal(review.Request.Object.Raw, &pod); err == nil { - if ops := m.Mutate(r.Context(), &pod); len(ops) > 0 { + ops, derr := m.MutateOrDeny(r.Context(), &pod) + switch { + case derr != nil: + // an explicit refusal, which failurePolicy Ignore does not override + resp.Allowed = false + resp.Result = &metav1.Status{ + Status: metav1.StatusFailure, + Reason: metav1.StatusReasonForbidden, + Code: http.StatusForbidden, + Message: derr.Error(), + } + log.Printf("[fluence-webhook] denied pod %s/%s: %v", pod.Namespace, pod.Name, derr) + case len(ops) > 0: if patch, err := json.Marshal(ops); err == nil { pt := admissionv1.PatchTypeJSONPatch resp.Patch = patch diff --git a/python/fluence/handoff.py b/python/fluence/handoff.py new file mode 100644 index 0000000..e8aa1c0 --- /dev/null +++ b/python/fluence/handoff.py @@ -0,0 +1,127 @@ +""" +The handoff between the interceptor and the sidecar, a JSON file on a volume +the webhook mounts into both. Braket can find a task again by a tag the +interceptor stamped. QRMI cannot, so the interceptor writes the job id, and +the session when it acquired one, to $FLUENCE_HANDOFF_DIR/task.json and the +sidecar reads it. Writes are atomic and every failure is soft, since the +interceptor runs inside the user application. +""" + +from __future__ import annotations + +import json +import os +import tempfile +import time + +# the default matches the mount path the webhook uses +HANDOFF_DIR_ENV = "FLUENCE_HANDOFF_DIR" +DEFAULT_HANDOFF_DIR = "/opt/fluence-handoff" +HANDOFF_FILENAME = "task.json" + + +def handoff_path(path: "str | None" = None) -> str: + if path: + return path + d = os.environ.get(HANDOFF_DIR_ENV, DEFAULT_HANDOFF_DIR) + return os.path.join(d, HANDOFF_FILENAME) + + +def publish( + job_id, session_id=None, backend=None, pod_uid=None, tier=None, path=None +) -> bool: + """Write the record. Returns False on any failure and never raises.""" + record = {"job_id": job_id} + if session_id is not None: + record["session_id"] = session_id + if backend is not None: + record["backend"] = backend + if pod_uid is not None: + record["pod_uid"] = pod_uid + if tier is not None: + record["tier"] = tier + try: + dest = handoff_path(path) + os.makedirs(os.path.dirname(dest), exist_ok=True) + fd, tmp = tempfile.mkstemp( + dir=os.path.dirname(dest), prefix=".task.", suffix=".tmp" + ) + with os.fdopen(fd, "w") as f: + json.dump(record, f) + os.replace(tmp, dest) + return True + except Exception: + return False + + +def read(path=None) -> "dict | None": + """The record, or None when it is not there yet or cannot be parsed.""" + try: + with open(handoff_path(path)) as f: + record = json.load(f) + if isinstance(record, dict) and record.get("job_id"): + return record + except Exception: + pass + return None + + +def wait(timeout, interval=5, path=None, log=None) -> "dict | None": + """Poll for the record until it appears or timeout seconds pass.""" + deadline = time.time() + timeout + announced = False + while time.time() < deadline: + record = read(path) + if record is not None: + return record + if log and not announced: + log(f"waiting for the scout to publish its job id at {handoff_path(path)}") + announced = True + time.sleep(interval) + return None + + +# the sidecar writes this once the workers are done and the scout's release waits for it +DONE_FILENAME = "workers-done" + + +def done_path(path=None) -> str: + if path: + return path + d = os.environ.get(HANDOFF_DIR_ENV, DEFAULT_HANDOFF_DIR) + return os.path.join(d, DONE_FILENAME) + + +def mark_workers_done(path=None) -> bool: + """Write the marker. Never raises.""" + try: + dest = done_path(path) + os.makedirs(os.path.dirname(dest), exist_ok=True) + fd, tmp = tempfile.mkstemp( + dir=os.path.dirname(dest), prefix=".done.", suffix=".tmp" + ) + with os.fdopen(fd, "w") as f: + json.dump({"done_at": time.time()}, f) + os.replace(tmp, dest) + return True + except Exception: + return False + + +def workers_done(path=None) -> bool: + return os.path.exists(done_path(path)) + + +def wait_workers_done(timeout, interval=5, path=None, log=None) -> bool: + """Wait for the marker. A timeout of zero or less waits without limit.""" + deadline = time.time() + timeout if timeout and timeout > 0 else None + announced = False + while True: + if workers_done(path): + return True + if deadline is not None and time.time() >= deadline: + return False + if log and not announced: + log(f"waiting for the worker group to finish, marker {done_path(path)}") + announced = True + time.sleep(interval) diff --git a/python/fluence/interceptor.py b/python/fluence/interceptor.py index 8f5ef25..51b1368 100644 --- a/python/fluence/interceptor.py +++ b/python/fluence/interceptor.py @@ -26,12 +26,16 @@ def install() -> None: for provider in all_providers(): try: if provider.install_interceptor(pod_uid): - print(f"[fluence] interceptor installed for provider " - f"{provider.name!r} (pod_uid={pod_uid})", flush=True) + print( + f"[fluence] interceptor installed for provider " + f"{provider.name!r} (pod_uid={pod_uid})", + flush=True, + ) except Exception as e: # A provider's hook must never break the user app. - print(f"[fluence] interceptor for {provider.name!r} skipped: {e}", - flush=True) + print( + f"[fluence] interceptor for {provider.name!r} skipped: {e}", flush=True + ) # Install on import. diff --git a/python/fluence/providers/__init__.py b/python/fluence/providers/__init__.py index e7f4dc0..ce9c0f9 100644 --- a/python/fluence/providers/__init__.py +++ b/python/fluence/providers/__init__.py @@ -24,4 +24,4 @@ # Import provider modules so they self-register. Add new providers here. from fluence.providers import braket # noqa: F401,E402 -# from fluence.providers import ibm # noqa: F401 (when implemented) +from fluence.providers import qrmi # noqa: F401,E402 diff --git a/python/fluence/providers/base.py b/python/fluence/providers/base.py index 527dd45..9546784 100644 --- a/python/fluence/providers/base.py +++ b/python/fluence/providers/base.py @@ -24,7 +24,6 @@ import os from datetime import datetime, timezone - # Shared convention between every interceptor hook and every find_my_task. # The interceptor stamps this tag key with the pod UID; the sidecar searches # for it. Changing it is a coordinated change across all providers. @@ -123,8 +122,13 @@ def job_id(self, task: "Task") -> str: """Stable, vendor-neutral identifier handed to workers at ungate time.""" raise NotImplementedError + # ── registry ──────────────────────────────────────────────────────────────────── + + def session_id(self, task: "Task") -> "str | None": + """The session the task runs in, when the vendor has one. Workers are + handed it so they can submit into the same session.""" + return None -# ── registry ──────────────────────────────────────────────────────────────────── _REGISTRY: "list[Provider]" = [] @@ -163,5 +167,5 @@ def resolve_from_env() -> "Provider | None": attrs = {} for k, v in os.environ.items(): if k.startswith("FLUXION_"): - attrs[k[len("FLUXION_"):].lower()] = v - return resolve(attrs) \ No newline at end of file + attrs[k[len("FLUXION_") :].lower()] = v + return resolve(attrs) diff --git a/python/fluence/providers/braket.py b/python/fluence/providers/braket.py index bd11e9d..87a671d 100644 --- a/python/fluence/providers/braket.py +++ b/python/fluence/providers/braket.py @@ -40,9 +40,11 @@ def _region_from_arn(arn: str) -> str: # generic AWS_DEFAULT_REGION. A QPU like IQM Emerald lives in eu-north-1, # while the pod's AWS_DEFAULT_REGION is often us-east-1 — searching the wrong # region means the task is never found and workers never ungate. - return (region - or os.environ.get("FLUXION_REGION") - or os.environ.get("AWS_DEFAULT_REGION", "us-east-1")) + return ( + region + or os.environ.get("FLUXION_REGION") + or os.environ.get("AWS_DEFAULT_REGION", "us-east-1") + ) class BraketProvider(Provider): @@ -59,11 +61,8 @@ def install_interceptor(self, pod_uid: str) -> bool: original_run = AwsDevice.run def patched_run(self, task_specification, *args, **kwargs): - # Tag the submission with the pod-uid so the sidecar can find this task - # in the queue. The interceptor is staged only on pods that actually - # submit (producers and standalone/independent pods); consumers are - # role-aware (FLUENCE_COORDINATION_ROLE=consumer) and never call run(), - # so there is no submit to intercept and no faux mode to select. + # tag the task with the pod uid so the sidecar can find it. Only + # scouts and standalone pods are staged with this if pod_uid: tags = kwargs.get("tags", {}) tags[TAG_KEY] = pod_uid @@ -84,9 +83,9 @@ def matches(self, attrs: "dict[str, str]") -> bool: if qrmi.startswith("braket"): return True # Fallbacks for graphs that don't set qrmi_type: vendor or backend/arn. - vendor = (attrs.get("vendor") or "").lower() + vendor = (attrs.get("vendor") or "").lower() backend = (attrs.get("backend") or "").lower() - arn = (attrs.get("arn") or "").lower() + arn = (attrs.get("arn") or "").lower() if vendor in ("braket", "amazon", "aws"): return True return "braket" in backend or arn.startswith("arn:aws:braket") @@ -96,14 +95,22 @@ def _region_client(self, region: str): can be stubbed in tests (find_my_task builds one client per candidate region through this method rather than calling boto3 directly).""" import boto3 + return boto3.client("braket", region_name=region) def _client(self, backend: str): - region = (_region_from_arn(backend) if backend.startswith("arn:") - else (os.environ.get("FLUXION_REGION") - or os.environ.get("AWS_DEFAULT_REGION", "us-east-1"))) - log(f"[braket] using region {region} for task search " - f"(backend={backend!r} FLUXION_REGION={os.environ.get('FLUXION_REGION')!r})") + region = ( + _region_from_arn(backend) + if backend.startswith("arn:") + else ( + os.environ.get("FLUXION_REGION") + or os.environ.get("AWS_DEFAULT_REGION", "us-east-1") + ) + ) + log( + f"[braket] using region {region} for task search " + f"(backend={backend!r} FLUXION_REGION={os.environ.get('FLUXION_REGION')!r})" + ) return self._region_client(region) def _candidate_regions(self, backend: str): @@ -122,14 +129,22 @@ def _candidate_regions(self, backend: str): all_regions = [r.strip() for r in override.split(",") if r.strip()] else: # The fixed set of regions Amazon Braket operates in. - all_regions = ["us-east-1", "us-west-1", "us-west-2", "eu-west-2", "eu-north-1"] + all_regions = [ + "us-east-1", + "us-west-1", + "us-west-2", + "eu-west-2", + "eu-north-1", + ] preferred = [] if backend.startswith("arn:aws:braket"): r = _region_from_arn(backend) if r: preferred.append(r) - for r in (os.environ.get("FLUXION_REGION"), - os.environ.get("AWS_DEFAULT_REGION")): + for r in ( + os.environ.get("FLUXION_REGION"), + os.environ.get("AWS_DEFAULT_REGION"), + ): if r and r not in preferred: preferred.append(r) # preferred first, then any remaining regions (keep only ones in the @@ -140,8 +155,10 @@ def _candidate_regions(self, backend: str): def find_my_task(self, pod_uid, backend, timeout): regions = self._candidate_regions(backend) - log(f"[braket] searching for task tagged {TAG_KEY}={pod_uid} " - f"across regions {regions}") + log( + f"[braket] searching for task tagged {TAG_KEY}={pod_uid} " + f"across regions {regions}" + ) deadline = time.time() + timeout # SearchQuantumTasks has NO tags filter and is REGION-SCOPED: a task only # shows up in the region it was created in. We don't reliably know that @@ -151,10 +168,13 @@ def find_my_task(self, pod_uid, backend, timeout): device_arn = backend if backend.startswith("arn:aws:braket") else None filters = [] if device_arn: - filters = [{"name": "deviceArn", "operator": "EQUAL", - "values": [device_arn]}] + filters = [ + {"name": "deviceArn", "operator": "EQUAL", "values": [device_arn]} + ] clients = {} - denied = set() # regions that returned AccessDenied — skip them after the first hit + denied = ( + set() + ) # regions that returned AccessDenied — skip them after the first hit while time.time() < deadline: for region in regions: if region in denied: @@ -178,7 +198,9 @@ def find_my_task(self, pod_uid, backend, timeout): if TAG_KEY in tags: seen_tagged += 1 if tags.get(TAG_KEY) == pod_uid: - if match is None or str(t.get("createdAt", "")) > str(match.get("createdAt", "")): + if match is None or str(t.get("createdAt", "")) > str( + match.get("createdAt", "") + ): match = t next_token = resp.get("nextToken") if not next_token: @@ -191,22 +213,32 @@ def find_my_task(self, pod_uid, backend, timeout): # carried OUR tag key at all? Distinguishes "task not here yet" # from "task here but tag not landing". if seen: - log(f"[braket] {region}: {seen} task(s) seen, " - f"{seen_tagged} with a {TAG_KEY} tag, none matching {pod_uid}") + log( + f"[braket] {region}: {seen} task(s) seen, " + f"{seen_tagged} with a {TAG_KEY} tag, none matching {pod_uid}" + ) except Exception as e: msg = str(e) - if "AccessDenied" in msg or "not authorized" in msg or "explicit deny" in msg: + if ( + "AccessDenied" in msg + or "not authorized" in msg + or "explicit deny" in msg + ): # This account can't search Braket in this region (e.g. a # service control policy denies it). Skip it from now on # instead of erroring every poll cycle. denied.add(region) - log(f"[braket] region {region} not accessible " - f"(access denied) — skipping it for the rest of this search") + log( + f"[braket] region {region} not accessible " + f"(access denied) — skipping it for the rest of this search" + ) else: log(f"[braket] search error in {region} (will retry): {e}") if len(denied) == len(regions): - log("[braket] ERROR: every candidate region was access-denied — " - "set FLUENCE_BRAKET_REGIONS to a region this account can search") + log( + "[braket] ERROR: every candidate region was access-denied — " + "set FLUENCE_BRAKET_REGIONS to a region this account can search" + ) return None time.sleep(10) log("[braket] task discovery timed out") @@ -214,8 +246,10 @@ def find_my_task(self, pod_uid, backend, timeout): def _aws_task(self, task: BraketTask): import asyncio + asyncio.set_event_loop(asyncio.new_event_loop()) from braket.aws import AwsQuantumTask + return AwsQuantumTask(arn=task.arn) def is_ready_to_ungate(self, task: BraketTask, position=None) -> bool: @@ -249,4 +283,4 @@ def job_id(self, task: BraketTask) -> str: PROVIDER = BraketProvider() -register(PROVIDER) \ No newline at end of file +register(PROVIDER) diff --git a/python/fluence/providers/qrmi.py b/python/fluence/providers/qrmi.py new file mode 100644 index 0000000..4853aa0 --- /dev/null +++ b/python/fluence/providers/qrmi.py @@ -0,0 +1,274 @@ +""" +The QRMI provider. QRMI gives one API over several vendors, so this provider +has no vendor branches. The vendor comes from the resource attributes the +webhook injects, FLUXION_QRMI_TYPE picks the QRMI resource type and +FLUXION_BACKEND names the backend. + +QRMI has no way to search for a task by tag, so the interceptor writes the job +id and the session to the handoff file and the sidecar reads it from there. +""" + +from __future__ import annotations + +import os +import time + +from fluence.providers.base import Provider, Task, log, register +from fluence import handoff + +# qrmi_type values whose ResourceType name cannot be derived from them +_RESOURCE_TYPE_ALIASES = { + "direct-access": "IBMQuantumSystem", # legacy name for ibm-quantum-system + "ibm-direct-access": "IBMQuantumSystem", +} + +# kept upper case when deriving a ResourceType name +_ACRONYMS = {"ibm", "iqm", "qrmi", "id", "api", "hpc"} + + +class QRMITask(Task): + def __init__( + self, + job_id: str, + backend: str, + resource_type: str, + session_id: "str | None" = None, + tier: "str | None" = None, + ): + self.job_id = job_id + self.backend = backend + self.resource_type = resource_type + self.session_id = session_id + self.tier = tier + self._resource = None + + +def _pascal(qrmi_type: str, acronyms=True) -> str: + parts = [p for p in qrmi_type.replace("_", "-").split("-") if p] + out = [] + for p in parts: + if acronyms and p.lower() in _ACRONYMS: + out.append(p.upper()) + else: + out.append(p[:1].upper() + p[1:]) + return "".join(out) + + +def _resource_type_enum(qrmi_type: str): + """The QRMI ResourceType for a qrmi_type string. An alias first, then the + name derived from the string. Falls back to the first member with a warning + rather than failing on a QRMI version we have not seen.""" + from qrmi import ResourceType + + q = (qrmi_type or "").lower() + candidates = [] + if q in _RESOURCE_TYPE_ALIASES: + candidates.append(_RESOURCE_TYPE_ALIASES[q]) + candidates.append(_pascal(q, acronyms=True)) + candidates.append(_pascal(q, acronyms=False)) + for name in candidates: + rt = getattr(ResourceType, name, None) + if rt is not None: + return rt + log( + f"[qrmi] WARNING: no ResourceType matched {qrmi_type!r} " + f"(tried {candidates}); falling back to a default member" + ) + members = [v for k, v in vars(ResourceType).items() if not k.startswith("_")] + return members[0] if members else None + + +def _status_name(status) -> str: + """The upper case name of a QRMI TaskStatus, whatever type it arrives as.""" + for attr in ("name", "value"): + v = getattr(status, attr, None) + if isinstance(v, str) and v: + return v.upper() + s = str(status) + return s.rsplit(".", 1)[-1].upper() + + +# the task has left the queue or already finished +_READY_STATES = { + "RUNNING", + "COMPLETED", + "COMPLETE", + "DONE", + "FAILED", + "CANCELLED", + "CANCELED", +} + + +class QRMIProvider(Provider): + name = "qrmi" + + # the interceptor half runs in the user container + + def install_interceptor(self, pod_uid: str) -> bool: + try: + import qrmi + + QuantumResource = qrmi.QuantumResource + except Exception: + return False + + backend_hint = os.environ.get("FLUXION_BACKEND", "") + published = {"session_id": None, "tier": None} + + # acquire records the session and tier and otherwise behaves as before + if hasattr(QuantumResource, "acquire") and not getattr( + QuantumResource.acquire, "_fluence_patched", False + ): + original_acquire = QuantumResource.acquire + + def patched_acquire(self, *args, **kwargs): + try: + token = original_acquire(self, *args, **kwargs) + except Exception: + published["tier"] = "free" + log("[qrmi] acquire not permitted, continuing submit only") + raise + published["session_id"] = str(token) if token is not None else None + published["tier"] = "premium" + log("[qrmi] acquired a session") + return token + + patched_acquire._fluence_patched = True + QuantumResource.acquire = patched_acquire + + # task_start publishes the job id for the sidecar + if hasattr(QuantumResource, "task_start") and not getattr( + QuantumResource.task_start, "_fluence_patched", False + ): + original_task_start = QuantumResource.task_start + + def patched_task_start(self, *args, **kwargs): + job_id = original_task_start(self, *args, **kwargs) + try: + handoff.publish( + job_id=str(job_id), + session_id=published["session_id"], + backend=backend_hint or _resource_name(self), + pod_uid=pod_uid, + tier=published["tier"], + ) + log( + f"[qrmi] published job id {job_id} to handoff " + f"(tier={published['tier']})" + ) + except Exception as e: + log(f"[qrmi] WARNING: could not publish handoff: {e}") + return job_id + + patched_task_start._fluence_patched = True + QuantumResource.task_start = patched_task_start + + # a scout keeps its session until the sidecar says the workers are done + if hasattr(QuantumResource, "release") and not getattr( + QuantumResource.release, "_fluence_patched", False + ): + original_release = QuantumResource.release + + def patched_release(self, *args, **kwargs): + if os.environ.get("FLUENCE_COORDINATION_ROLE") == "scout": + timeout = int( + os.environ.get("FLUENCE_SESSION_HOLD_TIMEOUT", "0") or 0 + ) + log("[qrmi] holding the session until the worker group is done") + if handoff.wait_workers_done(timeout, log=log): + log("[qrmi] worker group done, releasing the session") + else: + log( + f"[qrmi] WARNING: hold timed out after {timeout}s, releasing anyway" + ) + return original_release(self, *args, **kwargs) + + patched_release._fluence_patched = True + QuantumResource.release = patched_release + + return True + + # the sidecar half runs in the sidecar container + + def matches(self, attrs: "dict[str, str]") -> bool: + # an explicit transport attribute wins + transport = (attrs.get("transport") or "").lower() + if transport: + return transport == "qrmi" + # otherwise any qrmi_type that is not braket is ours + qrmi_type = (attrs.get("qrmi_type") or "").lower() + if qrmi_type: + return not qrmi_type.startswith("braket") + return False + + def find_my_task(self, pod_uid, backend, timeout): + log(f"[qrmi] waiting for the scout job id in the handoff (timeout {timeout}s)") + record = handoff.wait(timeout, log=log) + if record is None: + log("[qrmi] no handoff appeared, task not discovered") + return None + if record.get("pod_uid") and pod_uid and record["pod_uid"] != pod_uid: + log( + f"[qrmi] WARNING: handoff pod_uid {record['pod_uid']!r} is not {pod_uid!r}, using it anyway" + ) + qrmi_type = os.environ.get("FLUXION_QRMI_TYPE", "") + task = QRMITask( + job_id=record["job_id"], + backend=record.get("backend") or backend, + resource_type=qrmi_type, + session_id=record.get("session_id"), + tier=record.get("tier"), + ) + log( + f"[qrmi] discovered job {task.job_id} on {task.backend} " + f"(type={qrmi_type!r}, tier={task.tier}, session={task.session_id})" + ) + return task + + def _qrmi_resource(self, task: QRMITask): + """A QRMI handle for status polling. Reading status needs no acquire.""" + if task._resource is None: + import qrmi + + rt = _resource_type_enum(task.resource_type) + task._resource = qrmi.QuantumResource(task.backend, rt) + return task._resource + + def _status(self, task: QRMITask) -> "str | None": + try: + status = self._qrmi_resource(task).task_status(task.job_id) + return _status_name(status) + except Exception as e: + log(f"[qrmi] task_status error (will retry): {e}") + return None + + def is_ready_to_ungate(self, task: QRMITask) -> bool: + name = self._status(task) + return name in _READY_STATES if name else False + + def queue_position(self, task: QRMITask): + # QRMI has no queue position, so report 1 once the task runs and None before + name = self._status(task) + if name and name in _READY_STATES: + return 1 + return None + + def job_id(self, task: QRMITask) -> str: + return task.job_id + + def session_id(self, task: QRMITask): + return task.session_id + + +def _resource_name(resource) -> str: + """The backend name from a QRMI handle, when FLUXION_BACKEND is not set.""" + for attr in ("name", "resource_id", "backend", "id"): + v = getattr(resource, attr, None) + if isinstance(v, str) and v: + return v + return "" + + +PROVIDER = QRMIProvider() +register(PROVIDER) diff --git a/python/fluence/sidecar.py b/python/fluence/sidecar.py index 1924d8a..7a2a287 100644 --- a/python/fluence/sidecar.py +++ b/python/fluence/sidecar.py @@ -1,25 +1,23 @@ """ -fluence.sidecar — provider-agnostic quantum coordination sidecar main loop. - -Injected by the Fluence webhook into the one-off SUBMITTER pod (gang + submitter -model — there is no leader/worker split). Resolves its vendor at runtime from the -backend annotation, discovers the task the user application submitted (tagged by -the interceptor), polls readiness, and either ungates the gated GANG group (gang -mode) or just logs the queue-position series (observe-only mode). - -Entry point: `fluence-sidecar` console script (see pyproject.toml) -> main(). - -Environment (injected by the Fluence webhook): - FLUENCE_POD_UID UID of this pod (matches interceptor tag) - FLUENCE_NAMESPACE Kubernetes namespace - FLUENCE_GANG_GROUP group label of the gated gang to ungate - FLUENCE_GATED_PODS optional explicit comma-separated gang pod names - FLUENCE_OBSERVE "true" for observe-only telemetry mode - FLUXION_BACKEND / FLUXION_VENDOR scheduler-chosen backend / vendor - FLUENCE_TASK_DISCOVERY_TIMEOUT seconds to wait for discovery (default 300) +The coordination sidecar. The webhook adds it to the scout pod of a shared +quantum gang. It resolves the vendor provider from the injected FLUXION_* +environment, finds the task the scout submitted, polls until the task is about +to run, ungates the workers with the job id and the session, then waits for +the workers to finish so the scout keeps its session until they are done. In +observe mode it only logs the queue position. + +Environment, all set by the webhook + FLUENCE_POD_UID uid of this pod + FLUENCE_NAMESPACE namespace + FLUENCE_GANG_GROUP group label of the workers to ungate + FLUENCE_GATED_PODS optional explicit list of worker pod names + FLUENCE_OBSERVE true for observe mode + FLUXION_BACKEND the backend the scheduler chose + FLUENCE_TASK_DISCOVERY_TIMEOUT seconds to wait for the task (default 300) FLUENCE_POLL_INTERVAL seconds between polls (default 30) - FLUENCE_UNGATE_POSITION ungate at this queue position or closer - (default 1, next in line) + FLUENCE_UNGATE_POSITION ungate at this queue position or closer (default 1) + FLUENCE_WORKER_TIMEOUT seconds to wait for the workers after ungating, + 0 waits without limit (default 0) """ from __future__ import annotations @@ -30,8 +28,14 @@ from fluence.providers import resolve_from_env from fluence.providers.base import log, ungate_position -from fluence.ungate import ungate_pods, gated_pods_from_env, namespace_from_env, wait_for_gated_pods - +from fluence import handoff +from fluence.ungate import ( + ungate_pods, + gated_pods_from_env, + namespace_from_env, + wait_for_gated_pods, + wait_for_pods_done, +) def _poll(provider, task, poll_interval, ungate, position=1): @@ -56,14 +60,24 @@ def _poll(provider, task, poll_interval, ungate, position=1): time.sleep(poll_interval) +def _hold_for_workers(namespace, gang_group, pod_name): + """Keep this pod, and with it the scout's session and its allocation, + until the worker group has finished. Then write the marker the scout's + release() waits for.""" + timeout = int(os.environ.get("FLUENCE_WORKER_TIMEOUT", 0)) + log("holding until the worker group finishes") + wait_for_pods_done(namespace, gang_group, exclude=pod_name, timeout=timeout) + if handoff.mark_workers_done(): + log("worker group finished, the scout may release its session") + else: + log("WARNING: could not write the workers done marker") + + def main(): pod_uid = os.environ.get("FLUENCE_POD_UID", "") pod_name = os.environ.get("FLUENCE_POD_NAME", "") group = os.environ.get("FLUENCE_GROUP", "") - # Gang + submitter model: this sidecar runs in the one-off SUBMITTER pod - # (its own group-of-one, -submitter). The gated workload it must ungate - # is the GANG group, named by FLUENCE_GANG_GROUP (set by the webhook). There - # is no leader/worker split and no -workers subgroup. + # the workers to ungate are the gang named by FLUENCE_GANG_GROUP gang_group = os.environ.get("FLUENCE_GANG_GROUP", "") backend = os.environ.get("FLUXION_BACKEND", "") observe = os.environ.get("FLUENCE_OBSERVE", "").lower() == "true" @@ -74,9 +88,11 @@ def main(): namespace = namespace_from_env() - log("starting fluence quantum submitter sidecar") - log(f" pod_uid={pod_uid} namespace={namespace} group={group} " - f"gang_group={gang_group} backend={backend} observe={observe}") + log("starting the fluence sidecar") + log( + f" pod_uid={pod_uid} namespace={namespace} group={group} " + f"gang_group={gang_group} backend={backend} observe={observe}" + ) provider = resolve_from_env() if provider is None: @@ -88,44 +104,58 @@ def main(): if task is None: log("ERROR: could not discover quantum task") if not observe: - # Fail open: ungate the gang so it is not stranded forever. - ungate_pods(wait_for_gated_pods(namespace, gang_group, exclude=pod_name, - timeout=ungate_timeout), - "", namespace) + # fail open so the workers are not stranded + ungate_pods( + wait_for_gated_pods( + namespace, gang_group, exclude=pod_name, timeout=ungate_timeout + ), + "", + namespace, + ) + _hold_for_workers(namespace, gang_group, pod_name) sys.exit(1) job_id = provider.job_id(task) log(f"discovered task, job_id={job_id}") - ready = _poll(provider, task, poll_interval, ungate=not observe, - position=ungate_at) + ready = _poll(provider, task, poll_interval, ungate=not observe, position=ungate_at) if observe: log("observe-only run complete") sys.exit(0 if ready else 1) if not ready: - # fail open like a discovery failure does, so the gang is not stranded. - # The pods find no result and exit, and we exit non zero so it shows - log("ERROR: ungating anyway so the gang is not stranded, but the task " - "produced no result") - ungate_pods(gated_pods_from_env() or wait_for_gated_pods( - namespace, gang_group, exclude=pod_name, timeout=ungate_timeout), - job_id, namespace) + # fail open so the workers are not stranded. They find no result and exit + log( + "ERROR: ungating anyway so the gang is not stranded, but the task " + "produced no result" + ) + ungate_pods( + gated_pods_from_env() + or wait_for_gated_pods( + namespace, gang_group, exclude=pod_name, timeout=ungate_timeout + ), + job_id, + namespace, + ) + _hold_for_workers(namespace, gang_group, pod_name) sys.exit(1) - # Ungate the gang: discover the gated pods in the gang group and remove their - # gate, stamping the job-id so each can fetch results by id. The gang pods are - # created up front (Job/Deployment), so they are present by submit time. + # ungate the workers, stamping the job id and the session first gated_pods = gated_pods_from_env() or wait_for_gated_pods( - namespace, gang_group, exclude=pod_name, timeout=ungate_timeout) + namespace, gang_group, exclude=pod_name, timeout=ungate_timeout + ) log(f"ungating {len(gated_pods)} gang pod(s): {gated_pods}") - n_ok = ungate_pods(gated_pods, job_id, namespace) + session_id = provider.session_id(task) + n_ok = ungate_pods(gated_pods, job_id, namespace, session_id=session_id) if n_ok == len(gated_pods): log(f"done — {n_ok} gang pod(s) ungated") else: - log(f"WARNING: ungated only {n_ok}/{len(gated_pods)} gang pod(s) — see errors above") + log( + f"WARNING: ungated only {n_ok}/{len(gated_pods)} gang pod(s) — see errors above" + ) + _hold_for_workers(namespace, gang_group, pod_name) if __name__ == "__main__": - main() \ No newline at end of file + main() diff --git a/python/fluence/sitecustomize.py b/python/fluence/sitecustomize.py index 519fe00..b7db9b7 100644 --- a/python/fluence/sitecustomize.py +++ b/python/fluence/sitecustomize.py @@ -9,4 +9,5 @@ import fluence.interceptor # noqa: F401 (import side-effect installs hooks) except Exception as _e: # pragma: no cover import sys + print(f"[fluence] interceptor skipped: {_e}", file=sys.stderr, flush=True) diff --git a/python/fluence/stage.py b/python/fluence/stage.py index 70a66b5..16358a8 100644 --- a/python/fluence/stage.py +++ b/python/fluence/stage.py @@ -37,7 +37,8 @@ def stage(dest: str) -> None: if os.path.exists(pkg_dst): shutil.rmtree(pkg_dst) shutil.copytree( - pkg_src, pkg_dst, + pkg_src, + pkg_dst, ignore=shutil.ignore_patterns("__pycache__", "*.pyc", "tests"), ) @@ -51,7 +52,9 @@ def stage(dest: str) -> None: def main(argv=None): argv = argv if argv is not None else sys.argv[1:] - dest = argv[0] if argv else os.environ.get("FLUENCE_STAGE_DIR", "/opt/fluence-staged") + dest = ( + argv[0] if argv else os.environ.get("FLUENCE_STAGE_DIR", "/opt/fluence-staged") + ) stage(dest) diff --git a/python/fluence/ungate.py b/python/fluence/ungate.py index a40e662..8cdfd5b 100644 --- a/python/fluence/ungate.py +++ b/python/fluence/ungate.py @@ -17,6 +17,7 @@ from fluence.providers.base import log JOB_ID_ANNOTATION = "fluence.flux-framework.org/quantum-job-id" +SESSION_ID_ANNOTATION = "fluence.flux-framework.org/quantum-session-id" QUANTUM_GATE_NAME = "quantum.braket/ready" PRIORITY_CLASS = "fluence-quantum-classical" @@ -28,11 +29,12 @@ def kubectl(args): return result.stdout.strip() -def ungate_pods(gated_pods, job_id, namespace): +def ungate_pods(gated_pods, job_id, namespace, session_id=None): """ For each gated worker pod: - 1. Stamp the vendor-neutral job-id annotation so the worker can locate - the quantum result. + 1. Stamp the job id annotation so the worker can locate the quantum + result, and the session id when the scout has one, so the worker + can submit into the same session. 2. Remove the scheduling gate so the pod can be scheduled. NOTE: priorityClassName is NOT set here — it is immutable after pod creation @@ -50,22 +52,47 @@ def ungate_pods(gated_pods, job_id, namespace): continue log(f"ungating pod: {pod_name}") + notes = [] if job_id: + notes.append(f"{JOB_ID_ANNOTATION}={job_id}") + if session_id: + notes.append(f"{SESSION_ID_ANNOTATION}={session_id}") + if notes: try: - kubectl(["annotate", "pod", pod_name, "-n", namespace, - f"{JOB_ID_ANNOTATION}={job_id}", "--overwrite"]) - log(f" patched job id onto {pod_name}: {job_id}") + kubectl( + [ + "annotate", + "pod", + pod_name, + "-n", + namespace, + *notes, + "--overwrite", + ] + ) + log(f" patched onto {pod_name}: {' '.join(notes)}") except RuntimeError as e: log(f" WARNING: could not annotate {pod_name}: {e}") else: log(f" WARNING: no job id to patch onto {pod_name}") - patch = json.dumps([ - {"op": "remove", "path": "/spec/schedulingGates/0"}, - ]) + patch = json.dumps( + [ + {"op": "remove", "path": "/spec/schedulingGates/0"}, + ] + ) try: - kubectl(["patch", "pod", pod_name, "-n", namespace, - "--type=json", f"-p={patch}"]) + kubectl( + [ + "patch", + "pod", + pod_name, + "-n", + namespace, + "--type=json", + f"-p={patch}", + ] + ) log(f" removed gate from {pod_name}") ok += 1 except RuntimeError as e: @@ -74,8 +101,11 @@ def ungate_pods(gated_pods, job_id, namespace): def gated_pods_from_env(): - return [p.strip() for p in os.environ.get("FLUENCE_GATED_PODS", "").split(",") - if p.strip()] + return [ + p.strip() + for p in os.environ.get("FLUENCE_GATED_PODS", "").split(",") + if p.strip() + ] GROUP_LABEL = "fluence.flux-framework.org/group" @@ -94,15 +124,23 @@ def discover_gated_pods(namespace, group, exclude=""): if not group: return [] try: - out = kubectl([ - "get", "pods", "-n", namespace, - "-l", f"{GROUP_LABEL}={group}", - "-o", "json", - ]) + out = kubectl( + [ + "get", + "pods", + "-n", + namespace, + "-l", + f"{GROUP_LABEL}={group}", + "-o", + "json", + ] + ) except RuntimeError as e: log(f"could not list group pods: {e}") return [] import json as _json + names = [] for item in _json.loads(out).get("items", []): name = item["metadata"]["name"] @@ -133,5 +171,36 @@ def wait_for_gated_pods(namespace, group, exclude="", timeout=120, interval=3): return found +def pods_done(namespace, group, exclude=""): + """True when every pod in the group other than exclude has finished.""" + out = kubectl( + ["get", "pods", "-n", namespace, "-l", f"{GROUP_LABEL}={group}", "-o", "json"] + ) + others = [ + i for i in json.loads(out).get("items", []) if i["metadata"]["name"] != exclude + ] + return all( + i.get("status", {}).get("phase") in ("Succeeded", "Failed") for i in others + ) + + +def wait_for_pods_done(namespace, group, exclude="", timeout=0, interval=10): + """Wait until the other pods in the group have all finished. A timeout of + zero waits without limit. Returns True when they are done.""" + if not group: + return True + deadline = time.time() + timeout if timeout > 0 else None + while True: + try: + if pods_done(namespace, group, exclude): + return True + except Exception as e: + log(f"could not list group pods, will retry: {e}") + if deadline is not None and time.time() >= deadline: + log("WARNING: timed out waiting for the worker group to finish") + return False + time.sleep(interval) + + def namespace_from_env(): - return os.environ.get("FLUENCE_NAMESPACE", "default") \ No newline at end of file + return os.environ.get("FLUENCE_NAMESPACE", "default") diff --git a/python/pyproject.toml b/python/pyproject.toml index f48050e..4dfbbe7 100644 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -20,8 +20,8 @@ dependencies = [] [project.optional-dependencies] braket = ["amazon-braket-sdk", "boto3"] -# ibm = ["qiskit-ibm-runtime"] -all = ["amazon-braket-sdk", "boto3"] +qrmi = ["qrmi[ibm]"] +all = ["amazon-braket-sdk", "boto3", "qrmi[ibm]"] [project.scripts] fluence-sidecar = "fluence.sidecar:main" diff --git a/python/tests/test_fluence.py b/python/tests/test_fluence.py index 5f970bc..4e6c4e5 100644 --- a/python/tests/test_fluence.py +++ b/python/tests/test_fluence.py @@ -12,22 +12,28 @@ def test_braket_registered(): from fluence.providers import all_providers + assert "braket" in [p.name for p in all_providers()] def test_resolve_braket_by_vendor(): from fluence.providers import resolve + assert resolve({"vendor": "braket"}).name == "braket" def test_resolve_braket_by_arn_backend(): from fluence.providers import resolve - p = resolve({"arn": "arn:aws:braket:us-east-1::device/quantum-simulator/amazon/sv1"}) + + p = resolve( + {"arn": "arn:aws:braket:us-east-1::device/quantum-simulator/amazon/sv1"} + ) assert p is not None and p.name == "braket" def test_resolve_unknown_returns_none(): from fluence.providers import resolve + assert resolve({"vendor": "nope", "backend": "nope"}) is None @@ -35,17 +41,20 @@ def test_interceptor_failsoft_without_sdk(): # Importing the interceptor must not raise even though braket is absent. import importlib import fluence.interceptor + importlib.reload(fluence.interceptor) # re-run install(); must not raise def test_braket_install_interceptor_failsoft(): from fluence.providers.braket import PROVIDER + # No braket SDK installed -> returns False, never raises. assert PROVIDER.install_interceptor("uid") is False def test_job_id_is_arn(): from fluence.providers.braket import PROVIDER, BraketTask + arn = "arn:aws:braket:us-east-1::quantum-task/abc-123" assert PROVIDER.job_id(BraketTask(arn)) == arn @@ -88,22 +97,26 @@ def test_staged_sitecustomize_runs_interceptor(): "FLUENCE_POD_UID": "pod-xyz", "PYTHONPATH": staged + os.pathsep + fakesdk, } - out = subprocess.run([sys.executable, app], env=env, - capture_output=True, text=True) + out = subprocess.run( + [sys.executable, app], env=env, capture_output=True, text=True + ) assert "fluence-pod-uid" in out.stdout, out.stdout + out.stderr assert "pod-xyz" in out.stdout, out.stdout + out.stderr - def test_braket_matches_by_qrmi_type_and_vendor(): # The authoritative routing key is qrmi_type (braket-*), which holds even # when the hardware vendor is quera/rigetti/iqm rather than amazon. from fluence.providers import resolve + assert resolve({"vendor": "amazon", "backend": "sv1"}).name == "braket" # aquila: vendor=quera but qrmi_type=braket-ahs must still route to braket. - assert resolve({"vendor": "quera", "backend": "aquila", - "qrmi_type": "braket-ahs"}).name == "braket" - + assert ( + resolve( + {"vendor": "quera", "backend": "aquila", "qrmi_type": "braket-ahs"} + ).name + == "braket" + ) def test_find_my_task_matches_tag_client_side(): @@ -116,23 +129,35 @@ def test_find_my_task_matches_tag_client_side(): class FakeClient: def __init__(self): self.calls = 0 + def search_quantum_tasks(self, filters=None, maxResults=None, nextToken=None): - assert isinstance(filters, list) # empty list is valid + assert isinstance(filters, list) # empty list is valid # Two pages, to exercise nextToken paging. if nextToken is None: - return {"quantumTasks": [ - {"quantumTaskArn": "arn:other", "createdAt": "2026-01-01T00:00:00Z", - "tags": {TAG_KEY: "someone-else"}}, - ], "nextToken": "page2"} - return {"quantumTasks": [ - {"quantumTaskArn": "arn:mine", "createdAt": "2026-01-02T00:00:00Z", - "tags": {TAG_KEY: "me-123"}}, - ]} # no nextToken -> last page + return { + "quantumTasks": [ + { + "quantumTaskArn": "arn:other", + "createdAt": "2026-01-01T00:00:00Z", + "tags": {TAG_KEY: "someone-else"}, + }, + ], + "nextToken": "page2", + } + return { + "quantumTasks": [ + { + "quantumTaskArn": "arn:mine", + "createdAt": "2026-01-02T00:00:00Z", + "tags": {TAG_KEY: "me-123"}, + }, + ] + } # no nextToken -> last page p = BraketProvider() fake = FakeClient() - p._client = lambda backend: fake # bypass real boto3 (legacy path) - p._region_client = lambda region: fake # multi-region search uses this + p._client = lambda backend: fake # bypass real boto3 (legacy path) + p._region_client = lambda region: fake # multi-region search uses this task = p.find_my_task("me-123", "sv1", timeout=5) assert task is not None and task.arn == "arn:mine" diff --git a/python/tests/test_hold.py b/python/tests/test_hold.py new file mode 100644 index 0000000..53196ff --- /dev/null +++ b/python/tests/test_hold.py @@ -0,0 +1,215 @@ +""" +Tests for holding the scout until the worker group is done. The sidecar +stamps the session at ungate, waits for the group, and writes a marker, and a +scout's release() waits for that marker. + +Run: PYTHONPATH=python python3 python/tests/test_hold.py +""" + +import json +import os +import sys +import tempfile +import time +import types + + +def _fake_qrmi(released): + qrmi = types.ModuleType("qrmi") + + class QuantumResource: + def __init__(self, *a, **k): + pass + + def acquire(self): + return "tok" + + def task_start(self, *a, **k): + return "job" + + def release(self, lock=None): + released.append(lock) + + qrmi.QuantumResource = QuantumResource + qrmi.ResourceType = types.SimpleNamespace( + QiskitRuntimeService="QiskitRuntimeService" + ) + qrmi.Payload = types.SimpleNamespace() + qrmi.TaskStatus = types.SimpleNamespace() + sys.modules["qrmi"] = qrmi + return QuantumResource + + +def _clean(): + for k in ( + "FLUENCE_HANDOFF_DIR", + "FLUENCE_COORDINATION_ROLE", + "FLUENCE_SESSION_HOLD_TIMEOUT", + ): + os.environ.pop(k, None) + sys.modules.pop("qrmi", None) + + +def test_marker_roundtrip(): + from fluence import handoff + + with tempfile.TemporaryDirectory() as d: + p = os.path.join(d, "workers-done") + assert not handoff.workers_done(p) + assert handoff.mark_workers_done(p) + assert handoff.workers_done(p) + assert handoff.wait_workers_done(1, interval=0.1, path=p) + + +def test_wait_times_out_without_marker(): + from fluence import handoff + + with tempfile.TemporaryDirectory() as d: + assert ( + handoff.wait_workers_done( + 0.3, interval=0.1, path=os.path.join(d, "workers-done") + ) + is False + ) + + +def test_ungate_stamps_job_id_and_session(): + from fluence import ungate + + calls = [] + orig = ungate.kubectl + ungate.kubectl = lambda args: calls.append(args) or "" + try: + assert ungate.ungate_pods(["w-1"], "job-1", "ns", session_id="sess-9") == 1 + finally: + ungate.kubectl = orig + annotate = [c for c in calls if c[0] == "annotate"][0] + assert f"{ungate.JOB_ID_ANNOTATION}=job-1" in annotate + assert f"{ungate.SESSION_ID_ANNOTATION}=sess-9" in annotate + assert any(c[0] == "patch" for c in calls) + + +def test_ungate_without_session_stamps_only_job_id(): + from fluence import ungate + + calls = [] + orig = ungate.kubectl + ungate.kubectl = lambda args: calls.append(args) or "" + try: + ungate.ungate_pods(["w-1"], "job-1", "ns") + finally: + ungate.kubectl = orig + annotate = [c for c in calls if c[0] == "annotate"][0] + assert not any(ungate.SESSION_ID_ANNOTATION in a for a in annotate) + + +def test_wait_for_pods_done_ignores_the_scout(): + from fluence import ungate + + rounds = iter( + [["Running", "Running"], ["Succeeded", "Running"], ["Succeeded", "Failed"]] + ) + + def fake(args): + items = [{"metadata": {"name": "scout"}, "status": {"phase": "Running"}}] + items += [ + {"metadata": {"name": f"w-{i}"}, "status": {"phase": p}} + for i, p in enumerate(next(rounds)) + ] + return json.dumps({"items": items}) + + orig = ungate.kubectl + ungate.kubectl = fake + try: + assert ( + ungate.wait_for_pods_done("ns", "g", exclude="scout", timeout=5, interval=0) + is True + ) + finally: + ungate.kubectl = orig + + +def test_wait_for_pods_done_times_out(): + from fluence import ungate + + orig = ungate.kubectl + ungate.kubectl = lambda args: json.dumps( + {"items": [{"metadata": {"name": "w-0"}, "status": {"phase": "Running"}}]} + ) + try: + assert ungate.wait_for_pods_done("ns", "g", timeout=0.2, interval=0.05) is False + finally: + ungate.kubectl = orig + + +def test_scout_release_waits_for_marker(): + from fluence.providers.qrmi import QRMIProvider + from fluence import handoff + + released = [] + with tempfile.TemporaryDirectory() as d: + os.environ["FLUENCE_HANDOFF_DIR"] = d + os.environ["FLUENCE_COORDINATION_ROLE"] = "scout" + os.environ["FLUENCE_SESSION_HOLD_TIMEOUT"] = "5" + QR = _fake_qrmi(released) + try: + assert QRMIProvider().install_interceptor("uid") is True + handoff.mark_workers_done() + t0 = time.time() + QR().release("tok") + assert released == ["tok"] + assert time.time() - t0 < 2 + finally: + _clean() + + +def test_scout_release_gives_up_after_timeout(): + from fluence.providers.qrmi import QRMIProvider + + released = [] + with tempfile.TemporaryDirectory() as d: + os.environ["FLUENCE_HANDOFF_DIR"] = d + os.environ["FLUENCE_COORDINATION_ROLE"] = "scout" + os.environ["FLUENCE_SESSION_HOLD_TIMEOUT"] = "1" + QR = _fake_qrmi(released) + try: + QRMIProvider().install_interceptor("uid") + t0 = time.time() + QR().release("tok") + assert released == ["tok"] + assert time.time() - t0 >= 1 + finally: + _clean() + + +def test_non_scout_release_passes_through(): + from fluence.providers.qrmi import QRMIProvider + + released = [] + with tempfile.TemporaryDirectory() as d: + os.environ["FLUENCE_HANDOFF_DIR"] = d + QR = _fake_qrmi(released) + try: + QRMIProvider().install_interceptor("uid") + t0 = time.time() + QR().release("tok") + assert released == ["tok"] + assert time.time() - t0 < 1 + finally: + _clean() + + +if __name__ == "__main__": + fns = [ + v for k, v in sorted(globals().items()) if k.startswith("test_") and callable(v) + ] + failed = 0 + for fn in fns: + try: + fn() + print(f"PASS {fn.__name__}") + except Exception as e: + failed += 1 + print(f"FAIL {fn.__name__}: {e}") + print(f"\n{len(fns) - failed}/{len(fns)} passed") + sys.exit(1 if failed else 0) diff --git a/python/tests/test_qrmi.py b/python/tests/test_qrmi.py new file mode 100644 index 0000000..ff9e4d0 --- /dev/null +++ b/python/tests/test_qrmi.py @@ -0,0 +1,397 @@ +""" +Tests for the QRMI provider. Like test_fluence.py, none of these require the +real qrmi SDK — they exercise registration, routing, the resource-type mapping, +the handoff channel, fail-soft interceptor behavior, the status->ready mapping, +and the free/paid (acquire) session capture using small fakes. + +The provider is vendor-agnostic: the tests deliberately cover more than one QRMI +vendor (IBM's qiskit-runtime-service AND Pasqal's pasqal-cloud) to prove nothing +is IBM-specific. + +Run: PYTHONPATH=python python3 python/tests/test_qrmi.py +""" + +import os +import sys +import types +import tempfile + +# ── registration & routing ─────────────────────────────────────────────────── + + +def test_qrmi_registered(): + from fluence.providers import all_providers + + assert "qrmi" in [p.name for p in all_providers()] + + +def test_resolve_qrmi_ibm_runtime_service(): + # The qrmi_type the Fluence resource config ships for IBM. + from fluence.providers import resolve + + assert resolve({"qrmi_type": "qiskit-runtime-service"}).name == "qrmi" + + +def test_resolve_qrmi_ibm_quantum_system_and_legacy(): + from fluence.providers import resolve + + assert resolve({"qrmi_type": "ibm-quantum-system"}).name == "qrmi" + assert resolve({"qrmi_type": "direct-access"}).name == "qrmi" # legacy alias + + +def test_resolve_qrmi_pasqal_proves_vendor_agnostic(): + # A DIFFERENT vendor (Pasqal) must route to the same provider with no code + # change — routing is data-driven from qrmi_type. + from fluence.providers import resolve + + assert resolve({"qrmi_type": "pasqal-cloud"}).name == "qrmi" + + +def test_resolve_qrmi_by_explicit_transport_marker(): + from fluence.providers import resolve + + # transport=qrmi is authoritative even without a qrmi_type. + assert resolve({"transport": "qrmi"}).name == "qrmi" + # transport=braket must NOT route to qrmi. + assert ( + resolve({"transport": "braket", "qrmi_type": "pasqal-cloud"}) is None + or resolve({"transport": "braket", "qrmi_type": "pasqal-cloud"}).name != "qrmi" + ) + + +def test_qrmi_and_braket_do_not_collide(): + from fluence.providers import resolve + + # Braket-reached devices carry a braket-* qrmi_type and must go to braket. + assert resolve({"qrmi_type": "braket-gate", "vendor": "amazon"}).name == "braket" + assert resolve({"qrmi_type": "braket-ahs", "vendor": "quera"}).name == "braket" + # QRMI-reached devices must go to qrmi. + assert resolve({"qrmi_type": "qiskit-runtime-service"}).name == "qrmi" + assert resolve({"qrmi_type": "pasqal-cloud"}).name == "qrmi" + + +def test_no_routing_signal_returns_none(): + from fluence.providers import resolve + + assert resolve({"vendor": "someone"}) is None + + +# ── resource-type mapping (data-driven, vendor-agnostic) ────────────────────── + + +def test_resource_type_derivation_covers_multiple_vendors(): + # _resource_type_enum derives the ResourceType member name from the qrmi_type + # string (with acronym awareness + a small alias table), so a new vendor is + # just a new qrmi_type in config. Verify with a fake ResourceType enum. + from fluence.providers import qrmi as qmod + + fake = types.SimpleNamespace( + QiskitRuntimeService="QiskitRuntimeService", + IBMQuantumSystem="IBMQuantumSystem", + PasqalCloud="PasqalCloud", + ) + fake_mod = types.ModuleType("qrmi") + fake_mod.ResourceType = fake + sys.modules["qrmi"] = fake_mod + try: + assert ( + qmod._resource_type_enum("qiskit-runtime-service") == "QiskitRuntimeService" + ) + assert qmod._resource_type_enum("ibm-quantum-system") == "IBMQuantumSystem" + assert qmod._resource_type_enum("direct-access") == "IBMQuantumSystem" # alias + assert qmod._resource_type_enum("pasqal-cloud") == "PasqalCloud" + finally: + sys.modules.pop("qrmi", None) + + +def test_pascal_transform_acronym_aware(): + from fluence.providers.qrmi import _pascal + + assert _pascal("qiskit-runtime-service") == "QiskitRuntimeService" + assert _pascal("ibm-quantum-system") == "IBMQuantumSystem" + assert _pascal("pasqal-cloud") == "PasqalCloud" + + +# ── handoff channel ─────────────────────────────────────────────────────────── + + +def test_handoff_publish_and_read_roundtrip(): + from fluence import handoff + + with tempfile.TemporaryDirectory() as d: + path = os.path.join(d, "task.json") + assert ( + handoff.publish( + "job-123", + session_id="sess-abc", + backend="ibm_fez", + pod_uid="pod-1", + tier="premium", + path=path, + ) + is True + ) + rec = handoff.read(path) + assert rec["job_id"] == "job-123" + assert rec["session_id"] == "sess-abc" + assert rec["tier"] == "premium" + + +def test_handoff_read_missing_returns_none(): + from fluence import handoff + + with tempfile.TemporaryDirectory() as d: + assert handoff.read(os.path.join(d, "nope.json")) is None + + +def test_handoff_wait_times_out_without_file(): + from fluence import handoff + + with tempfile.TemporaryDirectory() as d: + assert handoff.wait(1, interval=1, path=os.path.join(d, "task.json")) is None + + +def test_handoff_write_is_atomic_no_partial_file(): + from fluence import handoff + + with tempfile.TemporaryDirectory() as d: + path = os.path.join(d, "task.json") + handoff.publish("j", path=path) + leftovers = [f for f in os.listdir(d) if f != "task.json"] + assert leftovers == [], leftovers + + +# ── interceptor: fail-soft + free/paid session capture ──────────────────────── + + +def test_qrmi_install_interceptor_failsoft_without_qrmi(): + from fluence.providers.qrmi import PROVIDER + + assert "qrmi" not in sys.modules + assert PROVIDER.install_interceptor("uid") is False + + +def _install_fake_qrmi(acquire_impl, task_start_returns="job-xyz"): + qrmi = types.ModuleType("qrmi") + + class QuantumResource: + def __init__(self, *a, **k): + pass + + def acquire(self, *a, **k): + return acquire_impl() + + def task_start(self, *a, **k): + return task_start_returns + + qrmi.QuantumResource = QuantumResource + qrmi.ResourceType = types.SimpleNamespace( + QiskitRuntimeService="QiskitRuntimeService", + PasqalCloud="PasqalCloud", + ) + qrmi.Payload = types.SimpleNamespace() + qrmi.TaskStatus = types.SimpleNamespace() + sys.modules["qrmi"] = qrmi + return QuantumResource + + +def test_paid_tier_captures_session_and_publishes_job_id(): + from fluence.providers.qrmi import QRMIProvider + from fluence import handoff + + with tempfile.TemporaryDirectory() as d: + path = os.path.join(d, "task.json") + os.environ["FLUENCE_HANDOFF_DIR"] = d + os.environ["FLUXION_BACKEND"] = "ibm_fez" + QR = _install_fake_qrmi(acquire_impl=lambda: "session-token-123") + try: + prov = QRMIProvider() + assert prov.install_interceptor("pod-uid-9") is True + qr = QR() + token = qr.acquire() + assert token == "session-token-123" + assert qr.task_start() == "job-xyz" + rec = handoff.read(path) + assert rec["job_id"] == "job-xyz" + assert rec["session_id"] == "session-token-123" + assert rec["tier"] == "premium" + assert rec["pod_uid"] == "pod-uid-9" + finally: + sys.modules.pop("qrmi", None) + os.environ.pop("FLUENCE_HANDOFF_DIR", None) + os.environ.pop("FLUXION_BACKEND", None) + + +def test_free_tier_acquire_reraises_and_submit_still_publishes(): + from fluence.providers.qrmi import QRMIProvider + from fluence import handoff + + def denied(): + raise RuntimeError("acquire not permitted on the open plan") + + with tempfile.TemporaryDirectory() as d: + path = os.path.join(d, "task.json") + os.environ["FLUENCE_HANDOFF_DIR"] = d + QR = _install_fake_qrmi(acquire_impl=denied) + try: + prov = QRMIProvider() + assert prov.install_interceptor("pod-free") is True + qr = QR() + raised = False + try: + qr.acquire() + except RuntimeError: + raised = True + assert raised, "patched acquire() must re-raise on a free account" + assert qr.task_start() == "job-xyz" + rec = handoff.read(path) + assert rec["job_id"] == "job-xyz" + assert rec.get("session_id") is None + assert rec.get("tier") == "free" + finally: + sys.modules.pop("qrmi", None) + os.environ.pop("FLUENCE_HANDOFF_DIR", None) + + +# ── status -> readiness mapping ─────────────────────────────────────────────── + + +class _FakeStatusResource: + def __init__(self, statuses): + self._statuses = list(statuses) + + def task_status(self, job_id): + return self._statuses.pop(0) if self._statuses else "RUNNING" + + +def _qrmi_task(statuses): + from fluence.providers.qrmi import QRMIProvider, QRMITask + + prov = QRMIProvider() + task = QRMITask( + job_id="j", backend="ibm_fez", resource_type="qiskit-runtime-service" + ) + task._resource = _FakeStatusResource(statuses) + return prov, task + + +def test_queued_is_not_ready(): + prov, task = _qrmi_task(["QUEUED"]) + assert prov.is_ready_to_ungate(task) is False + + +def test_running_is_ready_and_position_one(): + prov, task = _qrmi_task(["RUNNING", "RUNNING"]) + assert prov.is_ready_to_ungate(task) is True + assert prov.queue_position(task) == 1 + + +def test_terminal_completed_is_ready(): + prov, task = _qrmi_task(["COMPLETED"]) + assert prov.is_ready_to_ungate(task) is True + + +def test_enum_like_status_is_normalized(): + class E: + def __init__(self, s): + self.s = s + + def __str__(self): + return self.s + + prov, task = _qrmi_task([E("TaskStatus.Running")]) + assert prov.is_ready_to_ungate(task) is True + + +def test_status_error_is_not_ready_not_raising(): + class Boom: + def task_status(self, job_id): + raise RuntimeError("transient") + + from fluence.providers.qrmi import QRMIProvider, QRMITask + + prov = QRMIProvider() + task = QRMITask( + job_id="j", backend="ibm_fez", resource_type="qiskit-runtime-service" + ) + task._resource = Boom() + assert prov.is_ready_to_ungate(task) is False + + +# ── job id / session id accessors ───────────────────────────────────────────── + + +def test_job_id_and_session_id_accessors(): + from fluence.providers.qrmi import QRMIProvider, QRMITask + + prov = QRMIProvider() + task = QRMITask( + job_id="job-1", + backend="ibm_fez", + resource_type="qiskit-runtime-service", + session_id="sess-1", + ) + assert prov.job_id(task) == "job-1" + assert prov.session_id(task) == "sess-1" + free = QRMITask( + job_id="job-2", backend="ibm_fez", resource_type="qiskit-runtime-service" + ) + assert prov.session_id(free) is None + + +# ── find_my_task via handoff ────────────────────────────────────────────────── + + +def test_find_my_task_reads_handoff_uses_injected_type(): + from fluence.providers.qrmi import QRMIProvider + from fluence import handoff + + with tempfile.TemporaryDirectory() as d: + os.environ["FLUENCE_HANDOFF_DIR"] = d + os.environ["FLUXION_QRMI_TYPE"] = "pasqal-cloud" # a non-IBM vendor + try: + handoff.publish( + "job-77", + session_id="s-77", + backend="FRESNEL", + pod_uid="pod-a", + tier="premium", + ) + prov = QRMIProvider() + task = prov.find_my_task("pod-a", "FRESNEL", timeout=5) + assert task is not None + assert task.job_id == "job-77" + assert task.backend == "FRESNEL" + assert task.resource_type == "pasqal-cloud" # from the injected env + assert task.session_id == "s-77" + finally: + os.environ.pop("FLUENCE_HANDOFF_DIR", None) + os.environ.pop("FLUXION_QRMI_TYPE", None) + + +def test_find_my_task_times_out_to_none(): + from fluence.providers.qrmi import QRMIProvider + + with tempfile.TemporaryDirectory() as d: + os.environ["FLUENCE_HANDOFF_DIR"] = d + try: + prov = QRMIProvider() + assert prov.find_my_task("pod-a", "ibm_fez", timeout=1) is None + finally: + os.environ.pop("FLUENCE_HANDOFF_DIR", None) + + +if __name__ == "__main__": + fns = [ + v for k, v in sorted(globals().items()) if k.startswith("test_") and callable(v) + ] + failed = 0 + for fn in fns: + try: + fn() + print(f"PASS {fn.__name__}") + except Exception as e: + failed += 1 + print(f"FAIL {fn.__name__}: {e}") + print(f"\n{len(fns) - failed}/{len(fns)} passed") + sys.exit(1 if failed else 0) diff --git a/test/e2e/quantum/setup.sh b/test/e2e/quantum/setup.sh index 57f375a..984c689 100644 --- a/test/e2e/quantum/setup.sh +++ b/test/e2e/quantum/setup.sh @@ -51,4 +51,4 @@ done [ "$ok" = 1 ] || fail "no node advertised fluxion.flux-framework.org/qpu after the add-on (device plugin not registering)" log "qpu advertised on at least one node" -log "quantum setup complete: qpu add-on installed, scheduler restarted, sidecar image=$IMAGE" \ No newline at end of file +log "quantum setup complete: qpu add-on installed, scheduler restarted, sidecar image=$IMAGE" From 6b1386a8eab40a4f4cbf2060d786320c3e08c9e2 Mon Sep 17 00:00:00 2001 From: vsoch Date: Sun, 18 Oct 2026 10:55:12 -0700 Subject: [PATCH 2/2] nit: grammar and punctuation Signed-off-by: vsoch --- .devcontainer/Dockerfile | 6 +- .github/workflows/e2e-suite.yaml | 10 +- .github/workflows/e2e-tests.yaml | 2 +- .github/workflows/sidecar-build-deploy.yaml | 6 +- .github/workflows/unit-tests.yaml | 2 +- Makefile | 6 +- README.md | 432 +++--------------- cmd/deviceplugin/main.go | 4 +- cmd/fluence/main.go | 2 +- cmd/recovery-probe/main.go | 12 +- cmd/webhook/main.go | 9 +- deploy/device-plugin.yaml | 2 +- deploy/fluence-pull-test.yaml | 24 +- deploy/fluence-resources-test.yaml | 16 +- deploy/fluence-resources.yaml | 14 +- deploy/fluence-test.yaml | 28 +- deploy/fluence.yaml | 27 +- docs/coordination-handler-design.md | 140 +++--- docs/handlers.md | 44 +- docs/qrmi-coordination.md | 146 ++---- docs/quantum-scheduling.md | 155 ++----- examples/podgroup.yaml | 2 +- .../test/e2e/gang/multi-gang-contention.yaml | 4 +- .../test/e2e/gang/multi-gang-requeue.yaml | 2 +- examples/test/e2e/gang/single-podgroup.yaml | 2 +- examples/test/e2e/quantum-pod-mock-2.yaml | 2 +- .../test/e2e/quantum/quantum-gang-pods.yaml | 6 +- .../test/e2e/quantum/quantum-pod-mock.yaml | 2 +- examples/test/jobspec-cpu.yaml | 2 +- examples/test/jobspec-qpu.yaml | 2 +- pkg/cluster/cluster.go | 12 +- pkg/cluster/resources.go | 42 +- pkg/cluster/resources_graph.go | 26 +- pkg/cluster/resources_graph_test.go | 14 +- pkg/cluster/resources_test.go | 8 +- pkg/deviceplugin/plugin.go | 16 +- pkg/fluence/fluence.go | 186 ++++---- pkg/fluence/fluence_test.go | 24 +- pkg/graph/allocation.go | 14 +- pkg/graph/allocation_test.go | 10 +- pkg/graph/graph.go | 16 +- pkg/graph/graph_cancel_test.go | 12 +- pkg/graph/run_test.go | 2 +- pkg/jgf/jgf.go | 8 +- pkg/jgf/jgf_test.go | 4 +- pkg/jobspec/jobspec.go | 4 +- pkg/placement/placement.go | 62 +-- pkg/placement/placement_test.go | 12 +- pkg/webhook/admission.go | 6 +- pkg/webhook/handler.go | 22 +- pkg/webhook/handlers/admission_test.go | 2 +- pkg/webhook/handlers/dependency.go | 34 +- pkg/webhook/handlers/fluxion.go | 2 +- pkg/webhook/handlers/gang.go | 20 +- pkg/webhook/handlers/gang_test.go | 6 +- pkg/webhook/handlers/handlers_test.go | 2 +- pkg/webhook/handlers/quantum.go | 90 ++-- pkg/webhook/handlers/quantum_test.go | 32 +- pkg/webhook/handlers/registry_test.go | 6 +- pkg/webhook/handlers/sidecar.go | 10 +- pkg/webhook/spec/spec.go | 4 +- pkg/webhook/webhook.go | 18 +- python/Dockerfile | 4 +- python/README.md | 28 +- python/fluence/__init__.py | 2 +- python/fluence/interceptor.py | 6 +- python/fluence/providers/__init__.py | 8 +- python/fluence/providers/base.py | 24 +- python/fluence/providers/braket.py | 36 +- python/fluence/sidecar.py | 4 +- python/fluence/sitecustomize.py | 2 +- python/fluence/stage.py | 10 +- python/fluence/ungate.py | 12 +- python/tests/test_fluence.py | 8 +- python/tests/test_qrmi.py | 10 +- test/e2e/03-restart-recovery.sh | 4 +- test/e2e/gang/01-classical-gang.sh | 6 +- test/e2e/gang/02-postfilter-rematch.sh | 14 +- test/e2e/gang/03-multi-gang.sh | 14 +- test/e2e/gang/04-requeue-on-capacity.sh | 28 +- test/e2e/lib.sh | 2 +- test/e2e/quantum/01-quantum-placement.sh | 4 +- test/e2e/quantum/02-sidecar-ungate.sh | 60 +-- test/e2e/quantum/03-gang-producer.sh | 70 --- test/e2e/quantum/03-gang-scout.sh | 70 +++ test/e2e/quantum/04-gang-env-contract.sh | 70 +-- test/e2e/quantum/setup.sh | 10 +- 87 files changed, 924 insertions(+), 1419 deletions(-) delete mode 100644 test/e2e/quantum/03-gang-producer.sh create mode 100644 test/e2e/quantum/03-gang-scout.sh diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile index e0cea68..b1678e2 100644 --- a/.devcontainer/Dockerfile +++ b/.devcontainer/Dockerfile @@ -11,7 +11,7 @@ ENV USER_UID=${USER_UID} ENV USER_GID=${USER_GID} USER root -# Install extra buildrequires for flux-sched: +# Install extra buildrequires for flux-sched. RUN sudo apt-get update && \ sudo apt-get -qq install -y --no-install-recommends \ libboost-graph-dev \ @@ -45,7 +45,7 @@ RUN sudo apt-get update \ # --- Rust (for building QRMI) -------------------------------------------- # Installed system-wide so it is available regardless of the active user. -# QRMI pins its toolchain via rust-toolchain.toml (currently 1.91); rustup will +# QRMI pins its toolchain via rust-toolchain.toml (currently 1.91). Rustup will # fetch that automatically during the build below. ENV RUSTUP_HOME=/usr/local/rustup \ CARGO_HOME=/usr/local/cargo \ @@ -55,7 +55,7 @@ RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs \ && chmod -R a+rwX /usr/local/rustup /usr/local/cargo # --- QRMI native library (the bindings' cgo dependency) ------------------ -# Pin QRMI_REF to a tag/commit for reproducibility if you like; defaults to main. +# Pin QRMI_REF to a tag/commit for reproducibility if you like. Defaults to main. ARG QRMI_REPO=https://github.com/qiskit-community/qrmi ARG QRMI_REF=main RUN git clone ${QRMI_REPO} /opt/qrmi \ diff --git a/.github/workflows/e2e-suite.yaml b/.github/workflows/e2e-suite.yaml index deeb828..2c2b0c7 100644 --- a/.github/workflows/e2e-suite.yaml +++ b/.github/workflows/e2e-suite.yaml @@ -1,9 +1,9 @@ -# Reusable e2e workflow (workflow_call): shared setup (build image, kind, deploy -# fluence base), then run ONE test suite — a directory under test/e2e/. The -# suite's tests are DISCOVERED (every NN-*.sh, run in sorted order); adding a test +# Reusable e2e workflow (workflow_call). Shared setup (build image, kind, deploy +# fluence base), then run ONE test suite, a directory under test/e2e/. The +# suite's tests are DISCOVERED (every NN-*.sh, run in sorted order). Adding a test # is just dropping a file in the directory, no workflow edit. If the suite needs # special preparation it provides a setup.sh in its directory, which is run before -# the tests (the gang suite has none; the quantum suite installs the qpu add-on). +# the tests (the gang suite has none. The quantum suite installs the qpu add-on). name: e2e-suite on: workflow_call: @@ -81,7 +81,7 @@ jobs: echo "running $s" bash "$s" else - echo "no setup.sh for suite '${{ inputs.suite }}' — skipping" + echo "no setup.sh for suite '${{ inputs.suite }}', skipping" fi # Discover and run every NN-*.sh in the suite directory, in sorted order. diff --git a/.github/workflows/e2e-tests.yaml b/.github/workflows/e2e-tests.yaml index 021e611..638530c 100644 --- a/.github/workflows/e2e-tests.yaml +++ b/.github/workflows/e2e-tests.yaml @@ -10,7 +10,7 @@ concurrency: jobs: # Fan out the suites as parallel jobs, each a call into the reusable workflow. - # The shared setup (build, kind, deploy) lives once in e2e-suite.yaml; the + # The shared setup (build, kind, deploy) lives once in e2e-suite.yaml. The # matrix runs gang and quantum concurrently. e2e: strategy: diff --git a/.github/workflows/sidecar-build-deploy.yaml b/.github/workflows/sidecar-build-deploy.yaml index 16b55a4..4339f3c 100644 --- a/.github/workflows/sidecar-build-deploy.yaml +++ b/.github/workflows/sidecar-build-deploy.yaml @@ -1,7 +1,7 @@ name: sidecar-build-deploy # Builds and tests the single, provider-agnostic Fluence quantum sidecar image. -# The fluence Python package (python/) is baked into the image; the sidecar +# The fluence Python package (python/) is baked into the image. The sidecar # resolves its vendor provider at runtime, so there is one image, not one per # vendor. The same image also serves as the init container that stages the # package into user pods (Model C). @@ -28,7 +28,7 @@ env: jobs: # Test the fluence package before building the image. These tests need no - # vendor SDK — they cover registration, provider resolution, fail-soft + # vendor SDK, they cover registration, provider resolution, fail-soft # interceptor behavior, and the staging mechanism end to end. test: runs-on: ubuntu-latest @@ -78,7 +78,7 @@ jobs: id: meta uses: docker/metadata-action@v5 with: - # One image: ghcr.io//fluence-sidecar (matches the SidecarImage + # One image. ghcr.io//fluence-sidecar (matches the SidecarImage # constant the webhook injects). images: ${{ env.REGISTRY }}/${{ github.repository }}-sidecar tags: | diff --git a/.github/workflows/unit-tests.yaml b/.github/workflows/unit-tests.yaml index b0e7b91..11a7758 100644 --- a/.github/workflows/unit-tests.yaml +++ b/.github/workflows/unit-tests.yaml @@ -11,7 +11,7 @@ concurrency: jobs: test: runs-on: ubuntu-latest - # fluence-base carries flux-sched 0.52.0 (installed to /usr) + Go, so the + # fluence-base has flux-sched 0.52.0 (installed to /usr) + Go, so the # cgo matcher (pkg/graph) links and `make test` can build every package. container: image: ghcr.io/converged-computing/fluence-base:latest diff --git a/Makefile b/Makefile index 5e2c050..bb85911 100644 --- a/Makefile +++ b/Makefile @@ -1,15 +1,15 @@ -# fluence — Kubernetes scheduler plugin backed by Fluxion (flux-sched), with +# fluence, Kubernetes scheduler plugin backed by Fluxion (flux-sched), with # optional quantum resources modeled in the resource graph. # # The scheduler only schedules. It links flux-sched (the matcher) and does NOT -# depend on QRMI — quantum job submission lives in a separate workload container +# depend on QRMI, quantum job submission lives in a separate workload container # (github.com/converged-computing/qrmi-sampler), not here. FLUX_SCHED_ROOT ?= /opt/flux-sched IMG ?= ghcr.io/converged-computing/fluence:latest TEST_IMG ?= vanessa/fluence:test -# cgo flags for the scheduler binary: flux-sched only. +# cgo flags for the scheduler binary. flux-sched only. CGO_CFLAGS = -I$(FLUX_SCHED_ROOT) CGO_LDFLAGS = -L$(FLUX_SCHED_ROOT)/resource \ -L$(FLUX_SCHED_ROOT)/resource/libjobspec \ diff --git a/README.md b/README.md index aeee0ae..2dc4c47 100644 --- a/README.md +++ b/README.md @@ -5,21 +5,11 @@ 🚧 **UNDER DEVELOPMENT** 🚧 Thank you for your patience! -@vsoch -A Kubernetes scheduler plugin that places **pod groups** (and individual pods) by -matching them against a [Fluxion](https://github.com/flux-framework/flux-sched) -resource graph built from the live cluster. Beyond ordinary compute, it can model -**arbitrary virtual resources** — quantum backends today, anything with a count -and some attributes tomorrow — as first-class graph vertices that can be filtered -on and whose attributes are injected into the workload's environment. Nothing in -the design is quantum-specific: a resource `type` is an opaque string throughout. - -This updates [flux-k8s](https://github.com/flux-framework/flux-k8s) to use the -native Kubernetes PodGroup (Gang) API — no sidecar, and no -`kubernetes-sigs/scheduler-plugins` dependency. The quantum resource model is -proven out first in -[fluxion-quantum](https://github.com/converged-computing/fluxion-quantum). - -## How does it work? +Fluence is a Kubernetes scheduler plugin. It builds a [Fluxion](https://github.com/flux-framework/flux-sched) resource graph from the live cluster and places pods, alone or in groups, by matching them against it. The graph does not have to stop at compute. Anything with a count and a few attributes can be a vertex in it, and then you can filter on it and the pod that gets it learns what it got through its environment. Quantum backends were the first thing I modeled this way, but there is nothing quantum specific in the design. + +This replaces [flux-k8s](https://github.com/flux-framework/flux-k8s). Gangs go through the native Kubernetes PodGroup API now, and there is no dependency on kubernetes-sigs/scheduler-plugins. I worked out the quantum resource model first in [fluxion-quantum](https://github.com/converged-computing/fluxion-quantum). + +## How it works ```console resources.yaml --+ @@ -39,289 +29,93 @@ scheduler device plugin webhook webhook-injected env reads those annotations ``` -A single `resources.yaml` is the source of truth, read by three independent -processes: the **scheduler** builds the graph and validates requests, the **device -plugin** advertises the requestable types as Kubernetes extended resources (so -pods requesting them pass admission), and the **webhook** computes the -environment-variable contract it injects into workloads. Note that the device -plugin can be removed from the design, assuming we do not need a `NodeResourcesFit` check. -I have kept it for now anticipating future cases where we have node-specific resources. -For now, the counts can be viewed as API quotas. - -For one pod group the scheduler generates one or more jobspecs, match-allocates -each against the graph (all-or-nothing), combines them into a placement, binds the -pod, and stamps the matched backend + attributes as annotations. The webhook has -already injected downward-API env vars that read those annotations, so the -workload sees which backend and attributes it got. - -## The resource model - -**Every allocatable thing in the graph is a `node`-typed vertex** -- physical -compute nodes and every level of every virtual resource alike. This is deliberate: -Fluxion's RFC 31 property constraints (the only attribute-based pruning the matcher -offers) apply **only** to `node` and `storage_node` vertices, so modeling a virtual -resource as a node is what makes it filterable. - -- **Physical nodes** carry `virtual=false`. -- **Virtual resources** carry `virtual=true`, a `class=` for the resource's - own type *and each descendant type in its subtree*, and their attributes. +There is one resources.yaml used by the scheduler, a device plugin, and webhook. The scheduler builds the graph from it and matches requests. The device plugin advertises the types in it as extended resources, which is only there so a pod asking for one gets past the NodeResourcesFit check. The webhook reads it to know which environment variables to inject. I could drop the device plugin, and I have kept it for the day we have resources that really are tied to a node. Until then the counts are quotas. -Property values are encoded into the property **key** (`virtual=true`, -`class=qpu`, `region=us-east-1`) because RFC 31 matching is key-presence only -- it -never compares the value half. A resource is selected by a `class=` -constraint, so any level is independently requestable (`class=qdevice`, -`class=qpu`, `class=qubit`). Descendant classes are propagated **up** so a global -constraint selecting a nested type isn't pruned at an ancestor node on the way -down; attributes are inherited **down** (a child gets its parent's attributes -unless it overrides or clears a key) so an attribute filter combined with a nested -class still reaches the target. +When a group comes in, the scheduler writes one or more jobspecs for it, matches each one against the graph, and if any of them fails it cancels the rest. Any nested topology for quantum devices would be represented in the graph structure. Then fluence binds the pods and adds annotation attributes. The webhook already gave the containers downward API variables that read those annotations, so by the time the container starts it knows its backend. -Because a jobspec carries one global constraint, compute (`virtual=false`) and a -virtual resource (`virtual=true`) cannot be co-selected in one match -- one would -prune the other. A pod needing both produces **two** match-allocate calls, held -together all-or-nothing. - -## Components +## The resource model -### `pkg/jgf` — JGF graph builder +Everything in the graph that can be allocated is a node vertex. That includes the real nodes and every level of every virtual resource. I did this because Fluxion only applies property constraints, RFC 31, to node and storage_node vertices, so a virtual resource has to be a node if you want to filter on it. -Builds the JSON Graph Format document Fluxion consumes. `AddRoot`/`AddChild` -create vertices with `Options` (size, unit, exclusivity, rank, properties). -`Options.NodeProperties` are RFC 31 resource properties (a nested `properties` -object the matcher prunes against); `Options.Properties` are descriptive metadata -only. `Options.Rank` sets a real execution rank — every `node` vertex needs one. +Real nodes have the attribute virtual=false and virtual resources have virtual=true. A full set of attributes might include virtual=true, class=qpu, and region=us-east-1, because RFC 31 only checks whether a key is present. You select a resource by its class, which means any level of the tree can be asked for on its own. Classes are inherited so a nested type is not pruned at its parent on the way down. -### `pkg/cluster` — config schema and graph construction +## Resources -Parses the resource config and turns it (plus the live cluster nodes) into a graph. +Here is an example of resources for AWS Braket. ```yaml resources: - - type: qdevice # opaque type; the resource's class - name: rigetti_cepheus # vertex name / backend identity - parent: cluster # where it attaches (default "cluster") - attributes: aws-east # inline map OR a reference into the registry below - with: # recursive children, same schema + - type: qdevice # the resource class + name: rigetti_cepheus # vertex name, the backend identity + parent: cluster # where it attaches, cluster by default + attributes: aws-east # an inline map or a name from the registry below + with: # children, same schema - type: qpu count: 1 with: - type: qubit count: 80 -attributes: # named attribute-set registry (reuse across backends) +attributes: aws-east: region: us-east-1 connectivity: all-to-all ``` -`LoadResourcesConfig` parses YAML/JSON, resolves attributes (inline or by -reference) with downward inheritance, and errors on a missing `type`, an unknown -reference, or a non-empty config that defines no `resources:` (a schema mismatch, -so it fails loudly instead of building an empty graph). `BuildGraph` emits each -cluster node as a `virtual=false` node, then appends the resource trees: every -resource at every depth becomes a `virtual=true` node carrying its class set and -inherited attributes, on a real rank from a counter shared with the physical -nodes. `FluxionResourceNames` returns every requestable type (what the device -plugin advertises); `AttributeKeys` returns the union of attribute keys (the -webhook's env contract). - -### `pkg/jobspec` — jobspec types - -The Fluxion jobspec representation (`Jobspec`, `Resource`, `Task`) with YAML/JSON -round-tripping: a `resources` tree (a `slot` holding the request), an `attributes` -block (constraint + duration), and `tasks`. - -### `pkg/placement` — jobspec generation and allocation parsing - -`JobspecsForGroup` turns a pod group into the jobspecs to match: - -- Always one **compute** jobspec — a slot per pod holding core/memory/gpu, - constrained to `virtual=false`. A classical group produces only this (one match). -- One **device** jobspec per requested virtual type, requesting a `node` - constrained to `virtual=true` + `class=`. The class selects which virtual - node; the count comes from the request. -- Every pod gets at least one core (a device-only pod still needs a host). -- Every jobspec sets `duration: 0` — held until explicitly cancelled. -- A request for an unmodeled type is a hard error. - -Jobspecs are submitted as **JSON** (not YAML): the constraint parser requires -quoted property scalars, and JSON always quotes. - -`PlacementFromAllocation` classifies an allocation's node vertices by the -`virtual` marker — `virtual=false`/unmarked are compute bind targets, a -`virtual=true` node is the backend identity, and its `fluxion.flux-framework.org/` -properties are decomposed into attributes for env injection. This package also -owns the shared names: the `fluxion.flux-framework.org/` request prefix, the -`fluence.flux-framework.org/{backend,jobid,attr-*}` annotations, and the -`FLUXION_` env prefix. - -### `pkg/graph` — Fluxion binding (cgo) - -Wraps the cgo matcher (`FluxionGraph`: `Init`, `MatchAllocateSpec`, `Cancel`, -`Satisfy`) and parses allocations. Fluence uses the **jgf** match format: it emits -every allocated vertex with its properties regardless of type, so a virtual -allocation that bottoms out in nodes (no cores) still serializes — which rv1 -cannot. `NodesFromAllocation` returns node vertices with their properties for the -marker-based classification. This is the only cgo-dependent package, so it gates -local builds of everything importing it. - -### `pkg/fluence` — the scheduler plugin - -`New` lists the cluster, loads the config, builds and logs the graph, and inits the -matcher. - -- **PreFilter** runs per group: generate jobspecs, then `matchGroup` runs each as - an independently held allocation, **all-or-nothing** — any failure cancels the - successes, so the group never holds a partial set. -- **Filter** permits only the nodes Fluxion assigned. -- **PreBind** records durable state: the jobids on the owning object (PodGroup for - a gang, else the pod) for cancellation, and the matched backend + attributes as - annotations the webhook reads. -- **Cancellation** is informer-driven (no framework delete hook): deleting a - PodGroup or ungrouped pod frees its held allocations. The jobid annotation is the - durable source of truth; the graph (and allocations) are rebuilt on restart from - the same annotations. - -Gang semantics are delegated to the native PodGroup API; fluence only places. - -### `pkg/deviceplugin` — extended-resource advertisement - -Advertises each requestable type (`fluxion.flux-framework.org/`) as a -counted extended resource, so a pod requesting one passes `NodeResourcesFit` -admission. The real gating is Fluxion (and the backend's own limits); since a -virtual backend is reachable from any node, each type is advertised at a large -ceiling. Types come from the same config as the graph, so they can't drift. - -### `sidecars/` — quantum coordination sidecars - -Vendor-specific sidecar containers injected by the webhook into the scout pod -of a shared quantum workflow group. Each sidecar discovers the QPU task submitted -by the scout, polls the vendor queue, and ungates the worker pods when the -task reaches position==1. - -```console -sidecars/ - lib/ungate.py shared gate removal + ARN annotation logic - braket/ - sidecar.py AWS Braket sidecar (tag search, queue polling, ungate) - fluence_braket_intercept.py AwsDevice.run() monkey-patch (PYTHONSTARTUP) - Dockerfile build context is sidecars/ to include lib/ - design.md full design document - test/integration.sh local integration test (requires AWS credentials) -``` - -The sidecar is injected automatically — users only need the group label: +A missing type, an attribute name that is not in the registry, or a config with content but no resources key are all errors. I would rather it fail loudly than build an empty graph. The placement package turns a group into jobspecs. There is always one compute jobspec, a slot per pod, constrained to virtual=false. There is one more jobspec for each virtual type (e.g., quantum) the pods ask for. Every pod gets at least one core, and every jobspec has a duration of zero to ensure it is held until someone (the scout or user) cancels it. The graph package is what interfaces with the Fluxion matcher via cgo, meaning it needs flux-sched to build. We use jgf as the match format because rv1 will not pass through a virtual allocation without cores in it. The fluence package is the scheduler plugin, and PreFilter matches the whole group and cancels everything if any match fails. Filter only lets through the nodes Fluxion picked. Reserve and PreBind write the backend annotations and the Fluxion job ids onto the owning object, which is the PodGroup for a gang and the pod otherwise. When that object is deleted an informer cancels the allocations. The job id annotation will persist through a restart. The gang semantics come from the PodGroup API, and fluence only does placement. The deviceplugin advertises types as `fluxion.flux-framework.org/type`, and this is done as a courtesy to the user (to make it easy to use). For virtual resources these counts can be viewed as limits. In practical terms, the real limit is the number of hybrid pairs that can be supported (we cannot exceed the number of free and pre-emptible cores if we expect priority to work on a released reservation). The webhook is a mutating webhook that allows for registration of handlers, and the handlers mutate pods per a specific role. For example, the default fluence handler is a gang. There is also a quantum handler. Finally, the python module implements the sidecar and interceptors. We have to have one per vendor, but we do not have too many vendors, so we build just one sidecar for now. The user does not need to think about it, as fluence will inject it into containers when appropriate to submit quantum work. -```yaml -metadata: - labels: - fluence.flux-framework.org/group: my-workflow -spec: - schedulerName: fluence -``` - -Fluence creates the PodGroup, injects the sidecar, creates per-namespace -RBAC, and gates the worker pods. See `sidecars/braket/design.md` for -the full design including the SDK interceptor, queue position polling, and -the two-queue problem motivation. +## Quantum gangs -### `pkg/webhook` — environment injection +The two-queue problem is about co-scheduling of different queues. We typically control one, and not the other. A hybrid job without coscheduling would submit from classical resources and (likely) do a lot of waiting. Fluence holds them out of the cluster until the task is about to run and then lets them in with high priority. A shared quantum gang gets split into a scout and workers, and the scout monitors and ungates the workers when the quantum is ready. -A mutating webhook that surfaces scheduler-chosen values to a workload. Container -env is fixed at creation but the match happens after admission, so it injects -**downward-API** env vars whose values populate later from the annotations PreBind -writes. The injected set is a **config-derived contract**: `FLUXION_BACKEND` plus -one `FLUXION_` per attribute key across all backends — add an attribute and -its env var appears, no code change. The webhook self-manages TLS (generates a CA -and patches its own `caBundle`), so no cert-manager is needed. A vendor-agnostic -workload reads these normalized names regardless of which backend it matched. +The scout is one pod of the gang, the one with completion index 0 in an indexed Job. It runs the application, submits the one quantum task, and has a sidecar. It is never gated. The workers are everyone else. They start with a scheduling gate so they hold no node resources while the task is in the vendor queue. They know their role, and when they are ungated they get the scout's job id and its session, so they can fetch the result or submit into that same session. The sidecar is what finds the task, watches the vendor queue, ungates the workers when the task is next in line, and then waits for the workers to finish so the scout keeps its session until they are done. -## Commands +The vendor specific part lives in the providers. Braket tags a task and searches for it later. QRMI has no tag search, so its interceptor writes the job id and session to a file that the sidecar reads. QRMI covers IBM, Pasqal, IQM, and others through one API, and the vendor is just an attribute in the config. -- `cmd/fluence` — the scheduler binary (stock kube-scheduler + the plugin). -- `cmd/deviceplugin` — the extended-resource DaemonSet. -- `cmd/webhook` — the env-injection webhook. -- `cmd/recovery-probe` — verifies allocation replay survives a graph rebuild - (what a restart does); see `make test-restore`. +## Admission control -Note that the recovery probe (and graph restore) was implemented but removed because the code in fluxion is only part of a PR branch, and I feel nervous about depending on it. +Admission control is a check that asks "Can we support a new hybrid queue?" where that question is based on our ability to release a reservation with priority and guarantee that it can run given free cores and pre-emptible work. It is always on, and you can turn it off at your own risk by setting FLUENCE_ADMISSION=off in the webhook deployment read at startup. When turned on, the webhook will refuse a new quantum group that will not fit. This design is analogous to what we do with the flux-quantum jobtap plugin. In simple terms, we add up the cores every pod in the group will need, plus the cores already promised to groups that were admitted but have not started. That has to fit in the set of free plus preemptible cores (lower priority). A group that does not fit is refused when its first pod is created, so you see it at kubectl apply or on the Job. Every quantum pod gets marked protected so it never counts as preemptible. ## Configuration -These are environment variables for fluence. - -| Env var | Read by | Meaning | +| Variable | Read by | Meaning | |---|---|---| -| `FLUENCE_RESOURCES` | scheduler, device plugin, webhook | path to `resources.yaml`; absent = classical-only | -| `FLUENCE_MATCH_POLICY` | scheduler | Fluxion match policy (default `first`) | -| `FLUENCE_RESOURCE_CAPACITY` | device plugin | per-node ceiling per type (default 1000) | -| `FLUENCE_ADMISSION` | webhook | `on` turns admission control on (default off) | -| `FLUENCE_RESERVE_CORES` | webhook | cores held back from the admission budget (default 0) | -| `FLUENCE_WORKER_TIMEOUT` | sidecar | seconds to hold for the worker group after ungating, 0 waits without limit (default 0) | -| `FLUENCE_SESSION_HOLD_TIMEOUT` | scout container | seconds a scout's release waits for the worker group, 0 waits without limit (default 0) | +| `FLUENCE_RESOURCES` | scheduler, device plugin, webhook | path to resources.yaml, absent means classical only | +| `FLUENCE_MATCH_POLICY` | scheduler | Fluxion match policy, default first | +| `FLUENCE_RESOURCE_CAPACITY` | device plugin | per node ceiling per type, default 1000 | +| `FLUENCE_ADMISSION` | webhook | off turns admission control off, default on, read at startup | +| `FLUENCE_RESERVE_CORES` | webhook | cores held back from the admission budget, default 0 | +| `FLUENCE_WORKER_TIMEOUT` | sidecar | seconds to wait for the workers after ungating, 0 waits forever, default 0 | +| `FLUENCE_SESSION_HOLD_TIMEOUT` | scout container | seconds the scout's release waits for the workers, 0 waits forever, default 0 | -## Admission control - -A quantum group costs money once it holds a backend, so with FLUENCE_ADMISSION=on -the webhook only admits one that fits. It is off by default. This is the rule the flux-quantum jobtap plugin applies at submit. -The cores the whole group will need, plus what is already promised to admitted -groups that have not started, must fit in what is reachable. Reachable is the -free cores plus the cores held by running pods that could be preempted, meaning -pods that are unprotected and below the priority workers are ungated with. - -A group that does not fit is refused when its first pod is created, so kubectl -apply fails or the owning Job records the reason. It is not left pending with a -backend it cannot use. A standalone or independent quantum pod is a group of -one. A shared gang is one group, scout and workers together, admitted once -by whichever member arrives first and recorded on the gang PodGroup as -fluence.flux-framework.org/admitted-cores. Every quantum pod is marked -fluence.flux-framework.org/protected so it is never counted as preemptible, and -a pod that arrives already marked is refused. The code is in -pkg/webhook/admission.go. - -## Observability - -The scheduler logs (prefix `[fluence]`) the full graph and known devices at -startup and, per match, the submitted jobspec, the raw Fluxion allocation, and the -parsed placement. The webhook logs (`[fluence-webhook]`) the env contract at -startup. Because live behavior (cgo matcher, real Kubernetes) can't be fully -unit-tested, these logs are the primary debugging surface. +The scheduler logs the graph when it starts, and for every match it logs the jobspec it sent, the allocation it got back, and how it read it, all prefixed with [fluence]. The webhook logs its environment contract at startup. You can look at logs to debug. ## Build -The scheduler links flux-sched (the matcher). It does **not** link QRMI or any -quantum backend — quantum job submission lives in a separate workload container -([qrmi-sampler](https://github.com/converged-computing/qrmi-sampler)), not here. +The scheduler links flux-sched. It does not link QRMI or any vendor SDK. Submitting quantum work happens in the workload container. While we build and deploy images on GitHub packages, you can also build yourself to develop or test something. ```bash -# Inside the .devcontainer (flux-sched at /opt/flux-sched): -make build # bin/fluence (cgo+flux) + bin/fluence-deviceplugin + bin/fluence-webhook +# in the .devcontainer, with flux-sched at /opt/flux-sched +make build # bin/fluence, bin/fluence-deviceplugin, bin/fluence-webhook make test -make image # or build the container image with all three binaries +make image ``` ## Deploy -Create a cluster on a Kubernetes release with native gang scheduling and the -feature gates the kind config enables (`GangScheduling`, `GenericWorkload`, the -`scheduling.k8s.io/v1alpha2` API group): +You need a minimum Kubernetes version of 1.36.1 with the gang scheduling gates turned on. The kind config we provide does that. ```bash kind create cluster --image kindest/node:v1.36.1 --config deploy/kind-config.yaml kind load docker-image ghcr.io/converged-computing/fluence:latest ``` -### 1. Gang scheduling (classical — all you need for non-quantum) - -```bash -kubectl apply -f deploy/fluence.yaml # scheduler, RBAC, and the webhook -``` +### Gang scheduling -Pods opt in with `schedulerName: fluence`; a multi-pod gang adds a -`scheduling.k8s.io/pod-group` label (a single pod is a group of one, no label -needed). +If all you want is classical work. ```bash +kubectl apply -f deploy/fluence.yaml kubectl apply -f examples/podgroup.yaml kubectl get podgroups.scheduling.k8s.io ``` @@ -330,86 +124,50 @@ NAME POLICY WORKLOAD STATUS AGE training Gang Scheduled 15s ``` -Cleanup: +A pod opts in by setting schedulerName to fluence. A gang adds the group label and the webhook makes the PodGroup. I started out making people create the PodGroup themselves and found that annoying. To clean up, clear the finalizer and delete. ```bash kubectl patch podgroup training -n default --type=merge -p '{"metadata":{"finalizers":null}}' kubectl delete -f examples/podgroup.yaml ``` -### 2. Quantum (the resources add-on) +Importantly, the expectation is using fluence as the main scheduler. If you send something to another scheduler fluence cannot know about the result to update its resource graph. + +### Quantum resources -This supplies the `fluence-resources` ConfigMap (the source of truth for which -backends exist) and the device plugin that advertises them: +The resources ConfigMap says which backends exist and the device plugin advertises them. The scheduler and the webhook read the config when they start, so restart them after you apply it. ```bash kubectl apply -f deploy/fluence-resources.yaml kubectl apply -f deploy/device-plugin.yaml -# The scheduler and webhook read the config at startup — restart to pick it up: kubectl rollout restart deployment/fluence -n kube-system kubectl rollout restart deployment/fluence-webhook -n kube-system +kubectl get nodes -o jsonpath='{range .items[*]}{.metadata.name}{"\t"}{.status.allocatable}{"\n"}{end}' | grep fluxion.flux-framework.org ``` -Confirm the resources are advertised: +The workload holds the credentials. The scheduler never sees them. ```bash -kubectl get nodes -o jsonpath='{range .items[*]}{.metadata.name}{"\t"}{.status.allocatable}{"\n"}{end}' \ - | grep fluxion.flux-framework.org -``` - -Create the IBM credentials the **workload** uses to submit (the scheduler never -needs them): - -```bash -ibmcloud login --apikey export IBM_CLOUD_TOKEN= export IBM_CLOUD_CRN=$(ibmcloud resource service-instances --service-name quantum-computing --output json | jq -r '.[0].crn') -kubectl create secret generic ibm-quantum -n default \ - --from-literal=token="$IBM_CLOUD_TOKEN" --from-literal=crn="$IBM_CLOUD_CRN" +kubectl create secret generic ibm-quantum -n default --from-literal=token="$IBM_CLOUD_TOKEN" --from-literal=crn="$IBM_CLOUD_CRN" ``` -Run a single quantum pod — it just requests `fluxion.flux-framework.org/qpu`, with -no hard-coded backend (the webhook + PreBind supply `FLUXION_BACKEND`). The -example workload in examples/qrmi-workload does the QRMI side. It acquires a -session on the backend fluence chose, runs a Bell circuit through the QRMI -sampler, and releases the session. Without a Secret it prints what it would -need and exits. The image is ghcr.io/converged-computing/fluence-qrmi-workload, -built for amd64 and arm64, and examples/qrmi-pod.yaml runs it. -examples/quantum-pod.yaml runs the older qrmi-sampler: +A single quantum pod asks for `fluxion.flux-framework.org/qpu` and does not name a backend. The example in examples/qrmi-workload reads the backend fluence picked from FLUXION_BACKEND, opens a QRMI session on it, runs a Bell circuit, and closes the session. If there is no Secret it prints what it would have needed and exits, which is a fine way to check the placement without paying for anything. The image is ghcr.io/converged-computing/fluence-qrmi-workload. ```bash -kubectl apply -f examples/quantum-pod.yaml - -# fluence's chosen backend, also injected as $FLUXION_BACKEND in the container: -kubectl get pod sampler -o jsonpath='{.metadata.annotations.fluence\.flux-framework\.org/backend}{"\n"}' -kubectl logs sampler +kubectl apply -f examples/qrmi-pod.yaml +kubectl get pod qrmi-sampler -o jsonpath='{.metadata.annotations.fluence\.flux-framework\.org/backend}{"\n"}' +kubectl logs -f qrmi-sampler ``` -```console -2026/06/06 19:04:38 submitting sampler job to ibm_marrakesh -{"results": [ ... samples ... ]} -2026/06/06 19:04:50 done: 2070 bytes from ibm_marrakesh -``` - -When the pod completes, fluence cancels the Fluxion allocation, freeing the -resources (visible in the scheduler logs as `Cancel jobid: N`). Note that I lost access to -my IBM account so this has not been tested live since mid June. -Submission is **not** done by the scheduler — the workload container holds the -user's credentials and submits via qrmi-go. Fluence only schedules and hands off -the backend. (When we control local quantum devices this will change.) +When the pod finishes fluence cancels the allocation and you will see Cancel jobid in the scheduler log. -### 3. Quantum workflow groups (scout + workers) +### Quantum gangs -A quantum workflow group is a gang whose members share **one** quantum task: -one pod **submits** the work (the scout) and N−1 pods **wait** for the result -(the workers). All pods share a group label and run the *same* image; Fluence -co-schedules them, gives the scout a sidecar that watches the vendor queue, and -gates the workers so they consume no node resources during the (long, variable) -QPU queue wait — releasing them only when the task reaches `queue_position == 1`. +Every pod in the gang uses the group label, a coordination annotation, and schedulerName fluence. We use an indexed Job and that index 0 is the scout. ```yaml -# Every pod in the group carries the same group label + schedulerName: fluence, -# and opts into shared coordination. metadata: labels: fluence.flux-framework.org/group: my-qaoa-workflow @@ -419,83 +177,13 @@ spec: schedulerName: fluence ``` -#### Coordination modes - -`fluence.flux-framework.org/coordination` selects how the gang is coordinated; it -defaults to `independent`. - -- **`shared`** — the gang shares ONE quantum task. Fluence promotes one member to - scout and gates the rest as workers (see below). Use this for a coordinated - workflow where the classical post-processing should start together as the single - result lands. -- **`independent`** (default) — every member does its own quantum work: its own - real submit, its own queue wait, no gating. N members run N tasks. This is the - honest default; Fluence never invents coordination you did not ask for, and - never dedups tasks meant to be distinct. - -#### How the scout is chosen - -In `shared` mode the scout is the member the Job controller stamps with -`batch.kubernetes.io/job-completion-index: "0"` — so an **indexed Job** gives -deterministic, race-free election from a single identical template (every pod has -the same image and group label; only the index differs). This serves two contracts -with no extra configuration: - -- an **explicit-role script** that branches on the completion index (index 0 - submits; others wait and consume the result), and -- an **identical script** where every pod calls submit — the scout's submit is - real, and each worker's submit is transparently returned the scout's task - (the shared-result dedup), so the code need not branch at all. - -For loose pods with no completion index, the first pod admitted claims the scout -slot; an indexed Job is recommended when you need determinism. - -#### What Fluence does - -In `shared` mode the scout gets the interceptor (real mode) + sidecar and its -own group-of-one PodGroup `-scout` (`minCount: 1`), so it schedules -alone and runs the single real submit; it is never gated. The workers join the -`` gang (`minCount: N−1`), get a `quantum.braket/ready` scheduling gate, and -consume no node resources during the QPU queue wait. When the sidecar observes -`queue_position == 1`, it stamps the scout's task id onto each worker -(surfaced as `FLUENCE_QUANTUM_JOB_ID`) and removes their gates atomically with -setting the `fluence-quantum-classical` priority class so they reschedule promptly. -The scout is one of the N members, so the application runs exactly N times — -never N+1, and there is no separate submitter pod. - -Per-namespace RBAC (`fluence-sidecar` ServiceAccount/Role/RoleBinding) and the -interceptor staging are created automatically by the webhook on first use — no -manual setup required. - -```bash -# Apply your pods with the group label + coordination annotation + -# schedulerName: fluence. RBAC is created for you. -kubectl apply -f my-quantum-workflow.yaml -``` - -#### A note on the independent "all submit" case - -`coordination: independent` (the default) means *every* pod submits its own -quantum task: N independent submissions land in the vendor queue and run at -uncoordinated times. That is correct and sometimes exactly what you want (N -distinct circuits), but it offers no coordination benefit from grouping — it is -equivalent to N standalone quantum pods. For a single quantum submission, use a -standalone pod (no group label, see §2). For a coordinated workflow that shares -one result, use `coordination: shared` above. - +The default mode is independent. Every pod submits its own task and nobody is gated, which is what you want for N different circuits, and it is no different from N standalone pods. Set shared when the whole gang should wait on one task. The scout gets a PodGroup of one named group-scout, the interceptor, and the sidecar. The workers go in the gang PodGroup of N minus one with the quantum.braket/ready gate and the fluence-quantum-classical priority class. The sidecar RBAC and the staging are created by the webhook the first time they are needed. -### Notes - -- **Deletion hangs.** A PodGroup can hang on delete via finalizers if the workload - controller isn't running; clear them with the `kubectl patch` above. -- **State restore.** On restart the plugin repopulates the graph from each group's - jobid annotations and re-holds the allocations: `make test-restore`. +If a PodGroup hangs on delete, clear its finalizer the way shown above. ## License -Distributed under the MIT license; all contributions must be made under it. - -See [LICENSE](LICENSE), [COPYRIGHT](COPYRIGHT), and [NOTICE](NOTICE). +Distributed under the MIT license, and all contributions must be made under it. See [LICENSE](LICENSE), [COPYRIGHT](COPYRIGHT), and [NOTICE](NOTICE). SPDX-License-Identifier: MIT diff --git a/cmd/deviceplugin/main.go b/cmd/deviceplugin/main.go index 29cb55f..7f0c4b7 100644 --- a/cmd/deviceplugin/main.go +++ b/cmd/deviceplugin/main.go @@ -6,12 +6,12 @@ // The set of types is derived from the SAME resources config the scheduler uses // to build its graph (FLUENCE_RESOURCES), so the advertised resources and the // graph's resource types come from one source and cannot drift. If -// FLUENCE_RESOURCES is unset or the file is absent, nothing is advertised — the +// FLUENCE_RESOURCES is unset or the file is absent, nothing is advertised, the // node stays classical-only. // // A quantum backend is a remote API reachable from any node, not a node-local // device, so each type is advertised at a large per-node ceiling. That count is -// only a local admission gate (so NodeResourcesFit is satisfied); the real gates +// only a local admission gate (so NodeResourcesFit is satisfied). The real gates // are Fluxion (which backend, is one available) and the user's API limit. // // FLUENCE_RESOURCES path to the shared resources config diff --git a/cmd/fluence/main.go b/cmd/fluence/main.go index f613185..68331bd 100644 --- a/cmd/fluence/main.go +++ b/cmd/fluence/main.go @@ -1,6 +1,6 @@ // Command fluence is a Kubernetes scheduler built from the standard // kube-scheduler with the Fluence placement plugin registered out-of-tree. Run -// it as a second scheduler (schedulerName: fluence) alongside the default one. +// it as a second scheduler (schedulerName fluence) alongside the default one. package main import ( diff --git a/cmd/recovery-probe/main.go b/cmd/recovery-probe/main.go index 4d41bf0..df209af 100644 --- a/cmd/recovery-probe/main.go +++ b/cmd/recovery-probe/main.go @@ -1,9 +1,9 @@ // Command recovery-probe verifies the restart-recovery primitive against a real // flux-sched. It match-allocates a jobspec in one graph (capturing the rv1 R + -// jobid), then builds a SECOND, fresh graph from the same cluster file — exactly -// what a scheduler restart does, a graph with no allocation history — and +// jobid), then builds a SECOND, fresh graph from the same cluster file, exactly +// what a scheduler restart does, a graph with no allocation history, and // replays R verbatim with UpdateAllocate under the same jobid. Finally it -// re-matches against the fresh graph; for an exclusive, singular resource a +// re-matches against the fresh graph. For an exclusive, singular resource a // refusal proves the replay re-held it. // // Run in the devcontainer (links flux-sched via cgo). Throwaway diagnostic. @@ -47,7 +47,7 @@ func main() { // --- Graph instance #2: the "post-restart" scheduler. Fresh context, fresh // graph built from the same cluster file, NO allocation history. We never - // cancel jobid in instance #1; instance #2 simply never knew about it. --- + // cancel jobid in instance #1. Instance #2 simply never knew about it. --- fmt.Println("\n=== 2. instance #2: fresh graph (simulate restart) ===") post := newGraph(*graphFile) @@ -67,10 +67,10 @@ func main() { fmt.Println("\n=== 4. instance #2: re-MatchAllocate (refusal = replay held it) ===") req2, err := post.MatchAllocate(*specFile) if err != nil { - fmt.Println("PASS: second match refused — the replayed allocation is holding the resource.") + fmt.Println("PASS: second match refused, the replayed allocation is holding the resource.") return } - fmt.Printf("second match SUCCEEDED (jobid=%d) — interpret with your graph in mind:\n", req2.Number) + fmt.Printf("second match SUCCEEDED (jobid=%d), interpret with your graph in mind:\n", req2.Number) fmt.Println(" - spare capacity for this spec exists -> expected, not a failure;") fmt.Println(" - resource is exclusive & singular -> replay did NOT consume it, investigate.") } diff --git a/cmd/webhook/main.go b/cmd/webhook/main.go index 8510aa1..48f736e 100644 --- a/cmd/webhook/main.go +++ b/cmd/webhook/main.go @@ -60,7 +60,7 @@ func main() { } active, unknown := webhook.SetActiveHandlers(requested) for _, n := range unknown { - log.Printf("[fluence-webhook] WARNING: unknown handler %q — ignoring", n) + log.Printf("[fluence-webhook] WARNING: unknown handler %q, ignoring", n) } log.Printf("[fluence-webhook] active handlers (in dispatch order): %v", active) @@ -97,7 +97,7 @@ func main() { // The env contract is the union of attribute keys across the configured // backends (plus FLUXION_BACKEND), so the set of injected env vars tracks the // config automatically. Loaded from the same FLUENCE_RESOURCES the scheduler - // and device plugin use; absent/unset means just FLUXION_BACKEND. + // and device plugin use. absent/unset means just FLUXION_BACKEND. var attrKeys []string if path := os.Getenv("FLUENCE_RESOURCES"); path != "" { if data, rerr := os.ReadFile(path); rerr == nil { @@ -110,9 +110,10 @@ func main() { log.Printf("no resources config at %s (%v); injecting FLUXION_BACKEND only", path, rerr) } } - // admission control is off unless FLUENCE_ADMISSION=on, and + // admission control is always on unless the deployment sets + // FLUENCE_ADMISSION=off. It is read once here, nothing on a pod changes it. // FLUENCE_RESERVE_CORES is held back from its budget - admission := env("FLUENCE_ADMISSION", "off") == "on" + admission := env("FLUENCE_ADMISSION", "on") != "off" reserve, err := strconv.Atoi(env("FLUENCE_RESERVE_CORES", "0")) if err != nil || reserve < 0 { log.Fatalf("FLUENCE_RESERVE_CORES must be a non-negative integer, got %q", os.Getenv("FLUENCE_RESERVE_CORES")) diff --git a/deploy/device-plugin.yaml b/deploy/device-plugin.yaml index b42c852..ce0e8be 100644 --- a/deploy/device-plugin.yaml +++ b/deploy/device-plugin.yaml @@ -1,5 +1,5 @@ --- -# Device plugin: advertises the exotic Fluxion resource types (derived from the +# Device plugin. Advertises the exotic Fluxion resource types (derived from the # resources config above) on every node, so pods can request them via resources # and NodeResourcesFit is satisfied. apiVersion: apps/v1 diff --git a/deploy/fluence-pull-test.yaml b/deploy/fluence-pull-test.yaml index aadef20..a883e7b 100644 --- a/deploy/fluence-pull-test.yaml +++ b/deploy/fluence-pull-test.yaml @@ -33,7 +33,7 @@ subjects: name: fluence namespace: kube-system --- -# Delegated authentication: read the auth configmap in kube-system. This is the +# Delegated authentication. Read the auth configmap in kube-system. This is the # fix for the "extension-apiserver-authentication ... forbidden" errors. apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding @@ -49,7 +49,7 @@ subjects: name: fluence namespace: kube-system --- -# Extras the built-in scheduler role does not grant: the alpha PodGroup/Workload +# Extras the built-in scheduler role does not grant. The alpha PodGroup/Workload # API (gang), and leader-election leases under our scheduler name. apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole @@ -62,12 +62,12 @@ rules: - apiGroups: ["coordination.k8s.io"] resources: ["leases"] verbs: ["create", "get", "update", "list", "watch"] - # PreBind stamps the allocated backend onto the pod as an annotation; the + # PreBind stamps the allocated backend onto the pod as an annotation. The # built-in system:kube-scheduler role only allows patching pods/status, not # the pod object, so grant it here. - apiGroups: [""] resources: ["pods"] - # create/delete: the webhook creates the one-off quantum submitter pod + # create/delete. The webhook creates the one-off quantum submitter pod # (ensureSubmitterPod) and the scheduler reaps it during gang cleanup. verbs: ["get", "list", "watch", "create", "patch", "update", "delete"] # The webhook self-manages its TLS by patching its own config's caBundle. @@ -117,8 +117,8 @@ data: - schedulerName: fluence plugins: # multiPoint wires Fluence into every extension point its Go type - # implements: PreFilter, Filter, and PreBind (which stamps the backend - # annotation). Listing points individually risks omitting one — that is + # implements PreFilter, Filter, and PreBind (which stamps the backend + # annotation). Listing points individually risks omitting one, that is # exactly what left PreBind unwired and the backend annotation unset. multiPoint: enabled: [{name: Fluence}] @@ -166,9 +166,9 @@ spec: - configMap: {name: fluence-scheduler-config} - configMap: {name: fluence-resources, optional: true} --- -# Mutating webhook: injects scheduler-chosen values into pods at creation time +# Mutating webhook. Injects scheduler-chosen values into pods at creation time # (currently a downward-API QRMI_BACKEND env for quantum pods). It self-manages -# TLS — generates a CA + serving cert at startup and patches the caBundle below — +# TLS, generates a CA + serving cert at startup and patches the caBundle below # so no cert-manager and no committed keys. failurePolicy Ignore keeps a webhook # outage from blocking pod creation cluster-wide. apiVersion: apps/v1 @@ -192,7 +192,7 @@ spec: imagePullPolicy: Always command: ["/bin/fluence-webhook"] # The webhook derives the FLUXION_* env contract (FLUXION_VENDOR, - # FLUXION_QRMI_TYPE, ...) from the resource graph's attribute keys, so + # FLUXION_QRMI_TYPE...) from the resource graph's attribute keys, so # it needs the same graph the scheduler and device plugin read. Without # this it injects only FLUXION_BACKEND, and the sidecar can't route to # a provider (which keys on qrmi_type). @@ -256,16 +256,16 @@ webhooks: # # RBAC and supporting resources for the Fluence quantum sidecar. # -# The sidecar runs inside a leader pod and needs: +# The sidecar runs inside a leader pod and needs the following. # - patch/annotate on pods in its own namespace (to ungate workers and # propagate the task ARN annotation) # -# The sidecar ServiceAccount is namespace-scoped — it only has permissions +# The sidecar ServiceAccount is namespace-scoped, it only has permissions # in the namespace where the workflow runs. The webhook sets # spec.serviceAccountName on the leader pod to fluence-sidecar. # # The fluence Python package is staged into user containers by an init -# container (Model C): the webhook injects an init container from the +# container (Model C). The webhook injects an init container from the # sidecar image that copies the package + sitecustomize into a shared # volume on the user container's PYTHONPATH. No ConfigMap, no user install. # diff --git a/deploy/fluence-resources-test.yaml b/deploy/fluence-resources-test.yaml index 8c1f6a8..b302c88 100644 --- a/deploy/fluence-resources-test.yaml +++ b/deploy/fluence-resources-test.yaml @@ -1,18 +1,18 @@ # Resources add-on for fluence. Turns on fluence-managed resources by supplying # (1) the resources config and (2) the device plugin that advertises them. -# Quantum backends are just the example payload here; any resource type fluence -# can model goes in the same ConfigMap. Apply AFTER deploy/fluence.yaml: +# Quantum backends are just the example payload here. Any resource type fluence +# can model goes in the same ConfigMap. Apply AFTER deploy/fluence.yaml. # # kubectl apply -f deploy/fluence.yaml # base scheduler (no devices) # kubectl apply -f deploy/fluence-resources.yaml # + resources config + device plugin # kubectl rollout restart deployment/fluence -n kube-system # scheduler re-reads resources # # The base scheduler already mounts the `fluence-resources` ConfigMap optionally -# and reads FLUENCE_RESOURCES, so this add-on is purely additive — no edits to +# and reads FLUENCE_RESOURCES, so this add-on is purely additive, no edits to # the base Deployment. -# Resources config: the SINGLE source of truth for the resource types fluence -# injects/advertises. The scheduler builds qpu/qubit graph vertices from it; the +# Resources config. The SINGLE source of truth for the resource types fluence +# injects/advertises. The scheduler builds qpu/qubit graph vertices from it. The # device plugin derives which extended resources to advertise from the SAME # document (same rule), so the two cannot drift. apiVersion: v1 @@ -22,8 +22,8 @@ metadata: namespace: kube-system data: resources.yaml: | - # New generic schema: each backend is a resource tree attached under the - # cluster. Modeled as a qdevice carrying a requestable qpu child; attributes + # New generic schema. Each backend is a resource tree attached under the + # cluster. Modeled as a qdevice with a requestable qpu child. Attributes # become filterable graph properties AND injected env (FLUXION_). resources: - type: qdevice @@ -49,7 +49,7 @@ data: vendor: ibm qrmi_type: qiskit-runtime-service --- -# Device plugin: advertises the exotic Fluxion resource types (derived from the +# Device plugin. Advertises the exotic Fluxion resource types (derived from the # resources config above) on every node, so pods can request them via resources # and NodeResourcesFit is satisfied. apiVersion: apps/v1 diff --git a/deploy/fluence-resources.yaml b/deploy/fluence-resources.yaml index c11fea3..c406f0c 100644 --- a/deploy/fluence-resources.yaml +++ b/deploy/fluence-resources.yaml @@ -1,18 +1,18 @@ # Resources add-on for fluence. Turns on fluence-managed resources by supplying # (1) the resources config and (2) the device plugin that advertises them. -# Quantum backends are just the example payload here; any resource type fluence -# can model goes in the same ConfigMap. Apply AFTER deploy/fluence.yaml: +# Quantum backends are just the example payload here. Any resource type fluence +# can model goes in the same ConfigMap. Apply AFTER deploy/fluence.yaml. # # kubectl apply -f deploy/fluence.yaml # base scheduler (no devices) # kubectl apply -f deploy/fluence-resources.yaml # + resources config + device plugin # kubectl rollout restart deployment/fluence -n kube-system # scheduler re-reads resources # # The base scheduler already mounts the `fluence-resources` ConfigMap optionally -# and reads FLUENCE_RESOURCES, so this add-on is purely additive — no edits to +# and reads FLUENCE_RESOURCES, so this add-on is purely additive, no edits to # the base Deployment. -# Resources config: the SINGLE source of truth for the resource types fluence -# injects/advertises. The scheduler builds qpu/qubit graph vertices from it; the +# Resources config. The SINGLE source of truth for the resource types fluence +# injects/advertises. The scheduler builds qpu/qubit graph vertices from it. The # device plugin derives which extended resources to advertise from the SAME # document (same rule), so the two cannot drift. apiVersion: v1 @@ -22,8 +22,8 @@ metadata: namespace: kube-system data: resources.yaml: | - # New generic schema: each backend is a resource tree attached under the - # cluster. Modeled as a qdevice carrying a requestable qpu child; attributes + # New generic schema. Each backend is a resource tree attached under the + # cluster. Modeled as a qdevice with a requestable qpu child. Attributes # become filterable graph properties AND injected env (FLUXION_). resources: - type: qdevice diff --git a/deploy/fluence-test.yaml b/deploy/fluence-test.yaml index 585385f..a0087a7 100644 --- a/deploy/fluence-test.yaml +++ b/deploy/fluence-test.yaml @@ -33,7 +33,7 @@ subjects: name: fluence namespace: kube-system --- -# Delegated authentication: read the auth configmap in kube-system. This is the +# Delegated authentication. Read the auth configmap in kube-system. This is the # fix for the "extension-apiserver-authentication ... forbidden" errors. apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding @@ -49,7 +49,7 @@ subjects: name: fluence namespace: kube-system --- -# Extras the built-in scheduler role does not grant: the alpha PodGroup/Workload +# Extras the built-in scheduler role does not grant. The alpha PodGroup/Workload # API (gang), and leader-election leases under our scheduler name. apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole @@ -62,12 +62,12 @@ rules: - apiGroups: ["coordination.k8s.io"] resources: ["leases"] verbs: ["create", "get", "update", "list", "watch"] - # PreBind stamps the allocated backend onto the pod as an annotation; the + # PreBind stamps the allocated backend onto the pod as an annotation. The # built-in system:kube-scheduler role only allows patching pods/status, not # the pod object, so grant it here. - apiGroups: [""] resources: ["pods"] - # create/delete: the webhook creates the one-off quantum submitter pod + # create/delete. The webhook creates the one-off quantum submitter pod # (ensureSubmitterPod) and the scheduler reaps it during gang cleanup. verbs: ["get", "list", "watch", "create", "patch", "update", "delete"] # The webhook self-manages its TLS by patching its own config's caBundle. @@ -117,8 +117,8 @@ data: - schedulerName: fluence plugins: # multiPoint wires Fluence into every extension point its Go type - # implements: PreFilter, Filter, and PreBind (which stamps the backend - # annotation). Listing points individually risks omitting one — that is + # implements PreFilter, Filter, and PreBind (which stamps the backend + # annotation). Listing points individually risks omitting one, that is # exactly what left PreBind unwired and the backend annotation unset. multiPoint: enabled: [{name: Fluence}] @@ -153,8 +153,8 @@ spec: - --feature-gates=GenericWorkload=true,GangScheduling=true # Re-attempt unschedulable pods more often than the 5m default. In the # contention experiment a gang that loses the initial race for nodes is - # marked Unschedulable; this is how soon it is re-tried after capacity - # frees (the event-driven QueueingHint is best-effort; this is the + # marked Unschedulable. This is how soon it is re-tried after capacity + # frees (the event-driven QueueingHint is best-effort. This is the # backstop that bounds worst-case requeue latency). 30s keeps contended # gangs draining promptly without thrashing the queue. - --pod-max-in-unschedulable-pods-duration=30s @@ -174,9 +174,9 @@ spec: - configMap: {name: fluence-scheduler-config} - configMap: {name: fluence-resources, optional: true} --- -# Mutating webhook: injects scheduler-chosen values into pods at creation time +# Mutating webhook. Injects scheduler-chosen values into pods at creation time # (currently a downward-API QRMI_BACKEND env for quantum pods). It self-manages -# TLS — generates a CA + serving cert at startup and patches the caBundle below — +# TLS, generates a CA + serving cert at startup and patches the caBundle below # so no cert-manager and no committed keys. failurePolicy Ignore keeps a webhook # outage from blocking pod creation cluster-wide. apiVersion: apps/v1 @@ -201,7 +201,7 @@ spec: imagePullPolicy: Never command: ["/bin/fluence-webhook"] env: - # Use busybox as sidecar image in tests — avoids pulling the real + # Use busybox as sidecar image in tests, avoids pulling the real # sidecar image which is large and not cached in CI. - name: FLUENCE_SIDECAR_IMAGE value: "busybox:latest" @@ -255,16 +255,16 @@ webhooks: # # RBAC and supporting resources for the Fluence quantum sidecar. # -# The sidecar runs inside a leader pod and needs: +# The sidecar runs inside a leader pod and needs the following. # - patch/annotate on pods in its own namespace (to ungate workers and # propagate the task ARN annotation) # -# The sidecar ServiceAccount is namespace-scoped — it only has permissions +# The sidecar ServiceAccount is namespace-scoped, it only has permissions # in the namespace where the workflow runs. The webhook sets # spec.serviceAccountName on the leader pod to fluence-sidecar. # # The fluence Python package is staged into user containers by an init -# container (Model C): the webhook injects an init container from the +# container (Model C). The webhook injects an init container from the # sidecar image that copies the package + sitecustomize into a shared # volume on the user container's PYTHONPATH. No ConfigMap, no user install. # diff --git a/deploy/fluence.yaml b/deploy/fluence.yaml index 5168647..ab39698 100644 --- a/deploy/fluence.yaml +++ b/deploy/fluence.yaml @@ -33,7 +33,7 @@ subjects: name: fluence namespace: kube-system --- -# Delegated authentication: read the auth configmap in kube-system. This is the +# Delegated authentication. Read the auth configmap in kube-system. This is the # fix for the "extension-apiserver-authentication ... forbidden" errors. apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding @@ -49,7 +49,7 @@ subjects: name: fluence namespace: kube-system --- -# Extras the built-in scheduler role does not grant: the alpha PodGroup/Workload +# Extras the built-in scheduler role does not grant. The alpha PodGroup/Workload # API (gang), and leader-election leases under our scheduler name. apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole @@ -62,12 +62,12 @@ rules: - apiGroups: ["coordination.k8s.io"] resources: ["leases"] verbs: ["create", "get", "update", "list", "watch"] - # PreBind stamps the allocated backend onto the pod as an annotation; the + # PreBind stamps the allocated backend onto the pod as an annotation. The # built-in system:kube-scheduler role only allows patching pods/status, not # the pod object, so grant it here. - apiGroups: [""] resources: ["pods"] - # create/delete: the webhook creates the one-off quantum submitter pod + # create/delete. The webhook creates the one-off quantum submitter pod # (ensureSubmitterPod) and the scheduler reaps it during gang cleanup. verbs: ["get", "list", "watch", "create", "patch", "update", "delete"] # The webhook self-manages its TLS by patching its own config's caBundle. @@ -117,8 +117,8 @@ data: - schedulerName: fluence plugins: # multiPoint wires Fluence into every extension point its Go type - # implements: PreFilter, Filter, and PreBind (which stamps the backend - # annotation). Listing points individually risks omitting one — that is + # implements PreFilter, Filter, and PreBind (which stamps the backend + # annotation). Listing points individually risks omitting one, that is # exactly what left PreBind unwired and the backend annotation unset. multiPoint: enabled: [{name: Fluence}] @@ -165,9 +165,9 @@ spec: - configMap: {name: fluence-scheduler-config} - configMap: {name: fluence-resources, optional: true} --- -# Mutating webhook: injects scheduler-chosen values into pods at creation time +# Mutating webhook. Injects scheduler-chosen values into pods at creation time # (currently a downward-API QRMI_BACKEND env for quantum pods). It self-manages -# TLS — generates a CA + serving cert at startup and patches the caBundle below — +# TLS, generates a CA + serving cert at startup and patches the caBundle below # so no cert-manager and no committed keys. failurePolicy Ignore keeps a webhook # outage from blocking pod creation cluster-wide. apiVersion: apps/v1 @@ -190,13 +190,16 @@ spec: image: ghcr.io/converged-computing/fluence:latest command: ["/bin/fluence-webhook"] # The webhook derives the FLUXION_* env contract (FLUXION_VENDOR, - # FLUXION_QRMI_TYPE, ...) from the resource graph's attribute keys, so + # FLUXION_QRMI_TYPE...) from the resource graph's attribute keys, so # it needs the same graph the scheduler and device plugin read. Without # this it injects only FLUXION_BACKEND, and the sidecar can't route to # a provider (which keys on qrmi_type). env: - name: FLUENCE_RESOURCES value: /etc/fluence/resources.yaml + # admission control is on by default. Set to off here to disable it + - name: FLUENCE_ADMISSION + value: "on" ports: - containerPort: 8443 readinessProbe: @@ -254,16 +257,16 @@ webhooks: # # RBAC and supporting resources for the Fluence quantum sidecar. # -# The sidecar runs inside a leader pod and needs: +# The sidecar runs inside a leader pod and needs the following. # - patch/annotate on pods in its own namespace (to ungate workers and # propagate the task ARN annotation) # -# The sidecar ServiceAccount is namespace-scoped — it only has permissions +# The sidecar ServiceAccount is namespace-scoped, it only has permissions # in the namespace where the workflow runs. The webhook sets # spec.serviceAccountName on the leader pod to fluence-sidecar. # # The fluence Python package is staged into user containers by an init -# container (Model C): the webhook injects an init container from the +# container (Model C). The webhook injects an init container from the # sidecar image that copies the package + sitecustomize into a shared # volume on the user container's PYTHONPATH. No ConfigMap, no user install. # diff --git a/docs/coordination-handler-design.md b/docs/coordination-handler-design.md index 0f6631b..21da503 100644 --- a/docs/coordination-handler-design.md +++ b/docs/coordination-handler-design.md @@ -4,13 +4,13 @@ > (the coordination router + `mutateScout`/`mutateWorker`/`coordinationMode`/ > `isScout`), `pkg/webhook/handlers/gang.go` (classical gangs defer quantum > pods to the quantum handler), and `pkg/fluence/fluence.go` (reconcile reaps the -> `-scout` PodGroup, never the scout pod — it is a real member). +> `-scout` PodGroup, never the scout pod, it is a real member). > Unit tests are in `pkg/webhook/handlers/quantum_test.go`; structural e2e in -> `test/e2e/quantum/02–04`. Coordination is **role-aware**: the webhook stamps +> `test/e2e/quantum/02 to 04`. Coordination is **role-aware**: the webhook stamps > `FLUENCE_COORDINATION_ROLE` (scout/worker) and hands workers the > scout's task id (`FLUENCE_QUANTUM_JOB_ID`); the workload branches on the role > (scout submits, worker fetches the shared result by id). No submit -> interception, no faux flag — that earlier mechanism has been removed. +> interception, no faux flag, that earlier mechanism has been removed. ## Why this replaces the submitter-pod model @@ -18,16 +18,16 @@ The `add-sidecar-interface` branch coordinates a quantum gang by creating a *separate* one-off submitter pod (`-submitter`) that runs the user's application image to do the real submit, then ungates a gang of N faux-submitting members. That works, but it runs the user's application **N+1 times** for an -N-gang: once in the submitter (a full run whose post-processing nobody consumes) +N-gang. Once in the submitter (a full run whose post-processing nobody consumes) plus once in each of the N members. The redundant run is not an implementation -wart — it is a symptom of modeling quantum work as a scout/worker split +wart, it is a symptom of modeling quantum work as a scout/worker split while pretending one image plays both roles, selected at runtime by a faux flag. -This design keeps the split (it is correct) but removes the separate pod: the +This design keeps the split (it is correct) but removes the separate pod. The **scout is one of the N members**, promoted at admission, so the application -runs exactly **N times** — the needed number — with exactly **one real submit**. +runs exactly **N times**, the needed number, with exactly **one real submit**. -The core thesis is unchanged: Fluence is a generic gang scheduler (native gangs +The core thesis is unchanged. Fluence is a generic gang scheduler (native gangs since k8s 1.36), and per-resource nuance lives in handlers. This is entirely a change to the `quantum` handler plus a one-line deferral in the `gang` handler. @@ -35,22 +35,22 @@ change to the `quantum` handler plus a one-line deferral in the `gang` handler. A quantum task's content (the circuit) comes from user code, so **the pod that defines a task must run to submit it**. Therefore, per pod, *submit* and *gate* -are mutually exclusive — a pod either runs (and can submit) or is gated (and -cannot). Gating only ever buys resource savings for pods that **do not submit**: +are mutually exclusive, a pod either runs (and can submit) or is gated (and +cannot). Gating only ever buys resource savings for pods that **do not submit**. pods that consume a result someone else produced. -That partitions a quantum gang into two kinds, decided per pod: +That partitions a quantum gang into two kinds, decided per pod. -- **scout** — runs its code, submits its own task, holds a node through the +- **scout**, runs its code, submits its own task, holds a node through the queue wait. Not gateable, ever. -- **worker** — does not submit its own task. It reads the scout's result, or +- **worker**, does not submit its own task. It reads the scout's result, or submits into the scout's session. Fully gateable until that result is ready. ## Coordination modes (user-facing contract) Identical pod templates (a Job/Deployment) are genuinely ambiguous between "one shared task, fan the result out to N pods" and "N independent tasks." Fluence -cannot infer this; the user declares it with one annotation on the pod template: +cannot infer this. The user declares it with one annotation on the pod template. ```yaml metadata: @@ -63,63 +63,63 @@ metadata: | `independent` (default) | N pods each do their own quantum work | every pod | none possible (all are scouts) | N | N | | `shared` | one task; N pods consume the result | scout only | workers gated until task ready | N | 1 | -`coordination` is an open enum so future designs (e.g. `scatter` — index-paired +`coordination` is an open enum so future designs (e.g. `scatter`, index-paired task↔pod, §6.2 of the quantum doc) slot in as new modes without changing the -mechanism. Default is `independent`: never invent coordination the user did not +mechanism. Default is `independent`. Never invent coordination the user did not ask for, and never dedup tasks that were meant to be distinct. ### What each mode does to resources, honestly -- **shared**: the scout (1 node) holds its node through the queue wait; - workers (N−1) consume **zero** node resources while gated, then start at +- **shared** means the scout (1 node) holds its node through the queue wait. + workers (N-1) consume **zero** node resources while gated, then start at position==1. Idle cost during the wait ≈ 1 node, vs N for a traditional gang. -- **independent**: every pod is a scout, so every pod holds its node through - its own queue wait — N nodes idle. There is nothing to coordinate (no shared - result), so this is not a Fluence deficiency; it is the physics of "N +- **independent** means every pod is a scout, so every pod holds its node through + its own queue wait, N nodes idle. There is nothing to coordinate (no shared + result), so this is not a Fluence deficiency. It is the physics of "N independent tasks," and it is the user's explicit choice. The only way to reclaim even the scout's node in either mode is a resumable `.result()` (replay), and is deliberately **out of scope - for v1** (one idle node is cheap; replay imposes a replay-safe-code contract). + for v1** (one idle node is cheap. Replay imposes a replay-safe-code contract). ## Scout election Exactly one member must be the scout. Election is deterministic for the -recommended workload and best-effort otherwise: +recommended workload and best-effort otherwise. -- **Indexed Job (recommended):** the pod carries - `batch.kubernetes.io/job-completion-index`. **Index `0` is the scout**; - every other index is a worker. Deterministic, race-free, no recorded state — +- **Indexed Job (recommended):** the pod has + `batch.kubernetes.io/job-completion-index`. **Index `0` is the scout**. + every other index is a worker. Deterministic, race-free, no recorded state the controller already stamped the index, and identical templates yield differentiated behavior purely from it. This is why an indexed Job is the right shape and is what the experiments use. - **Non-indexed gang (Deployment / raw grouped pods):** first arrival claims the - scout slot by creating the scout PodGroup (create-if-absent); later pods + scout slot by creating the scout PodGroup (create-if-absent). Later pods find it present and become workers. Best-effort (racy under simultaneous - admission); documented, with indexed Job recommended for determinism. + admission). Documented, with indexed Job recommended for determinism. ## The two-group split -| | scout (index 0) | workers (indices 1..N−1) | +| | scout (index 0) | workers (indices 1..N-1) | |---|---|---| -| PodGroup | `-scout`, `minCount=1` | ``, `minCount=N−1` | +| PodGroup | `-scout`, `minCount=1` | ``, `minCount=N-1` | | schedules | immediately, alone | atomically as a gang, **after ungate** | | gate | none | `quantum.braket/ready` + preempting priority | | interceptor | staged (tags the real submit) | **not staged** (a worker's submits go into the scout session) | -| sidecar | yes — polls the task, ungates `` at position==1 | no | +| sidecar | yes, polls the task, ungates `` at position==1 | no | | app run | full; submits the one real task | full; reads role=worker and fetches the shared result by id, or submits into the scout session | `minCount=1` on the scout group is what removes the deadlock that forced a -separate submitter: a single-member group schedules alone, so the scout runs -during the wait while the `minCount=N−1` worker group sits gated. The two -groups have independent minCounts; neither blocks the other. The worker group -keeps a real gang `minCount` (N−1), so **gang scheduling is preserved and +separate submitter. A single-member group schedules alone, so the scout runs +during the wait while the `minCount=N-1` worker group sits gated. The two +groups have independent minCounts. Neither blocks the other. The worker group +keeps a real gang `minCount` (N-1), so **gang scheduling is preserved and demonstrable** (experiment requirement 1). -Coordination is role-aware rather than interception-based: the worker is told +Coordination is role-aware rather than interception-based. The worker is told `FLUENCE_COORDINATION_ROLE=worker` and handed the scout's task id (`FLUENCE_QUANTUM_JOB_ID`, stamped by the sidecar at ungate), and the workload fetches the shared result by that id instead of submitting. One real task, N -workers, each app run once, in full — and no SDK submit-interception. +workers, each app run once, in full, and no SDK submit-interception. ## Gate / ungate flow (shared mode) @@ -144,8 +144,8 @@ workers, each app run once, in full — and no SDK submit-interception. shared result); app post-processes. No worker submits. ``` -`independent` mode skips all of this: each pod is its own group-of-one, ungated, -real submit, optional observe-only sidecar — i.e. today's standalone path applied +`independent` mode skips all of this. Each pod is its own group-of-one, ungated, +real submit, optional observe-only sidecar, i.e. today's standalone path applied per pod. --- @@ -156,13 +156,13 @@ All changes are in `pkg/webhook/handlers/`. The webhook core, the `fluxion` handler, `dependency.go`, `sidecar.go`, and the Python interceptor/sidecar are **unchanged**. -### `gang.go` — defer on quantum pods (removes the ordering dependency) +### `gang.go`, defer on quantum pods (removes the ordering dependency) The gang handler currently calls `EnsurePodGroup` unconditionally and relies on idempotency to coexist with the quantum handler. With the two-group split the quantum handler owns *both* quantum PodGroups (and the scout's group differs from its admission-time label), so the gang handler must not also gang quantum -pods. Make it skip them: +pods. Make it skip them. ```go func (h *gangHandler) Applies(ctx context.Context, m webhook.MutatorAPI, pod *corev1.Pod) bool { @@ -176,7 +176,7 @@ func (h *gangHandler) Applies(ctx context.Context, m webhook.MutatorAPI, pod *co } ``` -### `quantum.go` — replace `Mutate` and the submitter machinery +### `quantum.go`, replace `Mutate` and the submitter machinery **Add** these constants (near the existing const block): @@ -206,9 +206,9 @@ const ( ) ``` -Keep `GangGroupEnv` (`FLUENCE_GANG_GROUP`) — it now tells the **scout's** +Keep `GangGroupEnv` (`FLUENCE_GANG_GROUP`), it now tells the **scout's** sidecar which worker group to ungate. **Delete** the separate-submitter -constants and helpers: `SubmitterAnnotation`, `GangGroupAnnotation`, +constants and helpers. `SubmitterAnnotation`, `GangGroupAnnotation`, `SubmitterGroupSuffix`, `SubmitterPodSuffix`, and the functions `mutateSubmitter` and `ensureSubmitterPod`. Everything else in the file (`resolveGroup`, `resolveGangSize`, `ownerReplicaSetN`, `countGroupPods`, @@ -260,7 +260,7 @@ func (h *quantumHandler) mutateScout(ctx context.Context, m webhook.MutatorAPI, sc.EnsureRBAC(ctx, pod.Namespace) // Tell the sidecar which worker group (the base group) to list + ungate. ops = append(ops, sc.ContainerOps(pod, false, []corev1.EnvVar{{Name: GangGroupEnv, Value: group}})...) - log.Printf("[fluence-webhook] quantum scout %s/%s — group %s (ungates %q)", + log.Printf("[fluence-webhook] quantum scout %s/%s, group %s (ungates %q)", pod.Namespace, pod.Name, pg, group) return ops } @@ -276,7 +276,7 @@ func (h *quantumHandler) mutateWorker(ctx context.Context, m webhook.MutatorAPI, dep := Dependency{Kind: DependencyKindQuantumSubmit, Scout: group + ScoutGroupSuffix, Gate: QuantumGate} ops = append(ops, dep.applyOps(pod)...) // gate + preempting priority + depends-on ops = append(ops, workerEnvOps(pod)...) // role=worker + FLUENCE_QUANTUM_JOB_ID - log.Printf("[fluence-webhook] quantum worker %s/%s — group %s minCount=%d, gated (role=worker)", + log.Printf("[fluence-webhook] quantum worker %s/%s, group %s minCount=%d, gated (role=worker)", pod.Namespace, pod.Name, group, n-1) return ops } @@ -290,7 +290,7 @@ func coordinationMode(pod *corev1.Pod) string { } // isScout decides whether THIS pod is the gang's single scout. Indexed Job -// (recommended): completion index 0 is the scout — deterministic, race-free. +// (recommended): completion index 0 is the scout, deterministic, race-free. // Otherwise: first arrival claims the scout slot by the absence of the // scout PodGroup (best-effort; prefer an indexed Job). func isScout(ctx context.Context, m webhook.MutatorAPI, pod *corev1.Pod, group string) bool { @@ -309,10 +309,10 @@ func isScout(ctx context.Context, m webhook.MutatorAPI, pod *corev1.Pod, group s } ``` -Note: `pod.Annotations` may be nil; the `idx, ok := pod.Annotations[...]` form is +Note that `pod.Annotations` may be nil. The `idx, ok := pod.Annotations[...]` form is nil-safe in Go (indexing a nil map yields the zero value, `ok=false`). -### Sidecar (Python) — no change +### Sidecar (Python), no change The scout's sidecar already resolves the vendor at runtime from `FLUXION_BACKEND`, discovers the task by the `fluence-pod-uid` tag, polls queue @@ -328,60 +328,60 @@ submitter pod. Two requirements, both demonstrable on a kind cluster with the mock path and on a real cluster with Braket. -### Requirement 1 — Fluence still gang-schedules +### Requirement 1, Fluence still gang-schedules -Unchanged classical-gang coverage plus a shared-mode assertion: +Unchanged classical-gang coverage plus a shared-mode assertion. - **Classical gang (regression):** keep `test/e2e/gang/*`. A `minCount=N` classical PodGroup schedules all-or-nothing. This proves the generic gang machinery is intact (the change only adds a quantum-pod deferral to `gang.Applies`). - **Shared worker gang (new assertion):** submit a `coordination: shared` - indexed Job of N. Assert: exactly one `-scout` PodGroup (minCount 1) and - one `` PodGroup (minCount N−1); the scout runs while the N−1 workers - are `SchedulingGated`; after ungate the N−1 schedule **together** (gang), not + indexed Job of N. Assert. Exactly one `-scout` PodGroup (minCount 1) and + one `` PodGroup (minCount N-1). The scout runs while the N-1 workers + are `SchedulingGated`. After ungate the N-1 schedule **together** (gang), not one-by-one. This proves gang scheduling still holds for the worker group. -### Requirement 2 — Both modes work and shared beats a traditional gang +### Requirement 2, Both modes work and shared beats a traditional gang The metric that isolates the win is **classical node-seconds consumed during the quantum queue wait** (lower is better), alongside correctness checks. -Three arms, same N, same workload (the QAOA sampler), same backend: +Three arms, same N, same workload (the QAOA sampler), same backend. | arm | how | expected node-seconds during queue wait | correctness | |-----|-----|------------------------------------------|-------------| -| **traditional gang** (baseline) | N pods all running, each waits the full queue (no Fluence coordination — e.g. a plain native gang, or `independent` with N=N) | ≈ **N × T_queue** | N pods each run; if they each submit, N real tasks | +| **traditional gang** (baseline) | N pods all running, each waits the full queue (no Fluence coordination, e.g. a plain native gang, or `independent` with N=N) | ≈ **N × T_queue** | N pods each run; if they each submit, N real tasks | | **shared** (new) | `coordination: shared` indexed Job, N pods | ≈ **1 × T_queue** (scout only; workers gated) | **1** real task; all N pods produce the **same** result; app runs N times, never N+1 | | **independent** (new) | `coordination: independent` indexed Job, N pods | ≈ **N × T_queue** (no coordination possible) | N distinct tasks/results; correct and the user's explicit choice (reported as the honest baseline, **not** claimed as an improvement) | -Headline comparison is **shared vs traditional**: same observable result to the N +Headline comparison is **shared vs traditional**. Same observable result to the N pods, but shared idles ~1 node through the queue wait instead of N, saving -≈ (N−1) × T_queue node-seconds, and runs the application N times rather than N+1 +≈ (N-1) × T_queue node-seconds, and runs the application N times rather than N+1 (the submitter-pod model's extra run is gone). -Instrumentation (reuse the Experiment 2 harness): -- per-pod `TIMING` lines → derive each pod's gated interval vs running interval; +Instrumentation (reuse the Experiment 2 harness). +- per-pod `TIMING` lines → derive each pod's gated interval vs running interval. sum running-but-pre-result node-seconds per arm. - scout's sidecar logs queue position over time → T_queue. -- assert real-submit count: shared = 1 (one tagged task on the backend), +- assert real-submit count. Shared = 1 (one tagged task on the backend), independent/traditional = N (count tagged tasks). -- assert shared correctness: all N pods log the **same** task id / result hash. +- assert shared correctness. All N pods log the **same** task id / result hash. -Suggested location: a new `experiments/4-coordination/` modeled on +Suggested location. A new `experiments/4-coordination/` modeled on `experiments/2-gang/` (it already measures idle reclamation), parameterized by the `coordination` annotation and N, emitting node-seconds-during-wait, real-submit -count, and result-agreement per arm. Plot node-seconds vs N for the three arms: -traditional and independent rise ~linearly in N; shared stays ~flat at one node. +count, and result-agreement per arm. Plot node-seconds vs N for the three arms. +traditional and independent rise ~linearly in N. Shared stays ~flat at one node. ### Build/run notes -- The scout/worker split needs no new image: scouts and workers run the - same role-aware sampler; the branch is `FLUENCE_COORDINATION_ROLE` - (scout submits; worker fetches the shared result by `FLUENCE_QUANTUM_JOB_ID`). +- The scout/worker split needs no new image. Scouts and workers run the + same role-aware sampler. The branch is `FLUENCE_COORDINATION_ROLE` + (scout submits. Worker fetches the shared result by `FLUENCE_QUANTUM_JOB_ID`). - Use an **indexed** Job (`completionMode: Indexed`, `parallelism == completions == N`) so scout election is deterministic (index 0) and `resolveGangSize` reads N from the owner. Stamp `fluence.flux-framework.org/coordination` in the pod template's annotations. - kind/mock runs exercise the structural assertions (groups, gating, ungate - ordering) without a backend; real-Braket runs add the node-seconds and + ordering) without a backend. real-Braket runs add the node-seconds and real-submit-count measurements. diff --git a/docs/handlers.md b/docs/handlers.md index 17ea0d7..aea80f9 100644 --- a/docs/handlers.md +++ b/docs/handlers.md @@ -1,10 +1,6 @@ # Webhook handlers & sidecar architecture -Fluence's value is not creating gangs (Kubernetes 1.36 native gang scheduling -already does that). It is **customizing the gang on the fly based on the -resources a pod requests** — e.g. a shared quantum gang becomes a size-1 -scout gang plus a size-(N-1) worker gang, with the scout running a -sidecar that ungates its workers when the quantum task is ready. +Fluence's value is not creating gangs (Kubernetes 1.36 native gang scheduling already does that). It is **customizing the gang on the fly based on the resources a pod requests** e.g. a shared quantum gang becomes a size-1 scout gang plus a size-(N-1) worker gang, with the scout running a sidecar that ungates its workers when the quantum task is ready. ## Handlers @@ -18,42 +14,31 @@ type Handler interface { } ``` -Handlers self-register by name (`init()` -> `webhook.Register`); a blank import -of the handlers package makes them AVAILABLE. The core never names a handler. - -**Ordering = the active list.** There is no per-handler priority. The active -handler list is BOTH the selection and the dispatch order: +Handlers self-register by name (`init()` -> `webhook.Register`); a blank import of the handlers package makes them AVAILABLE. The core never names a handler. **Ordering = the active list.** There is no per-handler priority. The active handler list is BOTH the selection and the dispatch order: ```go var DefaultHandlerOrder = []string{"fluxion", "quantum", "gang"} ``` -Dispatch walks this list in order. `gang` is last because it is last in the -list — the fallback that applies common defaults (honor `group-size`, else -owner-derived N) only if no earlier handler already shaped the gang. A -custom-resource handler is inserted into the list before `gang` to shape its own -gang first. To change the order, or disable a handler, pass a different list. +Dispatch walks this list in order. `gang` is last because it is last in the list, the fallback that applies common defaults (honor `group-size`, else owner-derived N) only if no earlier handler already shaped the gang. A custom-resource handler is inserted into the list before `gang` to shape its own gang first. To change the order, or disable a handler, pass a different list. ## Enabling/disabling handlers -By default ALL registered handlers are enabled. Restrict the active set on the -webhook command: +By default ALL registered handlers are enabled. Restrict the active set on the webhook command. ``` fluence-webhook --handlers=fluxion,gang # run without quantum FLUENCE_HANDLERS=fluxion,quantum,gang fluence-webhook ``` -Empty = the default list. The list is the order: `--handlers=gang,fluxion` runs -gang first; omitting a name disables it. Unknown names are warned and dropped. +Empty = the default list. The list is the order: `--handlers=gang,fluxion` runs gang first. Omitting a name disables it. Unknown names are warned and dropped. -(The handler set lives in the WEBHOOK, which mutates pods. `cmd/fluence` is the -scheduler plugin and runs no handlers.) +(The handler set lives in the WEBHOOK, which mutates pods. `cmd/fluence` is the scheduler plugin and runs no handlers.) ## Sidecar interface The coordination sidecar is a handler-owned capability, not a core one. Handlers -that need a sidecar use `handlers.Sidecar`: +that need a sidecar use `handlers.Sidecar`. ```go type Sidecar interface { @@ -63,21 +48,16 @@ type Sidecar interface { } ``` -The default `coreSidecar` delegates to the core's staging primitives. The quantum -handler uses it today; a custom handler can supply its own implementation -(different image, env, gating) without touching the core or other handlers. The -core's `MutatorAPI` keeps the staging primitives only so the default -implementation can delegate — handlers do not call them directly. +A custom handler can supply its own implementation (different image, env, gating) without touching the core or other handlers. ## Group size resolution (the default gang handler) -`minCount` (the atomic-schedule count) resolves as: +`minCount` (the atomic-schedule count) resolves in this order. -1. explicit `fluence.flux-framework.org/group-size` annotation — honored verbatim - (the override; e.g. a quantum split sets it directly); -2. else the owning indexed Job's `parallelism` (== MiniCluster size N); +1. explicit `fluence.flux-framework.org/group-size` annotation, honored verbatim (the override. e.g. a quantum split sets it directly). +2. else the owning indexed Job's `parallelism` (== MiniCluster size N). 3. else 1, logged. -This is a common default available to every gang; handler-specific annotations +This is a common default available to every gang. Handler-specific annotations (quantum coordination mode, completion index, etc.) live in their handlers and are not required by the core. diff --git a/docs/qrmi-coordination.md b/docs/qrmi-coordination.md index 0c87063..1eea06c 100644 --- a/docs/qrmi-coordination.md +++ b/docs/qrmi-coordination.md @@ -1,157 +1,67 @@ # QRMI in Fluence -Fluence has a **QRMI provider** that provides co-scheduling using [QRMI](https://github.com/qiskit-community/qrmi) (the Quantum Resource Management Interface) to open a session by the scout, and then deliver the session ID to any classical workers to use. Please read [`quantum-scheduling.md`](quantum-scheduling.md) first for the general design we are using across environments and interfaces. - -## One provider, many vendors - -QRMI is deliberately vendor-agnostic: a thin middleware (Rust core, Python/C -bindings) that exposes **one** lifecycle API over any resource it supports — -IBM Quantum, Pasqal, IQM, Alice & Bob, and more: +Fluence has a **QRMI provider** that provides co-scheduling using [QRMI](https://github.com/qiskit-community/qrmi) (the Quantum Resource Management Interface) to open a session by the scout, and then deliver the session ID to any classical workers to use. Please read [`quantum-scheduling.md`](quantum-scheduling.md) first for the general design we are using across environments and interfaces. QRMI is deliberately vendor-agnostic. It is a thin middleware (Rust core, Python/C bindings) that exposes a single API over any resource it supports. Examples include IBM Quantum, Pasqal, IQM, Alice & Bob, and more. Here is an example using the Python SDK. ```python from qrmi import QuantumResource, ResourceType, Payload, TaskStatus qrmi = QuantumResource("ibm_marrakesh", ResourceType.QiskitRuntimeService) -lock = qrmi.acquire() # reserve the device -> session token + +# reserve the device -> session token, requires paid account +lock = qrmi.acquire() job_id = qrmi.task_start(Payload.QiskitPrimitive(input=..., program_id=...)) -status = qrmi.task_status(job_id) # TaskStatus.{Queued,Running,Completed,...} + +# TaskStatus.{Queued,Running,Completed,...} +status = qrmi.task_status(job_id) result = qrmi.task_result(job_id) qrmi.task_stop(job_id); qrmi.release(lock) ``` -So Fluence has **one** provider — `fluence.providers.qrmi` — that drives this -API. It contains no per-vendor branches. Every vendor detail comes from the -Fluence **resource attributes**, which the scheduler stamps onto the match and -the webhook injects as `FLUXION_*` env into the sidecar and workload containers: +Fluence has the `fluence.providers.qrmi` provider that drives this API. We determine vendors via Fluence resource attributes. The attributes are set via environment variables by a scout sidecar that will ungate classical workers when we are at position 1 in the queue. They are set by the Fluence webhook on classical workers. They include. | resource attribute | env | role | |---|---|---| -| `qrmi_type` | `FLUXION_QRMI_TYPE` | the QRMI resource type — the single vendor-selecting knob (`qiskit-runtime-service`, `ibm-quantum-system`, `pasqal-cloud`, …). Maps to the QRMI `ResourceType` enum member. | +| `qrmi_type` | `FLUXION_QRMI_TYPE` | the QRMI resource type, the single vendor-selecting knob (`qiskit-runtime-service`, `ibm-quantum-system`, `pasqal-cloud`, …). Maps to the QRMI `ResourceType` enum member. | | `transport` (optional) | `FLUXION_TRANSPORT` | explicit `qrmi` marker for routing (see below). | | `backend` (the vertex name) | `FLUXION_BACKEND` | the backend/resource id (`ibm_marrakesh`, `FRESNEL`, …). | | `vendor` (optional) | `FLUXION_VENDOR` | informational / per-vendor sidecar-image routing. | -Adding a new QRMI vendor is therefore a **config** change (a `qrmi_type` in -`resources.yaml`), not a code change. The `qrmi_type → ResourceType` mapping is -derived generically (kebab-case → the enum's PascalCase member, acronym-aware so -`ibm-quantum-system → IBMQuantumSystem`), with a tiny alias table for legacy +Adding a new QRMI vendor is therefore a **config** change (a `qrmi_type` in `resources.yaml`), and not a code change. The `qrmi_type → ResourceType` mapping is derived generically (kebab-case → the enum's PascalCase member, acronym-aware so `ibm-quantum-system → IBMQuantumSystem`), with a tiny alias table for legacy names (`direct-access → IBMQuantumSystem`) and a version-defensive fallback. -## Routing (which provider handles a device) - -The sidecar resolves the provider at runtime from the injected attributes -(`resolve_from_env`). The QRMI provider matches when: - -1. `transport == qrmi` (explicit and future-proof — recommended in the resource - config), **or** -2. no `transport` is set but `qrmi_type` is present and is **not** a `braket-*` - type. Fluence uses `braket-*` qrmi_types for Braket-reached devices and the - QRMI resource-type strings for QRMI-reached ones, so "a qrmi_type that isn't - braket" cleanly means QRMI. This is data-driven: whatever the config sets - decides it, with no per-vendor list in the code. - -Braket and QRMI never collide. +## Routing -## Discovery: the one thing QRMI changes vs. Braket +The sidecar resolves the provider at runtime from the injected attributes. The QRMI provider matches when. -Braket's `SearchQuantumTasks` lets the sidecar find the scout's task by a tag -the interceptor stamped. QRMI has no tag-able submit and no searchable tag — -`task_start(payload)` takes no tags and returns an opaque id. So instead of a -vendor-side search, the provider uses an **in-pod handoff** (`fluence.handoff`): +1. `transport == qrmi` (explicit and future-proof, recommended in the resource config), **or** +2. no `transport` is set but `qrmi_type` is present and is **not** a `braket-*` type. -- the scout's interceptor patches `QuantumResource.task_start`, captures the - returned id, and writes `{job_id, session_id?, tier}` to - `$FLUENCE_HANDOFF_DIR/task.json`; -- the sidecar (always the same pod) reads that file, then polls - `task_status(job_id)` for readiness. +Braket and QRMI are mutually exclusive, as they both provide API interfaces to quantum queues. -The webhook mounts one writable `emptyDir` for this channel into the submitting -container(s) and the sidecar. It is RBAC-free because the scout and its -sidecar are co-located. This is a property of QRMI, not of any vendor, and the -handoff module is reusable by any future provider whose SDK can't be searched. -## Free vs. paid: the acquire nuance (also vendor-neutral) +## Discovery -QRMI separates *acquisition* (`acquire`/`release`, which reserves the device and, -where supported, grants a priority/session queue) from *submission*. +Braket's `SearchQuantumTasks` lets the sidecar find the scout's task by a tag the interceptor stamped. We currently write to a shared emptyDir, and it would be useful if QRMI could provide some kind of tag associated with the job that could be retrieved by the user tokebn in another environment. See `fluence.handoff` for how this works. -| | acquire() | queue | how Fluence coordinates | -|---|---|---|---| -| **free / open** | not permitted (raises) | fair-share | submit-only; ungate when the task starts RUNNING | -| **paid / premium** | returns a session token | priority | same wait; session token captured and handed to workers | +- the scout's interceptor patches `QuantumResource.task_start`, captures the returned id, and writes `{job_id, session_id?, tier}` to `$FLUENCE_HANDOFF_DIR/task.json`. +- the sidecar (always the same pod) reads that file, then polls `task_status(job_id)` for readiness. -The provider **auto-detects** the tier — no config, no change to the user's code -path: +The webhook mounts one writable `emptyDir` for this channel into the submitting container(s) and the sidecar. We don't need RBAC because the scout and its sidecar are co-located. The handoff module is reusable by any future provider whose SDK can't be searched. -- The interceptor patches `acquire()` so a **success** records the session token - + `tier=premium`, and a **failure** records `tier=free` and **re-raises - unchanged** so a free-tier app sees the real error and proceeds submit-only. - Fluence never forces `acquire()`. -- `task_start()` publishes `{job_id, session_id?, tier}` to the handoff. +## Free vs. paid -The **ungate decision is identical in both tiers**: ready once `task_status` is -`RUNNING` or terminal — the QRMI-visible analogue of Braket's -`queue_position == 1`. The tier only affects how long the wait is and whether a -session token exists. When present, the sidecar stamps it onto each worker -(`fluence.flux-framework.org/quantum-session-id` → `FLUENCE_QUANTUM_SESSION_ID`) -so a role-aware worker can submit into the same priority session. +QRMI separates *acquisition* (`acquire`/`release`, which reserves the device and, where supported, grants a priority/session queue) from *submission*. If you don't have a paid account you can only launch tasks (without priority) so the design here won't help you, as it gets a priority queue from a session. ## Holding the session for the workers -The scout acquired the session, so it would close when the scout -application calls release, long before the workers run. The interceptor -patches release on a scout so it waits for a marker the sidecar writes. -After the sidecar ungates the workers it waits for every pod in the gang -other than itself to reach Succeeded or Failed, then writes the marker to the -handoff volume and exits. The scout application then releases and exits, -the pod finishes, and fluence frees its allocation. The scout keeps its -session and its qpu the whole time, the way the scout does in flux-quantum. - -Two timeouts bound the hold. FLUENCE_WORKER_TIMEOUT on the sidecar is how -long it waits for the group, and FLUENCE_SESSION_HOLD_TIMEOUT in the scout -container is how long release waits for the marker. Both default to zero, -which waits without limit. - -> QRMI exposes no numeric queue position, so `queue_position()` is best-effort -> telemetry (1 once RUNNING, else unknown) and never required for the ungate. - -## The sidecar image (one image, per-vendor escape hatch) - -The default sidecar is a single provider-agnostic image (`pip install ".[all]"` -= braket + `qrmi[ibm]`) that resolves the vendor at runtime. QRMI's `ibm` and -`iqm` extras pin incompatible Qiskit versions, so `all` includes `ibm` (the -default reachable QRMI vendor) but not `iqm`. When a vendor's SDK can't share the -combined image, build a slim single-vendor image and route to it: - -```bash -docker build --build-arg EXTRAS=iqm -t ghcr.io/you/fluence-sidecar:iqm python/ -``` - -Point the webhook at it globally (`FLUENCE_SIDECAR_IMAGE_`, e.g. -`FLUENCE_SIDECAR_IMAGE_IQM`) or per-pod -(`fluence.flux-framework.org/sidecar-image`). A pod that hints a vendor -(`require-vendor: iqm`, or a `require-qrmi_type` the webhook maps to a vendor) -then picks up `FLUENCE_SIDECAR_IMAGE_IQM` automatically. - -## Credentials +The scout acquired the session, so it would close when the scout application calls release, long before the workers run. The interceptor patches release on a scout so it waits for a marker the sidecar writes. After the sidecar ungates the workers it waits for every pod in the gang other than itself to reach Succeeded or Failed, then writes the marker to the handoff volume and exits. The scout application then releases and exits, the pod finishes, and fluence frees its allocation. The scout keeps its session and its qpu the whole time, the way the scout does in [flux-quantum](https://github.com/converged-computing/flux-quantum). -The scheduler never needs vendor credentials. The workload container holds them -(e.g. IBM's `IBM_CLOUD_TOKEN`/`IBM_CLOUD_CRN` or QRMI's `QRMI_*` env), and the -webhook copies the workload's secret/configMap-sourced env onto the sidecar so -it can build its own `QuantumResource` handle and poll `task_status`. Polling -status needs no `acquire()`, so this works for both tiers. +The environment variable FLUENCE_WORKER_TIMEOUT on the sidecar is how long it waits for the group, and FLUENCE_SESSION_HOLD_TIMEOUT in the scout container is how long release waits for the marker. Both default to zero, which waits without limit. Since QRMI exposes no numeric queue position, the `queue_position()` is best-effort. We return 1 once it is RUNNING, else unknown. -## Files +## The sidecar image -- `python/fluence/providers/qrmi.py` — the provider (interceptor + sidecar). -- `python/fluence/handoff.py` — the shared in-pod job-id channel. -- `pkg/webhook/handlers/quantum.go` — mounts the writable handoff volume, sets - `FLUENCE_HANDOFF_DIR`, surfaces `FLUENCE_QUANTUM_SESSION_ID`, and resolves the - per-vendor sidecar image. +The default sidecar is a single provider-agnostic image that resolves the vendor at runtime. This means we put all the pip installs for vendor SDKs into one image. If we have too many vendors and this is unreasonable to do we can build more than one sidecar. -## Tests +## Credentials -```bash -PYTHONPATH=python python3 python/tests/test_qrmi.py # provider + handoff + tiers + multi-vendor routing -go test ./pkg/webhook/... # handoff wiring + per-vendor image selection -``` +The scheduler never needs vendor credentials. The workload container holds them (e.g. IBM's `IBM_CLOUD_TOKEN`/`IBM_CLOUD_CRN` or QRMI's `QRMI_*` env), and the webhook copies the workload's secret/configMap-sourced env onto the sidecar so it can build its own `QuantumResource` handle and poll `task_status`. Polling status needs no `acquire()`, so this works for both tiers. diff --git a/docs/quantum-scheduling.md b/docs/quantum-scheduling.md index 5270106..1bc220d 100644 --- a/docs/quantum-scheduling.md +++ b/docs/quantum-scheduling.md @@ -2,60 +2,29 @@ ## The problem -A hybrid quantum and classical job submits to two queues, the Kubernetes -scheduler for the classical pods and the vendor queue for the quantum task. The -classical pods start right away and then sit on their nodes waiting for a result -that may be minutes or hours away. That waste grows with concurrency. On real -QPUs we measured 15 to 30 percent classical idle time at ten concurrent jobs and -over 70 percent for single pods at twenty. - -Fluence holds the classical pods out of the cluster until the quantum task is -about to run, then releases them with high priority so they start as the result -lands. I think this beats having the pods run and wait, even when the task -ahead in the queue turns out to be slow. Better vendor estimates would help -everyone. This only matters when a gang of pods is waiting on one quantum task, -or a loop keeps resubmitting one. +A hybrid quantum and classical job submits to two queues, the Kubernetes scheduler for the classical pods and the vendor queue for the quantum task. The classical pods start right away and then sit on their nodes waiting for a result that may be minutes or hours away. That waste grows with concurrency. On real QPUs we measured 15 to 30 percent classical idle time at ten concurrent jobs and over 70 percent for single pods at twenty. + +Fluence holds the classical pods out of the cluster until the quantum task is about to run, then releases them with high priority so they start as the result lands. I think this beats having the pods run and wait, even when the task ahead in the queue turns out to be slow. Better vendor estimates would help everyone. This only matters when a gang of pods is waiting on one quantum task, or a loop keeps resubmitting one. ## The design A shared quantum gang is split into one scout and N minus one workers. -- The scout is one of the N pods. It runs the application, submits the one real - quantum task, and is never gated. It carries a sidecar. -- The workers are the other pods. They start with a scheduling gate, so they - hold no node resources while the task waits in the vendor queue. They are - told their role and, once ungated, the job id and the session. -- The sidecar runs in the scout pod. It finds the task the scout submitted, - polls the vendor queue, and when the task is next in line it ungates the - workers. It then waits for the workers to finish, so the scout keeps its - session until they are done. -- An interceptor is staged into the scout container. It patches the vendor SDK - so the sidecar can find the task. For Braket it tags the task with the pod - uid. For QRMI, which has no tag search, it writes the job id and the session - to a small file the sidecar reads. - -The workers are released with a high priority class, so if the cluster filled -up while they waited they preempt lower priority work rather than queue behind -it. This is what makes the hold useful. Fluence frees the preempted pods' graph -allocations as they go. +- The scout is one of the N pods. It runs the application, submits the one real quantum task, and is never gated. It has a sidecar. +- The workers are the other pods. They start with a scheduling gate, so they hold no node resources while the task waits in the vendor queue. They are told their role and, once ungated, the job id and the session. +- The sidecar runs in the scout pod. It finds the task the scout submitted, polls the vendor queue, and when the task is next in line it ungates the workers. It then waits for the workers to finish, so the scout keeps its session until they are done. +- An interceptor is staged into the scout container. It patches the vendor SDK so the sidecar can find the task. For Braket it tags the task with the pod uid. For QRMI, which has no tag search, it writes the job id and the session to a small file the sidecar reads. + +The workers are released with a high priority class, so if the cluster filled up while they waited they preempt lower priority work rather than queue behind it. This is what makes the hold useful. Fluence frees the preempted pods' graph allocations as they go. ## What fluence looks at Two properties of a pod with schedulerName fluence decide what happens. -- Does any container request a quantum resource, fluxion.flux-framework.org/qpu? - Then there is a vendor backend behind the pod and fluence places it against - the graph. -- Does the pod carry the group label fluence.flux-framework.org/group? Then it - is part of a gang, scheduled through the native PodGroup API in Kubernetes - 1.36 and later. +- Does any container request a quantum resource, fluxion.flux-framework.org/qpu? Then there is a vendor backend behind the pod and fluence places it against the graph. +- Does the pod have the group label fluence.flux-framework.org/group? Then it is part of a gang, scheduled through the native PodGroup API in Kubernetes 1.36 and later. -A quantum gang also has a coordination mode, the annotation -fluence.flux-framework.org/coordination. The default is independent, every pod -does its own quantum work. In shared mode the gang shares one task and gets the -scout and worker split. The quantum resource request is what triggers the -interceptor and sidecar. The group label only controls gang scheduling and the -split. +A quantum gang also has a coordination mode, the annotation fluence.flux-framework.org/coordination. The default is independent, every pod does its own quantum work. In shared mode the gang shares one task and gets the scout and worker split. The quantum resource request is what triggers the interceptor and sidecar. The group label only controls gang scheduling and the split. ## What the user writes @@ -69,79 +38,44 @@ spec: schedulerName: fluence ``` -That is all. The scout is the pod with completion index 0 in an indexed Job, -which the Job controller stamps, so an identical template yields the roles -with no state and no race. I started out requiring a PodGroup object and found -it annoying, so the webhook creates it from the label. +That is all. The scout is the pod with completion index 0 in an indexed Job, which the Job controller stamps, so an identical template yields the roles with no state and no race. I started out requiring a PodGroup object and found it annoying, so the webhook creates it from the label. -The two prefixes are deliberate. fluence.flux-framework.org is the scheduler -plugin, the group, the mode, the gate. fluxion.flux-framework.org is the -resource graph, the extended resources and the backend attributes. +The two prefixes are deliberate. fluence.flux-framework.org is the scheduler plugin, the group, the mode, the gate. fluxion.flux-framework.org is the resource graph, the extended resources and the backend attributes. ## The webhook -The webhook core knows nothing about quantum. It dispatches to handlers that -register themselves, and a pod flows through every handler that applies. There -are three. - -- fluxion applies to any pod requesting a fluxion resource and injects the - FLUXION_* environment from the annotations the scheduler writes. -- gang applies to grouped pods that request no quantum resource. It creates the - PodGroup and links the pod to it. -- quantum applies to any pod requesting the quantum resource. A standalone or - independent pod gets the interceptor and submits for itself. In shared mode - the scout gets its own PodGroup of one, named group-scout, the interceptor, - the handoff volume, and the sidecar. A worker joins the gang PodGroup of - N minus one, is gated, gets the high priority class, and is given its role - plus downward API variables for the job id and the session, read from - annotations the sidecar stamps at ungate. - -The interceptor is delivered by an init container that copies the fluence -package into a shared volume on the application container's PYTHONPATH. Python -imports sitecustomize on every start, so the interceptor loads before user code -and patches whichever vendor SDK is present. No user install and no change to -the user image. - -The quantum handler also enforces admission control when it is turned on. See -pkg/webhook/admission.go. A quantum group is only admitted when the cores all -of its pods will need fit in what is reachable, free cores plus cores held by -preemptible pods, less what is promised to groups that have not started. +The webhook core knows nothing about quantum. It dispatches to handlers that register themselves, and a pod flows through every handler that applies. There are three. + +- fluxion applies to any pod requesting a fluxion resource and injects the FLUXION_* environment from the annotations the scheduler writes. +- gang applies to grouped pods that request no quantum resource. It creates the PodGroup and links the pod to it. +- quantum applies to any pod requesting the quantum resource. A standalone or independent pod gets the interceptor and submits for itself. In shared mode the scout gets its own PodGroup of one, named group-scout, the interceptor, the handoff volume, and the sidecar. A worker joins the gang PodGroup of N minus one, is gated, gets the high priority class, and is given its role plus downward API variables for the job id and the session, read from annotations the sidecar stamps at ungate. + +The interceptor is delivered by an init container that copies the fluence package into a shared volume on the application container's PYTHONPATH. Python imports sitecustomize on every start, so the interceptor loads before user code and patches whichever vendor SDK is present. No user install and no change to the user image. + +The quantum handler also enforces admission control, which is always on unless the webhook deployment sets FLUENCE_ADMISSION=off. See pkg/webhook/admission.go. A quantum group is only admitted when the cores all of its pods will need fit in what is reachable, free cores plus cores held by preemptible pods, less what is promised to groups that have not started. ## The sidecar 1. Resolve the provider from the FLUXION_* environment. 2. Find the task. Braket searches by the pod uid tag. QRMI reads the handoff file. -3. Find the workers, the pods in the group that still carry the gate. +3. Find the workers, the pods in the group that still have the gate. 4. Poll the task. Log the position for experiments. -5. When the task is next in line, or already running, annotate each worker - with the job id and the session, then remove the gate. -6. Wait for the workers to reach Succeeded or Failed, write the done marker so - the scout's release proceeds, and exit. +5. When the task is next in line, or already running, annotate each worker with the job id and the session, then remove the gate. +6. Wait for the workers to reach Succeeded or Failed, write the done marker so the scout's release proceeds, and exit. -If the task is never found or fails, the sidecar ungates anyway so the workers -are not stranded, and exits non zero so it shows. +If the task is never found or fails, the sidecar ungates anyway so the workers are not stranded, and exits non zero so it shows. ## Priority and preemption -Workers get the fluence-quantum-classical priority class, value one million, -at creation, since priorityClassName cannot change after a pod exists. The -gate keeps the priority from mattering until ungate. Once ungated, Kubernetes -preemption evicts lower priority pods to make room, and fluence's pod deletion -handler cancels their Fluxion allocations so the graph is free for the workers. +Workers get the fluence-quantum-classical priority class, value one million, at creation, since priorityClassName cannot change after a pod exists. The gate keeps the priority from mattering until ungate. Once ungated, Kubernetes preemption evicts lower priority pods to make room, and fluence's pod deletion handler cancels their Fluxion allocations so the graph is free for the workers. -Because each gang is released when its own task is next in line, classical -resources follow quantum execution order rather than submission order. A gang -on a quiet backend gets its nodes before an earlier gang on a busy one. +Because each gang is released when its own task is next in line, classical resources follow quantum execution order rather than submission order. A gang on a quiet backend gets its nodes before an earlier gang on a busy one. ## Providers -The webhook cannot know the vendor at admission, since the scheduler picks the -backend later. So the interceptor is staged into every quantum container and -asks every provider to install its hook, each skipping if its SDK is absent. -The sidecar resolves the one provider that matches at runtime. +The webhook cannot know the vendor at admission, since the scheduler picks the backend later. So the interceptor is staged into every quantum container and asks every provider to install its hook, each skipping if its SDK is absent. The sidecar resolves the one provider that matches at runtime. -A provider implements both halves in one module under -python/fluence/providers and calls register at import. +A provider implements both halves in one module under python/fluence/providers and calls register at import. ``` install_interceptor(pod_uid) patch the SDK submit call @@ -153,13 +87,9 @@ job_id(task) the id handed to workers session_id(task) the session handed to workers, if any ``` -Braket and QRMI are implemented. QRMI covers IBM, Pasqal, IQM, and others -through one API, with the vendor chosen by the qrmi_type attribute. +Braket and QRMI are implemented. QRMI covers IBM, Pasqal, IQM, and others through one API, with the vendor chosen by the qrmi_type attribute. -A single quantum pod with no workers gets the interceptor and environment but -no sidecar. The label fluence.flux-framework.org/observe set to true adds the -sidecar in observe mode, which logs the queue position and ungates nothing. -Experiments use that to measure singletons and gangs the same way. +A single quantum pod with no workers gets the interceptor and environment but no sidecar. The label fluence.flux-framework.org/observe set to true adds the sidecar in observe mode, which logs the queue position and ungates nothing. Experiments use that to measure singletons and gangs the same way. ## Properties @@ -174,20 +104,13 @@ Experiments use that to measure singletons and gangs the same way. ## Limitations -- Ungating preempts lower priority work. That work requeues and runs later, - but it is a disruption. A Fluxion reservation at a vendor supplied start - time would allow draining instead, and no vendor supplies one today. -- One task per gang. Parameter shift gradients and other multi circuit loops - need a set of tasks. See the scatter design below. -- The sidecar RBAC is created per namespace on first use. A chart or operator - would manage it more cleanly. +- Ungating preempts lower priority work. That work requeues and runs later, but it is a disruption. A Fluxion reservation at a vendor supplied start time would allow draining instead, and no vendor supplies one today. +- One task per gang. Parameter shift gradients and other multi circuit loops need a set of tasks. See the scatter design below. +- The sidecar RBAC is created per namespace on first use. A chart or operator would manage it more cleanly. ## Future work -- A MatchReserveAt primitive in the Fluxion bindings, fed by the live queue - position, to drain nodes rather than preempt. -- A scatter design, where N independent tasks each pair with one worker and the - sidecar ungates each worker as its own task comes up. +- A MatchReserveAt primitive in the Fluxion bindings, fed by the live queue position, to drain nodes rather than preempt. +- A scatter design, where N independent tasks each pair with one worker and the sidecar ungates each worker as its own task comes up. - Vendor task start events in place of polling. -- Topology aware preemption in PostFilter, so eviction targets exactly the pods - blocking a worker. +- Topology aware preemption in PostFilter, so eviction targets exactly the pods blocking a worker. diff --git a/examples/podgroup.yaml b/examples/podgroup.yaml index 7184a8f..50a5c96 100644 --- a/examples/podgroup.yaml +++ b/examples/podgroup.yaml @@ -1,5 +1,5 @@ # Native gang scheduling (k8s >= 1.36, GangScheduling/GenericWorkload gates on). -# Fluence does placement; the PodGroup gives all-or-nothing semantics. Pods link +# Fluence does placement. The PodGroup gives all-or-nothing semantics. Pods link # to the PodGroup via the first-class field spec.schedulingGroup.podGroupName. apiVersion: scheduling.k8s.io/v1alpha2 kind: PodGroup diff --git a/examples/test/e2e/gang/multi-gang-contention.yaml b/examples/test/e2e/gang/multi-gang-contention.yaml index 14b0fd8..b6784ee 100644 --- a/examples/test/e2e/gang/multi-gang-contention.yaml +++ b/examples/test/e2e/gang/multi-gang-contention.yaml @@ -1,5 +1,5 @@ -# Two gangs that cannot both place: fluxion allocates one core per slot, so two -# 2-pod gangs need 4 cores, but the cluster graphs ~3 (3 workers, ~1 core each). One gang places entirely; the loser stays FULLY pending +# Two gangs that cannot both place. fluxion allocates one core per slot, so two +# 2-pod gangs need 4 cores, but the cluster graphs ~3 (3 workers, ~1 core each). One gang places entirely. The loser stays FULLY pending # (all-or-nothing), never partial. apiVersion: apps/v1 kind: Deployment diff --git a/examples/test/e2e/gang/multi-gang-requeue.yaml b/examples/test/e2e/gang/multi-gang-requeue.yaml index a5d34f6..909087c 100644 --- a/examples/test/e2e/gang/multi-gang-requeue.yaml +++ b/examples/test/e2e/gang/multi-gang-requeue.yaml @@ -1,7 +1,7 @@ # Requeue-on-capacity + gang-atomicity test (test/e2e/gang/09). # gang-win: a 2-pod gang that runs a SHORT job and COMPLETES (pods -> Succeeded), # freeing its nodes. -# gang-wait: a 2-pod gang needing the same nodes; loses the initial race and sits +# gang-wait. A 2-pod gang needing the same nodes. Loses the initial race and sits # Unschedulable. When gang-win completes, gang-wait must be re-attempted # (via the shortened unschedulable-recheck timeout) and place atomically. # On a 3-worker (~3-core) cluster the two 2-pod gangs (4 cores) cannot co-run. diff --git a/examples/test/e2e/gang/single-podgroup.yaml b/examples/test/e2e/gang/single-podgroup.yaml index e44780b..253984f 100644 --- a/examples/test/e2e/gang/single-podgroup.yaml +++ b/examples/test/e2e/gang/single-podgroup.yaml @@ -1,5 +1,5 @@ # Native gang scheduling (k8s >= 1.36, GangScheduling/GenericWorkload gates on). -# Fluence does placement; the PodGroup gives all-or-nothing semantics. Pods link +# Fluence does placement. The PodGroup gives all-or-nothing semantics. Pods link # to the PodGroup via the first-class field spec.schedulingGroup.podGroupName. apiVersion: scheduling.k8s.io/v1alpha2 kind: PodGroup diff --git a/examples/test/e2e/quantum-pod-mock-2.yaml b/examples/test/e2e/quantum-pod-mock-2.yaml index e8c443b..e6e6996 100644 --- a/examples/test/e2e/quantum-pod-mock-2.yaml +++ b/examples/test/e2e/quantum-pod-mock-2.yaml @@ -1,5 +1,5 @@ # A second qpu pod, identical request to sampler-mock. Used by the restart-recovery -# test: after fluence restarts and replays the first allocation, this one must NOT be +# test after fluence restarts and replays the first allocation, this one must NOT be # given the same exclusive backend (it should stay Pending if the backend is taken). apiVersion: v1 kind: Pod diff --git a/examples/test/e2e/quantum/quantum-gang-pods.yaml b/examples/test/e2e/quantum/quantum-gang-pods.yaml index eaa0978..1fe1280 100644 --- a/examples/test/e2e/quantum/quantum-gang-pods.yaml +++ b/examples/test/e2e/quantum/quantum-gang-pods.yaml @@ -2,7 +2,7 @@ # # Two identical pods, both requesting the quantum resource, in group "qgang" with # coordination=shared. The user authors NO roles and NO submitter pod. The webhook -# splits the gang by completion index: +# splits the gang by completion index. # qgang-0 (index 0) -> SCOUT: its own group-of-one "qgang-scout" # (minCount 1), real submit, sidecar, NOT gated. It is a # real member, so the app runs N times, never N+1. @@ -10,12 +10,12 @@ # quantum.braket/ready + preempting priority, interceptor # told role=worker; it fetches the scout's result by id. # -# These are raw pods (not a Job) so the e2e can reference stable names; the +# These are raw pods (not a Job) so the e2e can reference stable names. The # completion-index annotation is set manually to make scout election # deterministic (a real workload uses an indexed Job, which the controller stamps # with batch.kubernetes.io/job-completion-index automatically). group-size makes N # deterministic for raw pods, which have no owning Job to derive it from. busybox -# stands in for the quantum app; the interceptor staging fails soft (no python), +# stands in for the quantum app. The interceptor staging fails soft (no python), # which is fine for the structural assertions in 02/03/04. apiVersion: v1 kind: Pod diff --git a/examples/test/e2e/quantum/quantum-pod-mock.yaml b/examples/test/e2e/quantum/quantum-pod-mock.yaml index 3290611..33115a2 100644 --- a/examples/test/e2e/quantum/quantum-pod-mock.yaml +++ b/examples/test/e2e/quantum/quantum-pod-mock.yaml @@ -1,4 +1,4 @@ -# CI mock of examples/quantum-pod.yaml: requests the same fluxion.flux-framework.org/qpu +# CI mock of examples/quantum-pod.yaml. Requests the same fluxion.flux-framework.org/qpu # resource so fluence matches a backend and the webhook injects QRMI_BACKEND, but runs a # trivial container instead of qrmi-sampler -- no IBM secrets, no real submission. Proves # placement + the backend handoff end to end. diff --git a/examples/test/jobspec-cpu.yaml b/examples/test/jobspec-cpu.yaml index 164fa54..7553394 100644 --- a/examples/test/jobspec-cpu.yaml +++ b/examples/test/jobspec-cpu.yaml @@ -1,5 +1,5 @@ # Classical spec (2 cores). Cores aren't exclusive and there are 2 of them, so -# this exercises the round-trip but step 4 may re-match against spare capacity; +# this exercises the round-trip but step 4 may re-match against spare capacity. # use qpu-jobspec.yaml for the unambiguous exclusive PASS signal. version: 9999 resources: diff --git a/examples/test/jobspec-qpu.yaml b/examples/test/jobspec-qpu.yaml index d10fd15..45864ff 100644 --- a/examples/test/jobspec-qpu.yaml +++ b/examples/test/jobspec-qpu.yaml @@ -1,4 +1,4 @@ -# One slot holding one exclusive qpu — the clean case for the probe: the graph +# One slot holding one exclusive qpu, the clean case for the probe. The graph # has exactly one exclusive qpu, so after replay the re-match MUST be refused. version: 9999 resources: diff --git a/pkg/cluster/cluster.go b/pkg/cluster/cluster.go index 56a845b..c769c51 100644 --- a/pkg/cluster/cluster.go +++ b/pkg/cluster/cluster.go @@ -1,11 +1,11 @@ // Package cluster builds a Fluxion resource graph from the live Kubernetes -// cluster. Physical compute (cpu/memory/gpu) is discovered from node capacity; +// cluster. Physical compute (cpu/memory/gpu) is discovered from node capacity. // virtual resources (e.g. quantum backends, but nothing here is quantum-specific) // are injected from a generic resource-tree configuration so the same graph can -// carry both physical and virtual vertices. +// have both physical and virtual vertices. // -// Model: every allocatable thing is a "node" vertex. Physical nodes carry the -// property virtual=false; configured virtual resources carry virtual=true plus +// Model. Every allocatable thing is a "node" vertex. Physical nodes have the +// property virtual=false; configured virtual resources have virtual=true plus // their configured attributes as RFC 31 properties. A pod's jobspec constrains // virtual=false for the compute it lands on and virtual=true for any virtual // device it requests, so the two are matched from disjoint vertex sets out of a @@ -38,7 +38,7 @@ type Options struct { // // Physical nodes are built first, then the configured resource trees are // appended under their parent vertices. A single contiguous rank counter spans -// physical and virtual node vertices: every node-typed vertex gets a real rank +// physical and virtual node vertices. Every node-typed vertex gets a real rank // (the rv1 writer requires it), and the virtual=true/false property keeps the // two sets apart at match time. func BuildGraph(nodes []corev1.Node, opts Options) ([]byte, error) { @@ -110,7 +110,7 @@ var controlPlaneTaints = map[string]bool{ "node-role.kubernetes.io/master": true, } -// isControlPlane reports whether a node carries a control-plane taint. +// isControlPlane reports whether a node has a control-plane taint. func isControlPlane(n *corev1.Node) bool { for i := range n.Spec.Taints { if controlPlaneTaints[n.Spec.Taints[i].Key] { diff --git a/pkg/cluster/resources.go b/pkg/cluster/resources.go index f0fc1aa..c718930 100644 --- a/pkg/cluster/resources.go +++ b/pkg/cluster/resources.go @@ -10,38 +10,38 @@ import ( // This file defines the generic resource-tree configuration that fluence reads // to inject virtual resources into the Fluxion graph. It is deliberately free -// of any quantum-specific assumptions: a Resource has an arbitrary string type +// of any quantum-specific assumptions. A Resource has an arbitrary string type // and may nest children, so any shape (a qdevice over qpus over qubits, a -// license server, an FPGA pool, ...) is expressible without per-type code. +// license server, an FPGA pool...) is expressible without per-type code. // -// Two top-level keys: +// Two top-level keys. // -// resources: a forest of resource trees to add to the graph. Each tree is +// resources is a forest of resource trees to add to the graph. Each tree is // attached under a named parent vertex (default "cluster"), so the // virtual subtree sits alongside the physical "rack" tree without // being linked to any specific physical node (today). A future // design could set parent to a node name to link them. -// attributes: a registry of named attribute sets. A Resource's Attributes may +// attributes. A registry of named attribute sets. A Resource's Attributes may // be given inline (a map) or by reference (a string naming an entry // here), so a shared set (e.g. a region/connectivity profile) is // defined once and reused. // // Every attribute becomes both a queryable/prunable RFC 31 graph property and an // injected environment value, so what a user can constrain on, they can also -// read back. (Graph/jobspec wiring lives in later pieces; this file only parses +// read back. (Graph/jobspec wiring lives in later pieces. This file only parses // and validates.) // DefaultParent is the vertex a resource tree attaches under when none is given. const DefaultParent = "cluster" // Resource is one vertex in a configured resource tree. Type is the Fluxion -// resource type (any string); Count is the size at this vertex; With are child +// resource type (any string). Count is the size at this vertex. With are child // resources (recursive). Attributes are resolved to a concrete map after load. type Resource struct { // Type is the Fluxion vertex type (e.g. "qdevice", "qpu", "qubit"). Required. Type string `json:"type"` // Name is an explicit vertex name (e.g. a backend id like "rigetti_cepheus"). - // Optional; the builder derives one from type+index when empty. + // Optional. The builder derives one from type+index when empty. Name string `json:"name,omitempty"` // Count is the resource quantity at this vertex (maps to the graph vertex // size). Defaults to 1 when zero. @@ -52,13 +52,13 @@ type Resource struct { Parent string `json:"parent,omitempty"` // Attributes is either an inline map (object) or a reference (string) into // the top-level attributes registry. Resolved into ResolvedAttributes at - // load time; do not read this field directly afterward. + // load time. Do not read this field directly afterward. Attributes AttributeSpec `json:"attributes,omitempty"` // With are child resources contained by this one (recursive). With []Resource `json:"with,omitempty"` // ResolvedAttributes is the concrete attribute map after reference - // resolution. Populated by LoadResourcesConfig; nil if none. + // resolution. Populated by LoadResourcesConfig. nil if none. ResolvedAttributes map[string]string `json:"-"` // BackendName is the identity of the top-level device this resource belongs @@ -69,7 +69,7 @@ type Resource struct { BackendName string `json:"-"` } -// AttributeSpec is a polymorphic attributes field: either a reference (a string +// AttributeSpec is a polymorphic attributes field. Either a reference (a string // naming a registry entry) or an inline map. Exactly one form is used. type AttributeSpec struct { // Ref is the registry key when attributes were given as a string. @@ -109,7 +109,7 @@ func (a AttributeSpec) empty() bool { return a.Ref == "" && len(a.Inline) == 0 } -// ResourcesConfig is the on-disk configuration: a forest of resource trees plus +// ResourcesConfig is the on-disk configuration. A forest of resource trees plus // a registry of named attribute sets they may reference. type ResourcesConfig struct { Resources []Resource `json:"resources"` @@ -125,11 +125,11 @@ func LoadResourcesConfig(data []byte) (*ResourcesConfig, error) { if err := yaml.Unmarshal(data, &c); err != nil { return nil, fmt.Errorf("parse resources config: %w", err) } - // Guard against a silent empty graph: if the file has real content (not just + // Guard against a silent empty graph. If the file has real content (not just // whitespace/comments) but produced zero resources, the schema almost // certainly doesn't match (e.g. a top-level key other than "resources:", like // an older "backends:" layout). Building a classical-only graph silently in - // that case hides the misconfiguration — the configured devices simply never + // that case hides the misconfiguration, the configured devices simply never // appear in the graph. A genuinely empty file (an optional configmap not yet // populated) is fine and stays classical-only. if len(c.Resources) == 0 && hasContent(data) { @@ -167,11 +167,11 @@ func resolveResource(r *Resource, registry map[string]map[string]string, topLeve // resolveResourceInherited resolves a resource's attributes and threads them // down to children. A child inherits every attribute of its parent, then applies -// its own: a key the child also sets overrides the inherited value, and a key the +// its own. A key the child also sets overrides the inherited value, and a key the // child sets to the empty string clears it (removed from the resolved set). This // makes a constraint that selects a nested node by class AND filters on an -// attribute reachable — every node on the path (parent down to the target) -// carries the attribute, so none is pruned mid-descent. +// attribute reachable, every node on the path (parent down to the target) +// has the attribute, so none is pruned mid-descent. func resolveResourceInherited( r *Resource, registry map[string]map[string]string, @@ -195,7 +195,7 @@ func resolveResourceInherited( own = resolved } - // Merge: start from inherited, apply own (override), honor explicit-clear. + // Merge. Start from inherited, apply own (override), honor explicit-clear. merged := make(map[string]string, len(inherited)+len(own)) for k, v := range inherited { merged[k] = v @@ -213,7 +213,7 @@ func resolveResourceInherited( r.ResolvedAttributes = nil } - // Thread the device identity down: a named top-level device sets it; children + // Thread the device identity down. A named top-level device sets it. Children // inherit it. Stamped as a match-only property by virtualProperties. if topLevel && r.Name != "" { r.BackendName = r.Name @@ -227,8 +227,8 @@ func resolveResourceInherited( return nil } -// resolveAttributes turns an AttributeSpec into a concrete map: inline is used -// directly; a reference is looked up in the registry (error if absent). +// resolveAttributes turns an AttributeSpec into a concrete map. Inline is used +// directly. A reference is looked up in the registry (error if absent). func resolveAttributes( spec AttributeSpec, registry map[string]map[string]string, diff --git a/pkg/cluster/resources_graph.go b/pkg/cluster/resources_graph.go index 418e729..db49547 100644 --- a/pkg/cluster/resources_graph.go +++ b/pkg/cluster/resources_graph.go @@ -10,7 +10,7 @@ import ( // Property keys fluence stamps on graph vertices. VirtualProperty marks whether // a node vertex is a configured virtual resource (true) or a physical compute -// node (false); it is the discriminator the jobspec constrains on and that +// node (false). It is the discriminator the jobspec constrains on and that // placement uses to tell a bind target from a backend. ClassProperty preserves // the configured resource type when a virtual resource is modeled as a node, so // the original type (e.g. "qdevice") is not lost. @@ -21,7 +21,7 @@ const ( // ComposeProperty encodes a key/value as the single "key=value" string that // Fluxion stores as a property key. RFC 31 PropertyConstraint matching is -// key-presence only — it never compares the value half of a vertex property — +// key-presence only, it never compares the value half of a vertex property // so to filter on a value the value must be part of the key. This is also // exactly how the rv1 match writer emits properties (prop + "=" + value), so the // encoding round-trips symmetrically between what we put in and what we read @@ -34,7 +34,7 @@ func ComposeProperty(key, value string) string { } // virtualProperties builds the property SET (composed key=value strings, stored -// as JGF property keys with empty values) for a configured virtual resource: the +// as JGF property keys with empty values) for a configured virtual resource. The // virtual marker, the class of this resource AND of every descendant in its // subtree, and each user attribute namespaced under the Fluxion resource prefix. // Every entry is filterable by an identical composed string in a jobspec @@ -44,7 +44,7 @@ func ComposeProperty(key, value string) string { // evaluated against every node vertex the matcher descends through (RFC 31 // constraints prune a node, and its subtree, on mismatch). To select a nested // node by class (e.g. class=qpu where qpu sits under a qdevice node), every -// ancestor node on the path must also satisfy class=qpu — so each node carries +// ancestor node on the path must also satisfy class=qpu, so each node has // the classes of everything beneath it. The trees are shallow, so these sets // stay small. A class= constraint then reaches any node of that type // because every ancestor down to it advertises that class. @@ -67,7 +67,7 @@ func virtualProperties(res *Resource) map[string]string { } // subtreeClasses returns the set of resource types in res's subtree, including -// res itself — the classes a node must advertise so a constraint selecting any +// res itself, the classes a node must advertise so a constraint selecting any // type within the subtree is not pruned at this node on the way down. func subtreeClasses(res *Resource) map[string]bool { types := map[string]bool{} @@ -111,8 +111,8 @@ func appendResources( } // addResource adds one configured resource, at ANY depth, as a node vertex -// carrying virtual=true, its configured type as the class property, and its -// attributes as RFC 31 properties — then recurses on its children the same way. +// with virtual=true, its configured type as the class property, and its +// attributes as RFC 31 properties, then recurses on its children the same way. // // Every level is a node so that every level is independently selectable by a // class= constraint (RFC 31 property constraints match only node and @@ -121,9 +121,9 @@ func appendResources( // and class= is meaningful; only the graph TYPE is "node". // // Every node consumes a real, unique rank from the shared counter. Ranks must be -// real (not -1) for any node vertex; the virtual sub-resources therefore appear +// real (not -1) for any node vertex. The virtual sub-resources therefore appear // in the rv1 nodelist, which is fine here because the allocation is a hold whose -// response we read directly — we never dispatch work to those ranks (no +// response we read directly, we never dispatch work to those ranks (no // flux-core execution). func addResource(b *jgf.Builder, parent *jgf.Vertex, res *Resource, rank *int64) error { r := *rank @@ -148,12 +148,12 @@ func addResource(b *jgf.Builder, parent *jgf.Vertex, res *Resource, rank *int64) } // FluxionResourceNames returns the distinct extended-resource names a device -// plugin should advertise for the configured resources: every counted resource +// plugin should advertise for the configured resources. Every counted resource // TYPE under a top-level resource (the top-level virtual node is matched via the // FluxionResourceNames returns the distinct extended-resource names a device -// plugin should advertise: every resource TYPE at every level of the configured +// plugin should advertise. Every resource TYPE at every level of the configured // trees. Every virtual resource is a node selectable by class=, so every -// type — top-level (qdevice) and nested (qpu, qubit) alike — is requestable. +// type, top-level (qdevice) and nested (qpu, qubit) alike, is requestable. // Prefixed so they match what the scheduler strips off a pod request. func FluxionResourceNames(resources []Resource) []string { types := map[string]bool{} @@ -177,7 +177,7 @@ func collectTypes(res *Resource, types map[string]bool) { // AttributeKeys returns the sorted union of every user attribute key across all // configured resources (and their children). This is the set of attributes that -// may be injected into a workload's environment; computing it from the config +// may be injected into a workload's environment. Computing it from the config // (rather than hardcoding) means the env contract automatically tracks whatever // attributes the backends declare. The backend identity itself is separate // (FLUXION_BACKEND) and not included here. diff --git a/pkg/cluster/resources_graph_test.go b/pkg/cluster/resources_graph_test.go index 8a89792..6e312fe 100644 --- a/pkg/cluster/resources_graph_test.go +++ b/pkg/cluster/resources_graph_test.go @@ -51,7 +51,7 @@ func metaByName(nodes []map[string]any, name string) map[string]any { return nil } -// A configured virtual resource is modeled as a node vertex carrying +// A configured virtual resource is modeled as a node vertex with // virtual=true, the class of itself AND its descendants, its (inherited) // attributes as namespaced RFC 31 properties, and a real rank. EVERY level // (qdevice, qpu, qubit) is a node so each is independently selectable by a @@ -92,7 +92,7 @@ resources: if _, ok := props["virtual=true"]; !ok { t.Errorf("missing virtual=true property; got %#v", props) } - // The qdevice node carries its own class AND its descendants' classes, so a + // The qdevice node has its own class AND its descendants' classes, so a // class=qpu / class=qubit constraint is not pruned at this node on the way // down to a nested target. for _, want := range []string{"class=qdevice", "class=qpu", "class=qubit"} { @@ -123,9 +123,9 @@ resources: } } // The qpu must NOT advertise class=qdevice (that is an ancestor, not in its - // subtree) — otherwise class=qdevice would wrongly select the qpu too. + // subtree), otherwise class=qdevice would wrongly select the qpu too. if _, ok := qprops["class=qdevice"]; ok { - t.Errorf("qpu node should not carry class=qdevice (ancestor class); got %#v", qprops) + t.Errorf("qpu node should not have class=qdevice (ancestor class); got %#v", qprops) } } @@ -185,7 +185,7 @@ resources: } // AttributeKeys returns the sorted union of attribute keys across all resources -// and children, deduplicated — this is the env-injection contract. +// and children, deduplicated, this is the env-injection contract. func TestAttributeKeysUnion(t *testing.T) { cfg := ` resources: @@ -209,7 +209,7 @@ resources: t.Fatal(err) } keys := AttributeKeys(c.Resources) - // union: connectivity, qubits, region, vendor (region deduped) + // union connectivity, qubits, region, vendor (region deduped) want := []string{"connectivity", "qubits", "region", "vendor"} if len(keys) != len(want) { t.Fatalf("keys = %v, want %v", keys, want) @@ -223,7 +223,7 @@ resources: // The device name must be stamped as a match-only property // (fluxion.flux-framework.org/backend=) on the device AND inherited by its -// descendant nodes, so a require-backend constraint can pin it — but it must NOT +// descendant nodes, so a require-backend constraint can pin it, but it must NOT // become a user attribute (that would double-inject FLUXION_BACKEND into pods). func TestBackendNameStampedAsMatchOnlyProperty(t *testing.T) { cfg := ` diff --git a/pkg/cluster/resources_test.go b/pkg/cluster/resources_test.go index b90370f..198b2c2 100644 --- a/pkg/cluster/resources_test.go +++ b/pkg/cluster/resources_test.go @@ -4,7 +4,7 @@ import ( "testing" ) -// A realistic config: a virtual qdevice (under the default parent) holding a qpu +// A realistic config. A virtual qdevice (under the default parent) holding a qpu // child, with attributes given by reference, plus a second device with inline // attributes. Exercises nesting, count, the parent default, and both attribute // forms. @@ -145,7 +145,7 @@ resources: } } -// A config with content but no resources: key (e.g. an older "backends:" layout) +// A config with content but no resources key (e.g. an older "backends:" layout) // must error loudly rather than silently yielding an empty graph. func TestLoadResourcesConfigRejectsSchemaMismatch(t *testing.T) { _, err := LoadResourcesConfig([]byte("backends:\n - name: ibm_fez\n num_qubits: 156\n")) @@ -167,9 +167,9 @@ func TestLoadResourcesConfigEmptyIsClassical(t *testing.T) { } } -// Attributes inherit downward: a child gets the parent's attributes, can override +// Attributes inherit downward. A child gets the parent's attributes, can override // a key, and can clear one by setting it to empty. This makes a nested class + -// attribute constraint reachable (every node on the path carries the attribute). +// attribute constraint reachable (every node on the path has the attribute). func TestAttributeInheritance(t *testing.T) { cfg := ` resources: diff --git a/pkg/deviceplugin/plugin.go b/pkg/deviceplugin/plugin.go index cc98012..3d04c4f 100644 --- a/pkg/deviceplugin/plugin.go +++ b/pkg/deviceplugin/plugin.go @@ -3,15 +3,15 @@ package deviceplugin // Package deviceplugin implements a Kubernetes device plugin that advertises // "quantum reachability" as a counted extended resource on every node. // -// A quantum backend is not hardware attached to a node — it is a remote API any +// A quantum backend is not hardware attached to a node, it is a remote API any // node can call (subject to the user's access). So the plugin advertises a large -// per-node ceiling of a single counted resource (quantum.flux-framework.org/qpu): +// per-node ceiling of a single counted resource (quantum.flux-framework.org/qpu). // this is what lets a pod write `resources.requests: {quantum.flux-framework.org/qpu: "1"}` // and have the in-tree NodeResourcesFit plugin be satisfied (no wrapper needed). // // The count is a local admission gate only. Whether a backend is actually // available and which one is matched is decided by Fluxion in the scheduler, and -// the real per-user limit lives on the IBM API — neither of which is node-local, +// the real per-user limit lives on the IBM API, neither of which is node-local, // which is exactly why the ceiling is large rather than a true quota. import ( @@ -125,7 +125,7 @@ func (p *Plugin) register(ctx context.Context) error { return err } -// GetDevicePluginOptions: no pre-start hook needed. +// GetDevicePluginOptions. No pre-start hook needed. func (p *Plugin) GetDevicePluginOptions(context.Context, *pluginapi.Empty) (*pluginapi.DevicePluginOptions, error) { return &pluginapi.DevicePluginOptions{PreStartRequired: false}, nil } @@ -135,7 +135,7 @@ func (p *Plugin) ListAndWatch(_ *pluginapi.Empty, stream pluginapi.DevicePlugin_ if err := stream.Send(&pluginapi.ListAndWatchResponse{Devices: p.devices}); err != nil { return err } - // Static list: keep the stream open, re-sending periodically until shutdown. + // Static list. Keep the stream open, re-sending periodically until shutdown. ticker := time.NewTicker(60 * time.Second) defer ticker.Stop() for { @@ -150,7 +150,7 @@ func (p *Plugin) ListAndWatch(_ *pluginapi.Empty, stream pluginapi.DevicePlugin_ } } -// Allocate is a no-op: a quantum "device" is just a reachability token, so no +// Allocate is a no-op. A quantum "device" is just a reachability token, so no // env vars, mounts, or device nodes are injected. The workload gets its backend // from the scheduler and its credentials from a Secret. func (p *Plugin) Allocate(_ context.Context, req *pluginapi.AllocateRequest) (*pluginapi.AllocateResponse, error) { @@ -161,12 +161,12 @@ func (p *Plugin) Allocate(_ context.Context, req *pluginapi.AllocateRequest) (*p return resp, nil } -// GetPreferredAllocation: no preference. +// GetPreferredAllocation. No preference. func (p *Plugin) GetPreferredAllocation(context.Context, *pluginapi.PreferredAllocationRequest) (*pluginapi.PreferredAllocationResponse, error) { return &pluginapi.PreferredAllocationResponse{}, nil } -// PreStartContainer: nothing to do. +// PreStartContainer. Nothing to do. func (p *Plugin) PreStartContainer(context.Context, *pluginapi.PreStartContainerRequest) (*pluginapi.PreStartContainerResponse, error) { return &pluginapi.PreStartContainerResponse{}, nil } diff --git a/pkg/fluence/fluence.go b/pkg/fluence/fluence.go index 235725c..f4d584b 100644 --- a/pkg/fluence/fluence.go +++ b/pkg/fluence/fluence.go @@ -44,22 +44,22 @@ type matcher interface { } // groupAlloc is the in-memory record of a group's Fluxion allocation. It is a -// rebuildable, within-lifetime memo: its job is race-free "match once per group" +// rebuildable, within-lifetime memo. Its job is race-free "match once per group" // dedup on the scheduling path (the durable record is the jobid annotation on -// the owning object). It does not survive a scheduler restart, which is fine — +// the owning object). It does not survive a scheduler restart, which is fine // the graph itself is rebuilt fresh on restart. type groupAlloc struct { place placement.Placement - // jobids are the Fluxion allocations backing this group — one per match + // jobids are the Fluxion allocations backing this group, one per match // (compute, plus one per requested virtual device). All are held (duration 0) - // and cancelled together; the group is all-or-nothing across them. + // and cancelled together. The group is all-or-nothing across them. jobids []uint64 } // Fluence is a scheduler-framework plugin that places whole pod groups by // matching them against a flux-sched resource graph built from the live cluster // (plus any configured virtual resources). Gang/all-or-nothing semantics are -// delegated to the native PodGroup API; Fluence only decides placement. +// delegated to the native PodGroup API. Fluence only decides placement. type Fluence struct { handle fwk.Handle matcher matcher @@ -70,7 +70,7 @@ type Fluence struct { knownDevices map[string]bool // matcherMu serializes all access to the cgo Fluxion client, which is not - // thread-safe. Match runs on the (sequential) scheduling path; Cancel runs in + // thread-safe. Match runs on the (sequential) scheduling path. Cancel runs in // informer handler goroutines, so the two can race without this. matcherMu sync.Mutex @@ -79,7 +79,7 @@ type Fluence struct { placement map[string]groupAlloc // excludedNodes maps a group key to the set of nodes that are GENUINELY // INCOMPATIBLE with that group (PostFilter saw UnschedulableAndUnresolvable - // from another plugin: a taint, affinity, or constraint Fluxion's graph does + // from another plugin. A taint, affinity, or constraint Fluxion's graph does // not model). PreFilter feeds them back as an RFC 31 negated-hostlist // constraint so the re-match is steered onto other nodes. Nodes that were // merely BUSY are deliberately NOT recorded here (excluding them would turn @@ -99,17 +99,17 @@ var ( // schedulableNodes returns only the nodes a normal pod could actually be placed // on, so the Fluxion graph never offers a node that Kubernetes will then reject -// in Filter. Two kinds are dropped: +// in Filter. Two kinds are dropped. // // - cordoned nodes (spec.unschedulable), and -// - nodes carrying a NoSchedule/NoExecute taint (e.g. the control-plane's +// - nodes with a NoSchedule/NoExecute taint (e.g. the control-plane's // node-role.kubernetes.io/control-plane:NoSchedule). // // Without this, Fluxion can place a gang slot on the control-plane (it looks like // a valid virtual=false compute node to the graph), the pod is then rejected by // TaintToleration with UnschedulableAndUnresolvable, and PostFilter abandons the -// whole allocation — on a small cluster that strands the gang permanently. We do -// not attempt to honor specific tolerations here: gang workloads in this setup do +// whole allocation, on a small cluster that strands the gang permanently. We do +// not attempt to honor specific tolerations here. Gang workloads in this setup do // not tolerate node taints, so any NoSchedule/NoExecute taint means "not for us". func schedulableNodes(nodes []corev1.Node) []corev1.Node { out := make([]corev1.Node, 0, len(nodes)) @@ -132,7 +132,7 @@ func schedulableNodes(nodes []corev1.Node) []corev1.Node { return out } -// New builds the plugin: discover cluster nodes, optionally inject quantum +// New builds the plugin. Discover cluster nodes, optionally inject quantum // resources, write the JGF graph, initialize the Fluxion matcher, and register // the delete handlers that cancel allocations when their owning object is gone. // @@ -142,7 +142,7 @@ func schedulableNodes(nodes []corev1.Node) []corev1.Node { func New(ctx context.Context, _ runtime.Object, h fwk.Handle) (fwk.Plugin, error) { // List nodes via the API. The scheduler's shared snapshot is empty at // plugin-construction time, so a direct List is what gives us the cluster's - // compute. Static cluster for now: read once, graph not updated live. + // compute. Static cluster for now. Read once, graph not updated live. nodeList, err := h.ClientSet().CoreV1().Nodes().List(ctx, metav1.ListOptions{}) if err != nil { return nil, fmt.Errorf("list nodes: %w", err) @@ -164,7 +164,7 @@ func New(ctx context.Context, _ runtime.Object, h fwk.Handle) (fwk.Plugin, error } opts.Resources = rc.Resources // The requestable device types are the FluxionResourceNames, minus - // the prefix — the suffixes a pod requests as + // the prefix, the suffixes a pod requests as // fluxion.flux-framework.org/. for _, name := range cluster.FluxionResourceNames(rc.Resources) { knownDevices[strings.TrimPrefix(name, placement.FluxionResourcePrefix)] = true @@ -194,12 +194,12 @@ func New(ctx context.Context, _ runtime.Object, h fwk.Handle) (fwk.Plugin, error } _ = tmp.Close() - // rv1 (full writer, with the scheduling key) is a superset of jgf: its + // rv1 (full writer, with the scheduling key) is a superset of jgf. Its // scheduling key is the same JGF vertex subgraph we parse for placement, and - // it carries the execution view flux uses to replay an allocation on restart. + // it has the execution view flux uses to replay an allocation on restart. // This is the format we persist and feed back to UpdateAllocate for recovery. - // jgf match format: emits every allocated vertex (with properties) as a graph, - // regardless of type. rv1 cannot represent our allocations — its R_lite is + // jgf match format. Emits every allocated vertex (with properties) as a graph, + // regardless of type. rv1 cannot represent our allocations, its R_lite is // built from core/gpu "reducer" children under a node, so a virtual allocation // that bottoms out in nodes (no cores) serializes to an empty R. jgf has no // such assumption and is exactly what PlacementFromAllocation parses (node @@ -218,7 +218,7 @@ func New(ctx context.Context, _ runtime.Object, h fwk.Handle) (fwk.Plugin, error // Periodic + startup reconcile of completed Fluence-created PodGroups, so a // gang's allocation is freed even if a completion/delete event is missed // (informer resync gaps, controller restart, force-deleted pods). The - // event-driven path (onPodUpdated) handles the common case promptly; this is + // event-driven path (onPodUpdated) handles the common case promptly. This is // the correctness backstop. go f.runReconcileSweeps(ctx) return f, nil @@ -226,7 +226,7 @@ func New(ctx context.Context, _ runtime.Object, h fwk.Handle) (fwk.Plugin, error // runReconcileSweeps reconciles all Fluence-created PodGroups on startup and then // periodically. On startup it also catches allocations that outlived a scheduler -// restart: a completed gang whose PodGroup still exists is deleted here, freeing +// restart a completed gang whose PodGroup still exists is deleted here, freeing // its allocation via onPodGroupDeleted (cancelGroup reads jobids from the // PodGroup annotation, which survives restart, so the free works even though the // in-memory placement memo was lost). @@ -269,7 +269,7 @@ func (f *Fluence) reconcileAll(ctx context.Context) { } // reconcileInterval is how often the backstop sweep runs. The event-driven path -// handles the prompt case; this only needs to be frequent enough to bound how +// handles the prompt case. This only needs to be frequent enough to bound how // long a leaked allocation can persist after a missed event. const reconcileInterval = 60 * time.Second @@ -277,7 +277,7 @@ const reconcileInterval = 60 * time.Second func (f *Fluence) Name() string { return Name } // PreFilter runs once per scheduling cycle for a pod. The first pod of a group -// triggers a single match for the whole gang; the resulting placement (and the +// triggers a single match for the whole gang. The resulting placement (and the // Fluxion jobid) is cached and consumed by Filter/PreBind for every pod. func (f *Fluence) PreFilter( ctx context.Context, @@ -313,7 +313,7 @@ func (f *Fluence) PreFilter( } // Run every jobspec as an independent held allocation (duration 0). The group - // is all-or-nothing: if any match fails, cancel the ones that already + // is all-or-nothing. If any match fails, cancel the ones that already // succeeded so we never hold a partial allocation (e.g. compute without its // device, or vice versa). place, jobids, status := f.matchGroup(specs) @@ -331,13 +331,13 @@ func (f *Fluence) PreFilter( } // matchGroup runs each jobspec as an independent held Fluxion allocation and -// combines them into one placement. It is all-or-nothing: on the first failure +// combines them into one placement. It is all-or-nothing. On the first failure // it cancels every allocation already made and returns an Unschedulable status, // so the group never holds a partial set (compute without its device, etc.). // -// The combined placement unions the per-match results: the compute match +// The combined placement unions the per-match results. The compute match // supplies the bind nodes, a device match supplies the backend identity. (The -// per-match split of nodes vs backend is PlacementFromAllocation's job; here we +// per-match split of nodes vs backend is PlacementFromAllocation's job. Here we // merge.) func (f *Fluence) matchGroup(specs []*jobspec.Jobspec) (placement.Placement, []uint64, *fwk.Status) { var combined placement.Placement @@ -350,7 +350,7 @@ func (f *Fluence) matchGroup(specs []*jobspec.Jobspec) (placement.Placement, []u // (e.g. "- virtual=false"), which the parser rejects with "non-string // property specified" -> the whole match fails with -1. JSON always quotes // strings, and JSON is valid YAML input to the matcher, so this is the - // reliable encoding for jobspecs that carry constraints. + // reliable encoding for jobspecs that have constraints. spec, err := js.JSON() if err != nil { f.cancelJobids(jobids) @@ -364,7 +364,7 @@ func (f *Fluence) matchGroup(specs []*jobspec.Jobspec) (placement.Placement, []u req, err := f.matcher.MatchAllocateSpec(spec) f.matcherMu.Unlock() if err != nil { - log.Printf("[fluence] MATCH %d/%d FAILED: %v — rolling back jobids %v", + log.Printf("[fluence] MATCH %d/%d FAILED: %v, rolling back jobids %v", i+1, len(specs), err, jobids) f.cancelJobids(jobids) return placement.Placement{}, nil, fwk.NewStatus( @@ -392,7 +392,7 @@ func (f *Fluence) matchGroup(specs []*jobspec.Jobspec) (placement.Placement, []u } if len(combined.Nodes) == 0 && combined.Backend == "" { - log.Printf("[fluence] match produced no nodes and no backend — unschedulable") + log.Printf("[fluence] match produced no nodes and no backend, unschedulable") f.cancelJobids(jobids) return placement.Placement{}, nil, fwk.NewStatus( fwk.Unschedulable, "fluxion returned no allocation") @@ -403,7 +403,7 @@ func (f *Fluence) matchGroup(specs []*jobspec.Jobspec) (placement.Placement, []u } // cancelJobids frees a set of held allocations, used to unwind a partial group -// match. Cancel is idempotent and best-effort; errors are logged, not returned. +// match. Cancel is idempotent and best-effort. Errors are logged, not returned. func (f *Fluence) cancelJobids(jobids []uint64) { for _, id := range jobids { f.matcherMu.Lock() @@ -415,7 +415,7 @@ func (f *Fluence) cancelJobids(jobids []uint64) { } } -// PreFilterExtensions: no add/remove pod handling for now. +// PreFilterExtensions. No add/remove pod handling for now. func (f *Fluence) PreFilterExtensions() fwk.PreFilterExtensions { return nil } // Filter permits a node only if Fluxion assigned it to this group. @@ -432,7 +432,7 @@ func (f *Fluence) Filter( f.mu.Unlock() // A quantum-only allocation pins no node (the backend is a remote API any - // node can reach), so impose no constraint; the qpu device plugin already + // node can reach), so impose no constraint. The qpu device plugin already // gates which nodes can admit the pod. if len(nodes) == 0 { return fwk.NewStatus(fwk.Success) @@ -446,7 +446,7 @@ func (f *Fluence) Filter( return fwk.NewStatus(fwk.Unschedulable, "node not in fluxion allocation for this group") } -// PostFilter runs when a pod could not be scheduled after Filter — for a Fluence +// PostFilter runs when a pod could not be scheduled after Filter, for a Fluence // group, this means the cached Fluxion allocation's nodes did not all survive the // other scheduler plugins' Filter checks. Without intervention the group would // retry forever against the same cached allocation while the Fluxion reservation @@ -457,21 +457,21 @@ func (f *Fluence) Filter( // the cached placement) so the next PreFilter re-matches fresh. The careful part // is WHICH nodes we then permanently exclude from the group's future matches, // because a group reaches PostFilter for two very different reasons and they must -// be handled oppositely (see fwk.Code docs): +// be handled oppositely (see fwk.Code docs). // -// - UnschedulableAndUnresolvable: the node genuinely cannot host this pod and +// - UnschedulableAndUnresolvable. The node genuinely cannot host this pod and // re-trying it is pointless (a taint the pod does not tolerate, node affinity -// mismatch, a constraint Fluxion's graph does not model). EXCLUDE it; the +// mismatch, a constraint Fluxion's graph does not model). EXCLUDE it. The // next PreFilter feeds the exclusion set back as an RFC 31 negated-hostlist // constraint so Fluxion is steered onto other nodes. // -// - Unschedulable (plain): the node could host the pod, just not at this -// instant (it is momentarily full). This is TRANSIENT. Do NOT exclude it — +// - Unschedulable (plain). The node could host the pod, just not at this +// instant (it is momentarily full). This is TRANSIENT. Do NOT exclude it // excluding a merely-busy node converts ordinary contention into permanent // group failure, and in a saturated cluster (a gang that needs the whole node // set) it strands the gang forever even though it would fit once a node frees. // -// So contention excludes nothing and the group recovers by waiting/retrying; +// So contention excludes nothing and the group recovers by waiting/retrying. // only durable incompatibility accumulates in excludedNodes (cleared on group // teardown), which keeps the exclusion-driven re-match finite and correct. func (f *Fluence) PostFilter( @@ -485,25 +485,25 @@ func (f *Fluence) PostFilter( f.mu.Lock() alloc, ok := f.placement[group] if !ok { - // No cached allocation for this group — nothing of ours to reconcile. + // No cached allocation for this group, nothing of ours to reconcile. // (Another plugin's PostFilter, or a non-group pod.) f.mu.Unlock() return nil, fwk.NewStatus(fwk.Unschedulable) } // Exclude ONLY nodes that are genuinely incompatible with this pod, never // nodes that were merely busy this cycle. The framework gives us a per-node - // status: UnschedulableAndUnresolvable means the node cannot host the pod and + // status. UnschedulableAndUnresolvable means the node cannot host the pod and // re-trying it is pointless (a taint the pod does not tolerate, node affinity // mismatch, a constraint Fluxion's graph does not model) -> exclude it so the // re-match is steered elsewhere. A plain Unschedulable means the node could // host the pod but not right now (it is momentarily full) -> do NOT exclude - // it; it must stay eligible so the group can land there once capacity frees. + // it. It must stay eligible so the group can land there once capacity frees. // - // This is the whole point: a group enters PostFilter for many reasons, and + // This is the whole point. A group enters PostFilter for many reasons, and // "the cluster is just full at this instant" is the common one. Permanently // banning the busy nodes (the old whole-allocation exclusion) turned transient - // contention into permanent group failure — exactly backwards. Now contention - // excludes nothing; the group simply abandons this cycle's reservation and + // contention into permanent group failure, exactly backwards. Now contention + // excludes nothing. The group simply abandons this cycle's reservation and // retries the same nodes when they free. if f.excludedNodes[group] == nil { f.excludedNodes[group] = map[string]bool{} @@ -537,7 +537,7 @@ func (f *Fluence) PostFilter( "incompatible(excluded)=%v busy(retryable, NOT excluded)=%v; %d node(s) excluded total", group, jobids, incompatible, busy, excludedCount) - // Returning Unschedulable (no nominated node) lets the pod be requeued; the + // Returning Unschedulable (no nominated node) lets the pod be requeued. The // next PreFilter re-matches (with any incompatible nodes excluded, but busy // nodes still in play). Fluxion, not PostFilter preemption, chooses placement. return nil, fwk.NewStatus(fwk.Unschedulable) @@ -562,13 +562,13 @@ func (f *Fluence) PreBindPreFlight( } // Reserve stamps the chosen backend (and matched attributes) onto the pod as -// early as possible — at reservation, in the scheduling cycle — rather than in +// early as possible, at reservation, in the scheduling cycle, rather than in // PreBind. The webhook injects FLUXION_BACKEND (and FLUXION_) as a -// downward-API env sourced from these annotations; downward-API env is resolved +// downward-API env sourced from these annotations. downward-API env is resolved // by the kubelet when the container starts and is NOT updated afterward, so the // annotation must be present well before the container starts. PreBind runs in // the (asynchronous) binding cycle, milliseconds before Bind, which races the -// kubelet — Reserve runs earlier and synchronously, giving the annotation time +// kubelet, Reserve runs earlier and synchronously, giving the annotation time // to propagate so the value reliably surfaces in the container. func (f *Fluence) Reserve( ctx context.Context, @@ -582,7 +582,7 @@ func (f *Fluence) Reserve( return fwk.NewStatus(fwk.Success) } -// Unreserve is a no-op: a stale backend annotation from a reservation that was +// Unreserve is a no-op. A stale backend annotation from a reservation that was // later rejected is harmless (it is overwritten on the next attempt and the // value is correct for the allocation that produced it), and clearing it would // cost an extra API call. Required to satisfy fwk.ReservePlugin. @@ -591,7 +591,7 @@ func (f *Fluence) Unreserve(ctx context.Context, state fwk.CycleState, pod *core // stampBackend writes the allocated backend name and matched attributes onto the // pod (idempotent merge patch). No-op when there is no cached allocation or the -// allocation carries no backend (classical, non-quantum gangs). +// allocation has no backend (classical, non-quantum gangs). func (f *Fluence) stampBackend(ctx context.Context, pod *corev1.Pod) error { f.mu.Lock() alloc, ok := f.placement[groupKey(pod)] @@ -608,12 +608,12 @@ func (f *Fluence) stampBackend(ctx context.Context, pod *corev1.Pod) error { return f.patchPodAnnotations(ctx, pod.Namespace, pod.Name, ann) } -// PreBind records, in the commit phase, the durable state for this group: +// PreBind records, in the commit phase, the durable state for this group. // the Fluxion jobid onto the owning object (the PodGroup for a gang, else the // pod) so the allocation can be cancelled when that object is deleted. The // backend annotation is stamped earlier, in Reserve (see stampBackend), because // the webhook-injected downward-API env (FLUXION_BACKEND) must be present before -// the container starts; PreBind is too late and races the kubelet. +// the container starts. PreBind is too late and races the kubelet. func (f *Fluence) PreBind( ctx context.Context, state fwk.CycleState, @@ -630,7 +630,7 @@ func (f *Fluence) PreBind( if err := f.recordJobIDs(ctx, pod, alloc.jobids); err != nil { return fwk.AsStatus(fmt.Errorf("record jobids: %w", err)) } - // Backstop: if Reserve was skipped for any reason, ensure the backend is + // Backstop. If Reserve was skipped for any reason, ensure the backend is // stamped before bind anyway (idempotent). if err := f.stampBackend(ctx, pod); err != nil { return fwk.AsStatus(fmt.Errorf("stamp backend annotations: %w", err)) @@ -639,8 +639,8 @@ func (f *Fluence) PreBind( } // recordJobIDs writes the jobid annotation (a comma-separated list of all the -// group's held allocations) onto the allocation's owning object: a grouped pod's -// allocation belongs to the PodGroup; an ungrouped pod owns its own. +// group's held allocations) onto the allocation's owning object. A grouped pod's +// allocation belongs to the PodGroup. An ungrouped pod owns its own. func (f *Fluence) recordJobIDs(ctx context.Context, pod *corev1.Pod, jobids []uint64) error { val := formatJobIDs(jobids) if group := placement.PodGroupName(pod); group != "" { @@ -704,7 +704,7 @@ func (f *Fluence) registerCancelHandlers() { } // podTerminal reports whether a pod has reached a terminal phase (run to -// completion or failed) — at which point its held allocation should be freed. +// completion or failed), at which point its held allocation should be freed. func podTerminal(p *corev1.Pod) bool { return p.Status.Phase == corev1.PodSucceeded || p.Status.Phase == corev1.PodFailed } @@ -712,7 +712,7 @@ func podTerminal(p *corev1.Pod) bool { // onPodUpdated frees an ungrouped pod's allocation when the pod finishes, and // for a GROUPED pod triggers gang cleanup when the whole group has finished. A // completed pod is not deleted, so this is the path that releases resources when -// a job ends normally; the DeleteFunc path remains as a backstop. Cancel and +// a job ends normally. The DeleteFunc path remains as a backstop. Cancel and // PodGroup deletion are both idempotent, so repeated triggers are safe. func (f *Fluence) onPodUpdated(oldObj, newObj interface{}) { oldPod, ok := oldObj.(*corev1.Pod) @@ -724,19 +724,19 @@ func (f *Fluence) onPodUpdated(oldObj, newObj interface{}) { return } // Only act on the transition INTO a terminal phase (UpdateFunc fires on every - // pod update; this avoids re-acting on each subsequent update). + // pod update. This avoids re-acting on each subsequent update). if podTerminal(oldPod) || !podTerminal(newPod) { return } group := placement.PodGroupName(newPod) if group == "" { - // Ungrouped pod: its allocation is owned by the pod; free on completion. + // Ungrouped pod. Its allocation is owned by the pod. Free on completion. f.cancelGroup(newPod.Namespace+"/"+newPod.Name, newPod.Annotations) return } - // Grouped pod: the gang's allocation is owned by the PodGroup, so we must not + // Grouped pod. The gang's allocation is owned by the PodGroup, so we must not // free it when a single member finishes. Instead, when the LAST member goes - // terminal, reconcile the group — which (for a Fluence-created PodGroup) + // terminal, reconcile the group, which (for a Fluence-created PodGroup) // deletes the PodGroup and lets onPodGroupDeleted free the allocation. f.reconcileGroup(context.Background(), newPod.Namespace, group) } @@ -746,19 +746,19 @@ func (f *Fluence) onPodUpdated(oldObj, newObj interface{}) { // the single decision point for "is this gang done, and may we clean it up?" and // is safe to call repeatedly (PodGroup deletion is idempotent). // -// Safety rules, in order: -// 1. The PodGroup must exist and carry the Fluence ownership annotation. A +// Safety rules, in order. +// 1. The PodGroup must exist and have the Fluence ownership annotation. A // user-created PodGroup is NEVER deleted, even if all its pods are terminal. -// 2. The gang must have actually materialized: at least one member pod must +// 2. The gang must have actually materialized. At least one member pod must // exist. This guards the window right after the leader is admitted, before -// the gated workers are created — we must not reap a group that has not yet +// the gated workers are created, we must not reap a group that has not yet // grown its members. // 3. EVERY member pod (gated or not) must be in a terminal phase. A single -// running or still-gated member keeps the gang alive — exactly the property +// running or still-gated member keeps the gang alive, exactly the property // that prevents freeing resources out from under workers that ungated after // the leader finished. func (f *Fluence) reconcileGroup(ctx context.Context, namespace, group string) { - // Defensive: the scheduler handle (and thus the API client / informers) may + // Defensive. The scheduler handle (and thus the API client / informers) may // be absent in unit tests or before the framework is fully wired. Reconcile // needs the API to read/delete PodGroups, so without a handle it is a no-op // rather than a nil dereference. @@ -771,7 +771,7 @@ func (f *Fluence) reconcileGroup(ctx context.Context, namespace, group string) { return // gone already, or not ours to inspect } if pg.Annotations[placement.CreatedByAnnotation] != placement.CreatedByValue { - return // rule 1: not Fluence-created — never delete a user's PodGroup + return // rule 1: not Fluence-created, never delete a user's PodGroup } members, err := f.handle.SharedInformerFactory().Core().V1().Pods().Lister(). @@ -793,31 +793,31 @@ func (f *Fluence) reconcileGroup(ctx context.Context, namespace, group string) { } } - // Rule 2 (refined): distinguish two "zero live members" states, which member - // count alone cannot: - // (a) brand-new gang: PodGroup just created, workers not yet created. Must - // NOT reap — that would kill a gang before it starts. - // (b) finished/abandoned gang: pods completed and were already removed, + // Rule 2 (refined). Distinguish two "zero live members" states, which member + // count alone cannot. + // (a) brand-new gang. PodGroup just created, workers not yet created. Must + // NOT reap, that would kill a gang before it starts. + // (b) finished/abandoned gang. Pods completed and were already removed, // leaving the PodGroup holding a Fluxion allocation. MUST reap, or the // allocation leaks and blocks future gangs ("qpu match failed -1"). - // We tell them apart by whether the gang was ever scheduled: a scheduled gang - // carries the jobid annotation (written in PreBind). A jobid + zero live + // We tell them apart by whether the gang was ever scheduled. A scheduled gang + // has the jobid annotation (written in PreBind). A jobid + zero live // members ⇒ case (b), reap. No jobid + zero members ⇒ case (a) or never - // scheduled; protect it with a creation grace period as a backstop. + // scheduled. Protect it with a creation grace period as a backstop. if seen == 0 { _, scheduled := parseJobIDs(pg.Annotations) if !scheduled { if time.Since(pg.CreationTimestamp.Time) < reconcileGraceForEmpty { return // case (a): brand-new, give workers time to appear } - // old, never scheduled, no members — nothing was ever allocated, but + // old, never scheduled, no members, nothing was ever allocated, but // it is safe (and tidy) to remove a stale empty PodGroup we created. } - // else: scheduled (has jobid) with no live members ⇒ case (b), reap. + // else scheduled (has jobid) with no live members ⇒ case (b), reap. } // All members terminal (or a scheduled gang whose pods are already gone) and - // the PodGroup is ours: delete it. onPodGroupDeleted then frees the Fluxion + // the PodGroup is ours. Delete it. onPodGroupDeleted then frees the Fluxion // allocation (cancelGroup reads jobids from the annotation). if err := f.handle.ClientSet().SchedulingV1alpha2().PodGroups(namespace). Delete(ctx, group, metav1.DeleteOptions{}); err != nil { @@ -825,14 +825,14 @@ func (f *Fluence) reconcileGroup(ctx context.Context, namespace, group string) { namespace, group, err) return } - log.Printf("fluence: reconciled completed gang %s/%s — deleted Fluence-created PodGroup, allocation freed", + log.Printf("fluence: reconciled completed gang %s/%s, deleted Fluence-created PodGroup, allocation freed", namespace, group) - // Scout-group cleanup: in shared coordination the gang is split into the + // Scout-group cleanup. In shared coordination the gang is split into the // worker group (this group) and the scout's group-of-one // -scout (a Fluence-created PodGroup, minCount 1). The scout POD // is a real member of the user's workload (indexed-Job index 0), so we must - // NOT delete it — only its Fluence-created PodGroup, as a backstop to free its + // NOT delete it, only its Fluence-created PodGroup, as a backstop to free its // allocation (its own reconcile pass also reaps it once the scout pod is // terminal). Skip when this group is itself a scout group, to avoid // recursing on -scout-scout. @@ -852,10 +852,10 @@ const reconcileGraceForEmpty = 2 * time.Minute // package (the scheduler must not depend on the webhook). Kept in sync with it. const webhookGroupLabel = "fluence.flux-framework.org/group" -// scoutGroupSuffix mirrors handlers.ScoutGroupSuffix: in shared +// scoutGroupSuffix mirrors handlers.ScoutGroupSuffix. In shared // coordination the scout (indexed-Job index 0) is its own group-of-one named // -scout. Duplicated here to avoid importing the webhook handlers package -// into the scheduler plugin; keep the two in sync. +// into the scheduler plugin. Keep the two in sync. const scoutGroupSuffix = "-scout" // onPodGroupDeleted frees the gang's allocation when its PodGroup is deleted. @@ -874,7 +874,7 @@ func (f *Fluence) onPodGroupDeleted(obj interface{}) { } // onPodDeleted frees an ungrouped pod's allocation when the pod is deleted. -// Grouped pods are ignored: their allocation is owned by the PodGroup and is +// Grouped pods are ignored. Their allocation is owned by the PodGroup and is // freed only when the PodGroup is deleted (freeing it on a single pod's delete // would release the whole gang's resources while its other pods still run). func (f *Fluence) onPodDeleted(obj interface{}) { @@ -889,11 +889,11 @@ func (f *Fluence) onPodDeleted(obj interface{}) { } } if g := placement.PodGroupName(pod); g != "" { - // Grouped pod deleted: do not free the gang's allocation on a single - // pod's removal, but DO reconcile — if this was the last member, the + // Grouped pod deleted. Do not free the gang's allocation on a single + // pod's removal, but DO reconcile, if this was the last member, the // PodGroup is now empty-but-allocated and reconcileGroup will reap it // (and free the allocation). This closes the gap where a completed gang's - // pods are deleted (not just completed): onPodUpdated never fires for a + // pods are deleted (not just completed). onPodUpdated never fires for a // deletion, so without this the cleanup would wait for the periodic sweep. f.reconcileGroup(context.Background(), pod.Namespace, g) return @@ -902,7 +902,7 @@ func (f *Fluence) onPodDeleted(obj interface{}) { } // cancelGroup frees all allocations for a deleted owning object. The jobids come -// from the object's annotation (the durable source of truth); if it is missing +// from the object's annotation (the durable source of truth). If it is missing // (e.g. deleted between PreFilter and PreBind, before the annotation was // written) it falls back to the in-memory memo by key. Cancel is idempotent. func (f *Fluence) cancelGroup(key string, ann map[string]string) { @@ -962,11 +962,11 @@ func parseJobIDs(ann map[string]string) ([]uint64, bool) { // list the namespace and filter in code. A pod with no scheduling group is its // own group of one. // -// Gated pods are excluded: a pod still carrying a scheduling gate is held at +// Gated pods are excluded. A pod that still has a scheduling gate is held at // PreEnqueue and is not part of the live scheduling cycle. In a quantum gang the // workers stay gated until the sidecar ungates them at QPU position==1, so // including them here would make the leader's match reserve resources for pods -// that are not yet ready to run — defeating the whole point of gating. An +// that are not yet ready to run, defeating the whole point of gating. An // ungated pod re-enters the queue and is matched then. func (f *Fluence) groupPods(pod *corev1.Pod) ([]corev1.Pod, error) { group := placement.PodGroupName(pod) @@ -983,7 +983,7 @@ func (f *Fluence) groupPods(pod *corev1.Pod) ([]corev1.Pod, error) { if placement.PodGroupName(p) != group { continue } - // Skip pods still holding a scheduling gate — they are not in the live + // Skip pods still holding a scheduling gate, they are not in the live // scheduling cycle yet. The pod currently being scheduled (pod) has // already cleared its gates by the time PreFilter runs, so it is included. if len(p.Spec.SchedulingGates) > 0 { diff --git a/pkg/fluence/fluence_test.go b/pkg/fluence/fluence_test.go index 5228f97..151bc5a 100644 --- a/pkg/fluence/fluence_test.go +++ b/pkg/fluence/fluence_test.go @@ -60,7 +60,7 @@ func ann(jobid string) map[string]string { } // groupedPod returns a pod that belongs to the named native PodGroup. -// NOTE: corev1.PodSchedulingGroup is the k8s 1.36 native gang field; if the +// Note that corev1.PodSchedulingGroup is the k8s 1.36 native gang field. If the // type name differs in your vendored k8s, adjust just this constructor. func groupedPod(ns, name, group string, annotations map[string]string) *corev1.Pod { g := group @@ -166,7 +166,7 @@ func TestCancelGroupUnknownNoop(t *testing.T) { } } -// Cancel is idempotent: a redelivered delete event for an already-freed group +// Cancel is idempotent. A redelivered delete event for an already-freed group // does nothing the second time. func TestCancelGroupIdempotent(t *testing.T) { m := &fakeMatcher{} @@ -209,7 +209,7 @@ func TestOnPodDeletedUngroupedCancels(t *testing.T) { } } -// Deleting a grouped pod must NOT cancel: the allocation belongs to the +// Deleting a grouped pod must NOT cancel. The allocation belongs to the // PodGroup and is freed only when the PodGroup is deleted. Cancelling here would // free the gang's allocation while other pods still hold it. func TestOnPodDeletedGroupedDoesNotCancelDirectly(t *testing.T) { @@ -219,9 +219,9 @@ func TestOnPodDeletedGroupedDoesNotCancelDirectly(t *testing.T) { // A grouped pod being deleted must NOT directly cancel the gang's allocation // (the allocation is owned by the PodGroup, freed only when the whole gang is // done). onPodDeleted now triggers reconcileGroup instead of ignoring the - // pod, but reconcile frees nothing here: there is no scheduler handle in this - // unit context, so reconcile is a safe no-op. The invariant under test — - // "one grouped pod's deletion never frees the gang" — still holds. + // pod, but reconcile frees nothing here. There is no scheduler handle in this + // unit context, so reconcile is a safe no-op. The invariant under test + // "one grouped pod's deletion never frees the gang", still holds. f.onPodDeleted(groupedPod("default", "training-abc", "training", ann("5"))) if len(m.cancelled) != 0 { @@ -344,7 +344,7 @@ func TestCancelGroupMultipleJobids(t *testing.T) { } // twoSpecs returns two minimal jobspecs (compute + device) to drive matchGroup -// tests; their content doesn't matter since the fake matcher scripts results. +// tests. Their content doesn't matter since the fake matcher scripts results. func twoSpecs() []*jobspec.Jobspec { return []*jobspec.Jobspec{ {Version: 9999}, @@ -354,7 +354,7 @@ func twoSpecs() []*jobspec.Jobspec { // --- PostFilter allocation reconciliation ----------------------------------- -// fakeNodeStatus is a minimal fwk.NodeToStatusReader for PostFilter tests: it +// fakeNodeStatus is a minimal fwk.NodeToStatusReader for PostFilter tests. It // maps node name -> status code so a test can mark some nodes incompatible // (UnschedulableAndUnresolvable) and others merely busy (Unschedulable). type fakeNodeStatus map[string]fwk.Code @@ -382,7 +382,7 @@ func TestPostFilterExcludesOnlyIncompatibleNodes(t *testing.T) { } pod := groupedPod("default", "training-0", "training", nil) - // node-a incompatible (taint); node-b busy; node-c survived Filter. + // node-a incompatible (taint). node-b busy. node-c survived Filter. status := fakeNodeStatus{ "node-a": fwk.UnschedulableAndUnresolvable, "node-b": fwk.Unschedulable, @@ -412,7 +412,7 @@ func TestPostFilterExcludesOnlyIncompatibleNodes(t *testing.T) { } // A group blocked purely by contention (every node merely busy) excludes NOTHING -// so it can retry the same nodes once they free — the saturated-cluster property. +// so it can retry the same nodes once they free, the saturated-cluster property. func TestPostFilterContentionExcludesNothing(t *testing.T) { m := &fakeMatcher{} f := newTestFluence(m) @@ -455,7 +455,7 @@ func TestPostFilterNilStatusMapExcludesNothing(t *testing.T) { } } -// Incompatible nodes accumulate across attempts; busy ones never do. +// Incompatible nodes accumulate across attempts. Busy ones never do. func TestPostFilterAccumulatesIncompatibleAcrossAttempts(t *testing.T) { m := &fakeMatcher{} f := newTestFluence(m) @@ -482,7 +482,7 @@ func TestPostFilterAccumulatesIncompatibleAcrossAttempts(t *testing.T) { } // PostFilter on a group with no cached allocation (not ours, or already cleared) -// is a safe no-op: no panic, no cancel, returns Unschedulable. +// is a safe no-op. No panic, no cancel, returns Unschedulable. func TestPostFilterUnknownGroupNoop(t *testing.T) { m := &fakeMatcher{} f := newTestFluence(m) diff --git a/pkg/graph/allocation.go b/pkg/graph/allocation.go index 2d77784..87d2189 100644 --- a/pkg/graph/allocation.go +++ b/pkg/graph/allocation.go @@ -6,16 +6,16 @@ import ( ) // vertexMeta is the subset of a JGF vertex metadata fluence needs from an -// allocation: type, name, and the RFC 31 properties object. Properties are +// allocation type, name, and the RFC 31 properties object. Properties are // stored as composed "key=value" keys (matching how the graph is built and how -// the rv1 writer emits them); the value half of the JSON object is unused. +// the rv1 writer emits them). The value half of the JSON object is unused. type vertexMeta struct { Type string `json:"type"` Name string `json:"name"` Properties map[string]string `json:"properties"` } -// graphVertices is the subset of a JGF graph we need: the metadata of each +// graphVertices is the subset of a JGF graph we need. The metadata of each // vertex. It appears at the top level of a "jgf" allocation, and under the // "scheduling" key of an "rv1" allocation. type graphVertices struct { @@ -25,7 +25,7 @@ type graphVertices struct { } // allocation captures the allocated vertices from a Fluxion allocation R in -// either match format: +// either match format. // - "jgf": the vertices are at the top level under "graph". // - "rv1": the vertices are under "scheduling.graph" (the rv1 scheduling key), // alongside an "execution" view we don't need for name extraction. @@ -61,7 +61,7 @@ type AllocatedNode struct { Properties map[string]string } -// HasProperty reports whether the node carries the given composed property key +// HasProperty reports whether the node has the given composed property key // (e.g. "virtual=true"). func (n AllocatedNode) HasProperty(key string) bool { _, ok := n.Properties[key] @@ -71,7 +71,7 @@ func (n AllocatedNode) HasProperty(key string) bool { // NodesFromAllocation returns every node-typed vertex in an allocation together // with its properties, so callers can classify them (physical vs virtual) by the // composed marker keys rather than by vertex type. Under the virtual-resource -// model every allocatable thing is type "node"; the virtual marker, not the +// model every allocatable thing is type "node". The virtual marker, not the // type, distinguishes a compute node from a virtual backend. func NodesFromAllocation(alloc string) ([]AllocatedNode, error) { var a allocation @@ -89,7 +89,7 @@ func NodesFromAllocation(alloc string) ([]AllocatedNode, error) { // NamesFromAllocation returns the names of every vertex of vertexType in a // Fluxion allocation (jgf or rv1). Retained for callers that key on a concrete -// vertex type (e.g. counting "qpu" children); placement now classifies node +// vertex type (e.g. counting "qpu" children). Placement now classifies node // vertices via NodesFromAllocation and the virtual marker instead. func NamesFromAllocation(alloc string, vertexType string) ([]string, error) { var a allocation diff --git a/pkg/graph/allocation_test.go b/pkg/graph/allocation_test.go index 1ddbfca..ecaa3e2 100644 --- a/pkg/graph/allocation_test.go +++ b/pkg/graph/allocation_test.go @@ -2,13 +2,13 @@ package graph import "testing" -// jgf match format: vertices at top-level graph. +// jgf match format. Vertices at top-level graph. const jgfAlloc = `{"graph":{"nodes":[ {"metadata":{"type":"node","name":"kind-worker"}}, {"metadata":{"type":"core","name":"core0"}}, {"metadata":{"type":"qpu","name":"ibm_marrakesh"}}]}}` -// rv1 match format: execution view + scheduling key holding the same vertices. +// rv1 match format. Execution view + scheduling key holding the same vertices. const rv1Alloc = `{"version":1, "execution":{"R_lite":[{"rank":"0","children":{"core":"0"}}],"nodelist":["kind-worker"]}, "scheduling":{"graph":{"nodes":[ @@ -38,8 +38,8 @@ func TestNamesFromAllocationRV1(t *testing.T) { } } -// markedAlloc has node vertices carrying composed virtual markers plus a qpu -// child — the shape PlacementFromAllocation classifies. +// markedAlloc has node vertices with composed virtual markers plus a qpu +// child, the shape PlacementFromAllocation classifies. const markedAlloc = `{"graph":{"nodes":[ {"metadata":{"type":"node","name":"kind-worker","properties":{"virtual=false":""}}}, {"metadata":{"type":"node","name":"rigetti","properties":{"virtual=true":"","class=qdevice":""}}}, @@ -70,6 +70,6 @@ func TestNodesFromAllocation(t *testing.T) { t.Errorf("rigetti missing class=qdevice: %v", byName["rigetti"].Properties) } if byName["kind-worker"].HasProperty("virtual=true") { - t.Error("kind-worker should not carry virtual=true") + t.Error("kind-worker should not have virtual=true") } } diff --git a/pkg/graph/graph.go b/pkg/graph/graph.go index a503c7a..7c0b6db 100644 --- a/pkg/graph/graph.go +++ b/pkg/graph/graph.go @@ -27,7 +27,7 @@ type FluxionGraph struct { cli *fluxcli.ReapiClient // MatchFormat selects the Fluxion allocation output format ("simple", - // "jgf", "rv1", ...). Empty defaults to "simple" (human-readable tree). + // "jgf", "rv1"...). Empty defaults to "simple" (human-readable tree). // Set it to "jgf" when you need to parse the allocation programmatically // (e.g. BackendFromAllocation). MatchFormat string @@ -52,7 +52,7 @@ func (f *FluxionGraph) Init(confFile string, matchPolicy string, label string) { matchPolicy = "first" } - // Allocation output format; default to the human-readable tree. + // Allocation output format. Default to the human-readable tree. matchFormat := f.MatchFormat if matchFormat == "" { matchFormat = "simple" @@ -66,7 +66,7 @@ func (f *FluxionGraph) Init(confFile string, matchPolicy string, label string) { // 2. Create the context, specify instead of JGF (default) we want graphml // 3. Remainder of defaults should work out of the box - // Note that the options get passed as a json string to here: + // Note that the options get passed as a json string to here. // https://github.com/flux-framework/flux-sched/blob/master/resource/reapi/bindings/c%2B%2B/reapi_cli_impl.hpp#L412 opts := `{"matcher_policy": "%s", "load_file": "%s", "load_format": "jgf", "match_format": "%s"}` p := fmt.Sprintf(opts, matchPolicy, confFile, matchFormat) @@ -90,7 +90,7 @@ func (f *FluxionGraph) MatchAllocate(specFile string) (MatchAllocateRequest, err } // MatchAllocateSpec match-allocates against a jobspec provided as a string. -// Fluxion accepts YAML or JSON; use the jobspec package to convert/normalize if +// Fluxion accepts YAML or JSON. Use the jobspec package to convert/normalize if // needed before calling this. func (f *FluxionGraph) MatchAllocateSpec(spec string) (MatchAllocateRequest, error) { request := MatchAllocateRequest{} @@ -111,7 +111,7 @@ func (f *FluxionGraph) MatchAllocateSpec(spec string) (MatchAllocateRequest, err } // Cancel frees a previously match-allocated jobid in the resource graph. It is -// idempotent: cancelling an unknown jobid is not treated as an error (noent_ok), +// idempotent cancelling an unknown jobid is not treated as an error (noent_ok), // so a double-cancel (e.g. a redelivered informer event) is harmless. func (f *FluxionGraph) Cancel(jobid uint64) error { fmt.Printf(" 🌀 Cancel jobid: %d\n", jobid) @@ -120,10 +120,10 @@ func (f *FluxionGraph) Cancel(jobid uint64) error { // UpdateAllocate replays a previously returned allocation R into the freshly // built graph under the given jobid, marking exactly the vertices named in R as -// allocated — no re-matching, no guessing. This is how a scheduler restart -// restores prior state: persist (jobid, R) when the allocation is made, rebuild +// allocated, no re-matching, no guessing. This is how a scheduler restart +// restores prior state. Persist (jobid, R) when the allocation is made, rebuild // the graph (empty of allocations) on startup, then UpdateAllocate each -// surviving allocation. R must be the rv1 form (with the scheduling key); the +// surviving allocation. R must be the rv1 form (with the scheduling key). The // jobid is preserved, so a later Cancel(jobid) still frees it. func (f *FluxionGraph) UpdateAllocate(jobid uint64, R string) error { fmt.Printf(" 🌀 UpdateAllocate (replay) jobid: %d\n", jobid) diff --git a/pkg/graph/graph_cancel_test.go b/pkg/graph/graph_cancel_test.go index 1016d86..5c1f32b 100644 --- a/pkg/graph/graph_cancel_test.go +++ b/pkg/graph/graph_cancel_test.go @@ -1,8 +1,8 @@ //go:build cgo // Package graph cancel test. This exercises the REAL Fluxion matcher (cgo + -// flux-sched), so it only builds/runs where flux-sched is linkable — i.e. in CI -// inside fluence-base, or the devcontainer. It verifies cancel's actual effect: +// flux-sched), so it only builds/runs where flux-sched is linkable, i.e. in CI +// inside fluence-base, or the devcontainer. It verifies cancel's actual effect. // a cancelled jobid frees its allocation so the resource can be matched again. package graph @@ -13,7 +13,7 @@ import ( ) // A tiny graph with a single exclusive core, so the first match consumes it and -// a second match must fail — until we cancel the first. +// a second match must fail, until we cancel the first. const tinyGraph = `{ "graph": { "nodes": [ @@ -63,7 +63,7 @@ func newGraph(t *testing.T) *FluxionGraph { return g } -// TestCancelFreesAllocation is the real "cancel of the jobid" check: match the +// TestCancelFreesAllocation is the real "cancel of the jobid" check. Match the // only core (consuming it), confirm a second match is refused, cancel the first // jobid, then confirm the core can be matched again. This proves Cancel actually // releases the allocation in the Fluxion graph, not just that it returns nil. @@ -79,7 +79,7 @@ func TestCancelFreesAllocation(t *testing.T) { t.Fatalf("expected a nonzero jobid from first match") } - // 2. Second match must be refused — the core is taken. + // 2. Second match must be refused, the core is taken. if _, err := g.MatchAllocateSpec(coreJobspec); err == nil { t.Fatal("second match should fail while the core is allocated, but it succeeded") } @@ -89,7 +89,7 @@ func TestCancelFreesAllocation(t *testing.T) { t.Fatalf("cancel jobid %d failed: %v", req1.Number, err) } - // 4. The core is free again: a fresh match must now succeed. + // 4. The core is free again. A fresh match must now succeed. req2, err := g.MatchAllocateSpec(coreJobspec) if err != nil { t.Fatalf("match after cancel failed (cancel did not free the allocation): %v", err) diff --git a/pkg/graph/run_test.go b/pkg/graph/run_test.go index 7f10cb5..c7886aa 100644 --- a/pkg/graph/run_test.go +++ b/pkg/graph/run_test.go @@ -5,7 +5,7 @@ import ( ) func TestBackendFromAllocation(t *testing.T) { - // Shape mirrors a Fluxion allocation graph: a qpu vertex named after the + // Shape mirrors a Fluxion allocation graph. A qpu vertex named after the // QRMI backend, sitting under a gateway under the cluster. alloc := `{"graph":{"nodes":[ {"metadata":{"type":"cluster","name":"tiny0"}}, diff --git a/pkg/jgf/jgf.go b/pkg/jgf/jgf.go index 05b0507..617ae70 100644 --- a/pkg/jgf/jgf.go +++ b/pkg/jgf/jgf.go @@ -1,5 +1,5 @@ // Package jgf builds a Fluxion JSON Graph Format (JGF) resource graph -// programmatically. The fluxion-quantum POC used hand-written JGF files; a +// programmatically. The fluxion-quantum POC used hand-written JGF files. A // scheduler has to generate the graph at runtime from the live cluster, so this // package provides a builder that assigns vertex ids, uniq ids, and containment // paths automatically. @@ -15,7 +15,7 @@ import ( const containment = "containment" // Vertex is a handle returned by the builder. Callers use it to attach children -// and edges; they do not set ids or paths themselves. +// and edges. They do not set ids or paths themselves. type Vertex struct { key string // graph node id (stringified uniq id) path string // containment path, e.g. /cluster0/node0/core @@ -67,7 +67,7 @@ type Options struct { // flattens descriptive fields into the top level), these populate // resource_t.properties in Fluxion and are what the matcher's constraint // pruning (by_constraint) and queries match against. Keys/values are - // strings; a bare tag is value "" (matching is by key presence). + // strings. A bare tag is value "" (matching is by key presence). NodeProperties map[string]string // Rank is the flux execution rank for a node-typed vertex. The rv1 match // writer builds its R_lite/nodelist from node vertices keyed by rank, and @@ -160,7 +160,7 @@ func (m metadata) MarshalJSON() ([]byte, error) { out[k] = val } } - // RFC 31 resource properties: a nested object Fluxion loads into + // RFC 31 resource properties. A nested object Fluxion loads into // resource_t.properties (what constraint pruning and queries match on). // Emitted only when present so vertices without properties are unchanged. if len(m.v.nodeProps) > 0 { diff --git a/pkg/jgf/jgf_test.go b/pkg/jgf/jgf_test.go index 89bf4b2..12f9799 100644 --- a/pkg/jgf/jgf_test.go +++ b/pkg/jgf/jgf_test.go @@ -40,7 +40,7 @@ func findByName(nodes []map[string]any, name string) map[string]any { } // A vertex given NodeProperties emits a nested "properties" object with those -// keys/values — the RFC 31 form Fluxion matches constraints against. +// keys/values, the RFC 31 form Fluxion matches constraints against. func TestNodePropertiesEmittedAsNestedObject(t *testing.T) { b := NewBuilder() cluster := b.AddRoot("cluster", "cluster", Options{Name: "cluster"}) @@ -84,7 +84,7 @@ func TestNoNodePropertiesMeansNoPropertiesKey(t *testing.T) { } } -// NodeProperties is distinct from Properties: the latter still flattens into the +// NodeProperties is distinct from Properties. The latter still flattens into the // top level, the former nests under "properties". func TestPropertiesVsNodeProperties(t *testing.T) { b := NewBuilder() diff --git a/pkg/jobspec/jobspec.go b/pkg/jobspec/jobspec.go index 2a27436..620f4fa 100644 --- a/pkg/jobspec/jobspec.go +++ b/pkg/jobspec/jobspec.go @@ -7,7 +7,7 @@ // y, err := js.YAML() // render as YAML // j, err := js.JSON() // render as JSON // -// One set of `json` struct tags drives both formats: sigs.k8s.io/yaml maps YAML +// One set of `json` struct tags drives both formats. sigs.k8s.io/yaml maps YAML // onto those tags (and JSON is a subset of YAML), and encoding/json uses them // directly. package jobspec @@ -49,7 +49,7 @@ type Task struct { } // Load parses a jobspec from a YAML or JSON string. JSON is a subset of YAML, -// so the YAML decoder accepts both; detection is only used to label errors. +// so the YAML decoder accepts both. Detection is only used to label errors. func Load(data string) (*Jobspec, error) { var js Jobspec if err := yaml.Unmarshal([]byte(data), &js); err != nil { diff --git a/pkg/placement/placement.go b/pkg/placement/placement.go index 0a13571..aeed7bf 100644 --- a/pkg/placement/placement.go +++ b/pkg/placement/placement.go @@ -13,8 +13,8 @@ import ( const ( // FluxionResourcePrefix marks an extended resource whose suffix is a Fluxion // graph type. A request for fluxion.flux-framework.org/ is translated - // generically into a jobspec count of — no per-type code. Anything the - // graph models as a count (qpu, qubit, ...) is requestable this way. + // generically into a jobspec count of , no per-type code. Anything the + // graph models as a count (qpu, qubit...) is requestable this way. FluxionResourcePrefix = "fluxion.flux-framework.org/" // BackendAnnotation is where the scheduler records the Fluxion-allocated @@ -23,16 +23,16 @@ const ( BackendAnnotation = "fluence.flux-framework.org/backend" // JobIDAnnotation records the Fluxion allocation (jobid) for a scheduled - // group. It is written onto the owning object — the PodGroup for a gang, or - // the pod itself for an ungrouped pod — so the allocation can be cancelled + // group. It is written onto the owning object, the PodGroup for a gang, or + // the pod itself for an ungrouped pod, so the allocation can be cancelled // when that object is deleted, and replayed on scheduler restart. JobIDAnnotation = "fluence.flux-framework.org/jobid" // CreatedByAnnotation marks an object as created by Fluence (the mutating // webhook). The scheduler's PodGroup reconciler deletes a completed gang's - // PodGroup — to free its Fluxion allocation — ONLY when this annotation is + // PodGroup, to free its Fluxion allocation, ONLY when this annotation is // present with CreatedByValue. A PodGroup the user created (e.g. a native - // gang) never carries it, so the reconciler never deletes user objects. + // gang) never has it, so the reconciler never deletes user objects. CreatedByAnnotation = "fluence.flux-framework.org/created-by" CreatedByValue = "fluence-webhook" @@ -45,13 +45,13 @@ const ( AttributeAnnotationPrefix = "fluence.flux-framework.org/attr-" // EnvVarPrefix is the normalized environment-variable namespace injected into - // the workload. The backend name becomes BACKEND; each attribute + // the workload. The backend name becomes BACKEND. Each attribute // becomes (uppercased). A vendor-agnostic container reads // these common names regardless of which backend it matched. EnvVarPrefix = "FLUXION_" ) -// EnvVarName maps an attribute key to its normalized environment-variable name: +// EnvVarName maps an attribute key to its normalized environment-variable name. // uppercased, non-alphanumeric runes to underscores, under EnvVarPrefix. E.g. // "region" -> "FLUXION_REGION", "price/min" -> "FLUXION_PRICE_MIN". func EnvVarName(attrKey string) string { @@ -73,7 +73,7 @@ func EnvVarName(attrKey string) string { // PodGroupName returns the native (Kubernetes 1.36) scheduling-group name a pod // belongs to, from spec.schedulingGroup.podGroupName, or "" if the pod is not // part of a group. This is the first-class field that links a Pod to its -// PodGroup object; the pre-1.36 label/annotation pattern is gone. +// PodGroup object. The pre-1.36 label/annotation pattern is gone. func PodGroupName(pod *corev1.Pod) string { if sg := pod.Spec.SchedulingGroup; sg != nil && sg.PodGroupName != nil { return *sg.PodGroupName @@ -84,7 +84,7 @@ func PodGroupName(pod *corev1.Pod) string { // ComputeProperty / VirtualProperty mirror the markers cluster stamps on graph // node vertices (composed key=value form). A compute jobspec constrains to // virtual=false nodes; a device jobspec constrains to virtual=true nodes. Kept -// here (not imported from cluster) to avoid an import cycle: cluster imports +// here (not imported from cluster) to avoid an import cycle. Cluster imports // placement. const ( // VirtualPropertyTrue selects configured virtual resource nodes. @@ -93,7 +93,7 @@ const ( VirtualPropertyFalse = "virtual=false" // ClassPropertyPrefix composes a class= constraint selecting a virtual node by // its configured resource type (e.g. class=qpu). Mirrors how the graph builder - // stamps the class property; kept here (not imported from cluster) to avoid an + // stamps the class property. Kept here (not imported from cluster) to avoid an // import cycle (cluster imports placement). ClassPropertyPrefix = "class=" ) @@ -104,10 +104,10 @@ func ComposeClassProperty(resourceType string) string { } // RequireAnnotationPrefix is how a workload constrains which backend Fluxion may -// select: an annotation fluence.flux-framework.org/require-: on the +// select. An annotation fluence.flux-framework.org/require-: on the // leader pod adds a property constraint fluxion.flux-framework.org/= // to the device match. e.g. require-qrmi_type: braket-gate keeps a gate workload -// off an AHS device; require-vendor: amazon, require-backend: sv1, etc. +// off an AHS device. require-vendor. Amazon, require-backend. sv1, etc. const RequireAnnotationPrefix = "fluence.flux-framework.org/require-" // ComposeAttributeProperty builds the graph property string for an attribute @@ -117,7 +117,7 @@ func ComposeAttributeProperty(attr, value string) string { } // requireConstraintsFromPods collects fluence.flux-framework.org/require- -// annotations across the group (the leader carries them) into graph property +// annotations across the group (the leader has them) into graph property // constraints to AND into the device match. func requireConstraintsFromPods(pods []corev1.Pod) []string { seen := map[string]bool{} @@ -146,7 +146,7 @@ var computeTypes = map[string]bool{"core": true, "memory": true, "gpu": true} // podResources distills a pod's container requests into Fluxion resource counts // keyed by Fluxion graph type. Native Kubernetes resources (cpu, memory, -// nvidia.com/gpu) map to compute types; every fluxion.flux-framework.org/ +// nvidia.com/gpu) map to compute types. Every fluxion.flux-framework.org/ // request passes through generically as (a virtual device type). func podResources(p *corev1.Pod) map[string]int { counts := map[string]int{} @@ -214,10 +214,10 @@ func withEntries(counts map[string]int) []jobspec.Resource { return with } -// systemAttributes builds the attributes.system block: a hold-until-cancel +// systemAttributes builds the attributes.system block. A hold-until-cancel // allocation (duration 0 runs to graph end) plus an RFC 31 property constraint // selecting the eligible node set. properties is the AND-set of composed -// key=value property strings a matched node must carry. +// key=value property strings a matched node must have. func systemAttributes(properties []string, excludeNodes []string) map[string]interface{} { // Base property constraint (the eligible-node property AND-set). constraints := map[string]interface{}{ @@ -252,7 +252,7 @@ func systemAttributes(properties []string, excludeNodes []string) map[string]int } } -// computeJobspec builds the physical-compute jobspec for a group: one slot per +// computeJobspec builds the physical-compute jobspec for a group. One slot per // pod holding the compute resources, constrained to virtual=false nodes. This is // the only jobspec for a group that requests no virtual devices. func computeJobspec(groupName string, slots int, compute map[string]int, excludeNodes []string) *jobspec.Jobspec { @@ -274,15 +274,15 @@ func computeJobspec(groupName string, slots int, compute map[string]int, exclude } // deviceJobspec builds a jobspec selecting a single virtual device type. Every -// configured virtual resource is a node carrying class= (and the classes +// configured virtual resource is a node with class= (and the classes // of its descendants, so a nested type is reachable), so a device is selected by // constraining to a virtual node of the requested class. count is the requested // quantity. // // The constraint is virtual=true (scope to virtual backends, not physical nodes) // AND class= (the requested resource type). The slot requests `node` -// because every virtual resource is a node; the class constraint — not the `with` -// type — picks which one. A nested type (e.g. qpu under a qdevice node) is +// because every virtual resource is a node. The class constraint, not the `with` +// type, picks which one. A nested type (e.g. qpu under a qdevice node) is // reachable because every ancestor node also advertises class= for the // types beneath it, so the global constraint does not prune the path. func deviceJobspec(groupName, deviceType string, count int, extraProps []string) *jobspec.Jobspec { @@ -309,10 +309,10 @@ func deviceJobspec(groupName, deviceType string, count int, extraProps []string) // JobspecsForGroup builds the set of Fluxion jobspecs to match for a pod group, // each held independently (duration 0, released by Cancel) and combined all-or- -// nothing by the caller: +// nothing by the caller. // -// - exactly one compute jobspec (slot per pod, virtual=false) — always present, -// so a plain pod or group with no virtual resources yields a single match; +// - exactly one compute jobspec (slot per pod, virtual=false), always present, +// so a plain pod or group with no virtual resources yields a single match. // - one device jobspec per distinct requested virtual resource type // (constraint virtual=true; the requested type+count rides the slot's `with`). // @@ -332,7 +332,7 @@ func JobspecsForGroup( } // Compute sizing comes from a representative pod (the group is homogeneous in // its per-pod compute slot), but DEVICE requests must be unioned across the - // whole group: in a quantum gang only the leader requests the qpu, and the + // whole group. In a quantum gang only the leader requests the qpu, and the // pod order here is not guaranteed (groupPods lists in informer order), so // keying off pods[0] alone would miss the leader's device entirely and emit // a compute-only match with no backend. @@ -377,15 +377,15 @@ func JobspecsForGroup( // Placement is the result of matching one of a group's jobspecs. Nodes are the // physical (virtual=false) compute nodes a pod binds to; Backend is the virtual // (virtual=true) resource's identity, surfaced to the pod as env. A single -// allocation yields one or the other: the compute match yields nodes, a device +// allocation yields one or the other. The compute match yields nodes, a device // match yields a backend. type Placement struct { Nodes []string // physical compute node names (virtual=false) Backend string // virtual backend identity (virtual=true), if any // BackendAttributes are the matched virtual resource's user attributes - // (region, qubits, ...), decomposed from the backend node's namespaced + // (region, qubits...), decomposed from the backend node's namespaced // properties. These are injected into the pod as env so the workload sees - // exactly what it matched/queried — the same set that is filterable is also + // exactly what it matched/queried, the same set that is filterable is also // readable back. BackendAttributes map[string]string } @@ -404,9 +404,9 @@ func decomposeProperty(prop string) (key, value string) { // node-typed vertices by the virtual marker property: virtual=false nodes are // physical compute (bind targets), virtual=true nodes are virtual backends // (their name is the backend identity injected into the pod). Everything is -// type "node" now, so the marker — not the vertex type — does the split. +// type "node" now, so the marker, not the vertex type, does the split. // -// A node carrying neither marker is treated as a physical compute node, so a +// A node with neither marker is treated as a physical compute node, so a // plain graph built without markers still binds correctly. For the chosen // backend, its user attributes (the fluxion.flux-framework.org/= // properties) are decomposed into BackendAttributes for env injection. @@ -433,7 +433,7 @@ func PlacementFromAllocation(alloc string) (Placement, error) { } // backendAttributes extracts the user attributes from a backend node's composed -// property keys: every property of the form +// property keys. Every property of the form // "fluxion.flux-framework.org/=" becomes -> . Reserved // markers (virtual=..., class=...) are skipped. func backendAttributes(props map[string]string) map[string]string { diff --git a/pkg/placement/placement_test.go b/pkg/placement/placement_test.go index fe68917..68d5223 100644 --- a/pkg/placement/placement_test.go +++ b/pkg/placement/placement_test.go @@ -57,7 +57,7 @@ func hasProp(props []string, want string) bool { return false } -// A classical group (no virtual resources) yields exactly one jobspec: the +// A classical group (no virtual resources) yields exactly one jobspec. The // compute slot, constrained to virtual=false. func TestClassicalSingleMatch(t *testing.T) { pods := []corev1.Pod{ @@ -92,7 +92,7 @@ func TestClassicalSingleMatch(t *testing.T) { // compute slot. // A quantum gang lists pods in arbitrary (informer) order, and only the leader // requests the qpu. The device match must be emitted even when the device -// requester is not pods[0]; otherwise the group schedules with no backend. +// requester is not pods[0]. Otherwise the group schedules with no backend. func TestGroupDeviceMatchWhenLeaderNotFirst(t *testing.T) { worker := podWith("w0", corev1.ResourceList{corev1.ResourceCPU: qty(1)}) leader := podWith("leader", corev1.ResourceList{ @@ -128,7 +128,7 @@ func qpuPodWithRequires(name string, requires map[string]string) corev1.Pod { return p } -// Zero require- annotations: the device match must carry ONLY the base +// Zero require- annotations. The device match must have ONLY the base // constraints, nothing extra (over-constraining would break unconstrained runs). func TestNoRequireAnnotationsAddsNoConstraints(t *testing.T) { p := qpuPodWithRequires("q", nil) @@ -257,7 +257,7 @@ func TestDeviceProducesSecondMatch(t *testing.T) { t.Errorf("device constraint must include virtual=true; got %v", props) } // The device is selected by class=: every virtual resource is - // a node carrying its class, so the constraint picks the right one. The slot + // a node with its class, so the constraint picks the right one. The slot // requests a node (every virtual resource is a node), count from the request. if !hasProp(props, "class=qpu") { t.Errorf("device constraint must include class=qpu; got %v", props) @@ -313,7 +313,7 @@ func TestHoldDurationZero(t *testing.T) { } } -// A compute allocation: node vertices carrying virtual=false are bind targets. +// A compute allocation: node vertices with virtual=false are bind targets. func TestPlacementComputeNodes(t *testing.T) { alloc := `{"graph":{"nodes":[ {"metadata":{"type":"node","name":"node-a","properties":{"virtual=false":""}}}, @@ -426,7 +426,7 @@ func TestExcludeNodesAddsNegatedHostlist(t *testing.T) { } // computeConstraints digs out attributes.system.constraints from the compute -// jobspec (the first spec; device specs do not carry node exclusions). +// jobspec (the first spec. Device specs do not have node exclusions). func computeConstraints(t *testing.T, spec *jobspec.Jobspec) map[string]interface{} { t.Helper() sys, ok := spec.Attributes["system"].(map[string]interface{}) diff --git a/pkg/webhook/admission.go b/pkg/webhook/admission.go index b1c910e..285600b 100644 --- a/pkg/webhook/admission.go +++ b/pkg/webhook/admission.go @@ -27,7 +27,7 @@ import ( const ( // ProtectedAnnotation marks a pod as one of ours and not preemptible. Set by - // the webhook on every quantum pod; a submission carrying it is refused. + // the webhook on every quantum pod. A submission with it is refused. ProtectedAnnotation = "fluence.flux-framework.org/protected" ProtectedValue = "quantum" @@ -54,7 +54,7 @@ type Capacity struct { Promised int // admitted to quantum groups whose members are not all running } -// Reachable is what a new group could have: free cores plus what preemption +// Reachable is what a new group could have. Free cores plus what preemption // could free. func (c Capacity) Reachable() int { return c.Free + c.Preemptible } @@ -135,7 +135,7 @@ func (m *Mutator) Capacity(ctx context.Context) (Capacity, error) { // cores running per admission record, keyed namespace/name runningOf := map[string]int{} - // the records themselves: PodGroups, and standalone pods that carry their own + // the records themselves. PodGroups, and standalone pods that have their own records := map[string]int{} for _, pg := range groups.Items { if n, err := strconv.Atoi(pg.Annotations[AdmittedCoresAnnotation]); err == nil && n > 0 { diff --git a/pkg/webhook/handler.go b/pkg/webhook/handler.go index 1cbcdc2..f74019f 100644 --- a/pkg/webhook/handler.go +++ b/pkg/webhook/handler.go @@ -11,11 +11,11 @@ import ( // MutatorAPI is the capability surface the webhook core exposes to handlers. // Handlers live in a subpackage and depend only on this interface, never on the -// concrete Mutator — so the core never has to import the handlers package, and +// concrete Mutator, so the core never has to import the handlers package, and // handlers cannot reach into core internals. The concrete *Mutator implements // this. // -// The core knows nothing about what any handler does with these capabilities; +// The core knows nothing about what any handler does with these capabilities. // quantum-specific policy (resource names, gate names, observe label) lives // entirely in the quantum handler. type MutatorAPI interface { @@ -28,7 +28,7 @@ type MutatorAPI interface { // EnsurePodGroup creates the group's PodGroup with the given gang minCount if // it does not already exist (idempotent). Group identity is the opaque value // of the group label. creatorPod is recorded only as the PodGroup's creator - // reference; the core ascribes no role semantics to it. + // reference. The core ascribes no role semantics to it. EnsurePodGroup(ctx context.Context, namespace, group, creatorPod string, minCount int32) // Admit applies admission control, see admission.go. key names the group @@ -48,8 +48,8 @@ type Admitter interface { } // Handler inspects a pod and, when it applies, contributes JSON patch ops. A pod -// flows through every registered handler whose Applies returns true; their ops -// are concatenated. Applies is fully general — it receives the pod and the +// flows through every registered handler whose Applies returns true. Their ops +// are concatenated. Applies is fully general, it receives the pod and the // MutatorAPI, so a handler may consult cluster state in deciding whether it // applies. type Handler interface { @@ -59,8 +59,8 @@ type Handler interface { } // DefaultHandlerOrder is the active set AND the dispatch order when the operator -// passes no --handlers flag. Order matters: specific handlers run before the -// generic gang fallback, so "gang" is LAST — it applies default gang sizing +// passes no --handlers flag. Order matters. Specific handlers run before the +// generic gang fallback, so "gang" is LAST, it applies default gang sizing // (group-size annotation or owner-derived N) only if no earlier handler already // shaped the gang. To change the order or disable a handler, pass a different // list (e.g. --handlers=fluxion,gang drops quantum). @@ -69,9 +69,9 @@ var DefaultHandlerOrder = []string{"fluxion", "quantum", "gang"} // ── registration ──────────────────────────────────────────────────────────────── // // Handlers self-register via Register() from their package's init(). The core -// never names a handler; importing the handlers package (a blank import in the +// never names a handler. Importing the handlers package (a blank import in the // webhook server wiring) is what populates the registry. This keeps the core -// domain-agnostic: adding or removing a handler does not touch core code. +// domain-agnostic. Adding or removing a handler does not touch core code. // available maps a handler's Name() to the handler. Populated by Register() from // each handler package's init(). This is the set of handlers that EXIST; which @@ -80,7 +80,7 @@ var available = map[string]Handler{} // activeOrder is the ordered list of handler names to dispatch. It is BOTH the // selection (names not present are disabled) and the order (dispatch follows the -// slice). Defaults to DefaultHandlerOrder; overridden by SetActiveHandlers. +// slice). Defaults to DefaultHandlerOrder. Overridden by SetActiveHandlers. var activeOrder = append([]string(nil), DefaultHandlerOrder...) // Register adds a handler to the available set under its Name(). Called from @@ -92,7 +92,7 @@ func Register(h Handler) { // SetActiveHandlers sets the active, ordered handler list (the --handlers value). // Empty/nil restores DefaultHandlerOrder. Names with no registered handler are // dropped and returned as `unknown` so the caller can warn. Order is preserved -// exactly as given — the list is the dispatch order. +// exactly as given, the list is the dispatch order. func SetActiveHandlers(names []string) (active, unknown []string) { if len(names) == 0 { activeOrder = append([]string(nil), DefaultHandlerOrder...) diff --git a/pkg/webhook/handlers/admission_test.go b/pkg/webhook/handlers/admission_test.go index feefa86..51e7741 100644 --- a/pkg/webhook/handlers/admission_test.go +++ b/pkg/webhook/handlers/admission_test.go @@ -106,7 +106,7 @@ func TestStandaloneAdmittedWhenItFits(t *testing.T) { t.Errorf("admitted pod must be marked protected, got %v", ann) } if ann[webhook.AdmittedCoresAnnotation] != "3" { - t.Errorf("standalone pod must carry its own record, got %v", ann) + t.Errorf("standalone pod must have its own record, got %v", ann) } if ann[webhook.AdmittedGroupAnnotation] != "default/sampler" { t.Errorf("standalone pod must point at its own record, got %v", ann) diff --git a/pkg/webhook/handlers/dependency.go b/pkg/webhook/handlers/dependency.go index 33f2980..a0a8a4b 100644 --- a/pkg/webhook/handlers/dependency.go +++ b/pkg/webhook/handlers/dependency.go @@ -7,24 +7,24 @@ import ( ) // Dependency is Fluence's GENERAL "this set of pods must wait for a scout to -// be ready" primitive. It is deliberately NOT quantum-specific: quantum is the +// be ready" primitive. It is deliberately NOT quantum-specific. Quantum is the // first resource type to use it (a gang waits for a quantum submission to reach // the device queue), but the same primitive applies to any resource type whose -// readiness is produced out-of-band — a license server, a data stage-in job, a +// readiness is produced out-of-band, a license server, a data stage-in job, a // warmed cache, another gang, etc. // -// A Dependency has three parts, each carried as a pod annotation so the +// A Dependency has three parts, each stored as a pod annotation so the // relationship lives at the GROUP level (not duplicated as bespoke per-resource // fields) and is readable by both the webhook (at admission) and the scheduler -// (in its reconcile loop): +// (in its reconcile loop). // -// - Kind: what KIND of readiness this is (the resource type's name). The -// scout side knows how to satisfy this kind; the worker side +// - Kind is what KIND of readiness this is (the resource type's name). The +// scout side knows how to satisfy this kind. The worker side // only knows it must wait. Quantum's kind is "quantum-submit". -// - Scout: the identity of the thing that will signal ready. For quantum it -// is the submitter's (base) group; generally it is whatever the +// - Scout. The identity of the thing that will signal ready. For quantum it +// is the submitter's (base) group. Generally it is whatever the // kind's handler records as the satisfier. -// - Gate: the scheduling gate held on the dependent (worker) pods until +// - Gate is the scheduling gate held on the dependent (worker) pods until // the scout signals ready. Removing the gate is the "ungate" // and is performed by whatever observes the scout's readiness // (the quantum sidecar for kind=quantum-submit; the scheduler's @@ -32,9 +32,9 @@ import ( // "another gang is Running"). // // The webhook PRODUCES a Dependency (gates the workers, stamps the -// annotations); REMOVING the gate is owned by the observer best placed to see -// the scout's readiness. That split — declare here, observe elsewhere — is -// what keeps the primitive general: a new resource type adds a Kind and an +// annotations). REMOVING the gate is owned by the observer best placed to see +// the scout's readiness. That split, declare here, observe elsewhere, is +// what keeps the primitive general. A new resource type adds a Kind and an // observer and reuses the gating/annotation machinery unchanged. type Dependency struct { Kind string // resource-type readiness kind, e.g. "quantum-submit" @@ -42,7 +42,7 @@ type Dependency struct { Gate string // scheduling gate held on dependents until ready } -// Dependency annotation keys (stamped on the dependent pods). Generic — no +// Dependency annotation keys (stamped on the dependent pods). Generic, no // quantum in the names, so any resource type reuses them. const ( // DependsOnKindAnnotation names the readiness kind the dependent waits for. @@ -56,8 +56,8 @@ const ( // applyOps gates the dependent pod and stamps the dependency annotations so the // relationship is self-describing on the pod. It reuses the gate machinery -// (gateWithName) verbatim — the gate is the universal "held until ready" -// mechanism regardless of resource type — so a new Kind costs only its readiness +// (gateWithName) verbatim, the gate is the universal "held until ready" +// mechanism regardless of resource type, so a new Kind costs only its readiness // observer, not new gating code. func (d Dependency) applyOps(pod *corev1.Pod) []spec.Op { ops := gateWithName(pod, d.Gate) @@ -68,7 +68,7 @@ func (d Dependency) applyOps(pod *corev1.Pod) []spec.Op { } // DependencyOf reads a dependent pod's declared Dependency, or ok=false if it -// carries none. The scheduler's reconcile loop and the sidecar use this to learn +// has none. The scheduler's reconcile loop and the sidecar use this to learn // what a gated pod is waiting for without hardcoding a kind. func DependencyOf(pod *corev1.Pod) (Dependency, bool) { kind := spec.Annotation(pod, DependsOnKindAnnotation) @@ -119,7 +119,7 @@ func gateWithName(pod *corev1.Pod, gateName string) []spec.Op { ops = append(ops, spec.Op{Op: "add", Path: "/spec/schedulingGates/-", Value: gate}) } // Gated dependents schedule reliably once ungated only if they outrank other - // pending work; priorityClassName is immutable post-creation so it must be + // pending work. priorityClassName is immutable post-creation so it must be // set now. Don't override a user's explicit class. spec.priority is cleared // to null so the priority admission controller recomputes it from the class // (add-null is valid whether the field is absent, 0, or set). diff --git a/pkg/webhook/handlers/fluxion.go b/pkg/webhook/handlers/fluxion.go index 007a8af..7d54c80 100644 --- a/pkg/webhook/handlers/fluxion.go +++ b/pkg/webhook/handlers/fluxion.go @@ -21,7 +21,7 @@ func init() { // fluxionHandler injects the FLUXION_* env contract into any container that // requests a fluxion.flux-framework.org/* resource. Generic to all Fluxion -// resources — not quantum-specific. (Named for the domain, not "fluxion_env", +// resources, not quantum-specific. (Named for the domain, not "fluxion_env", // since Fluxion concerns may grow beyond env injection.) type fluxionHandler struct{} diff --git a/pkg/webhook/handlers/gang.go b/pkg/webhook/handlers/gang.go index b827d17..80a571c 100644 --- a/pkg/webhook/handlers/gang.go +++ b/pkg/webhook/handlers/gang.go @@ -16,10 +16,10 @@ func init() { webhook.Register(&gangHandler{}) } -// gangHandler gang-schedules pods that carry the group label: it creates a +// gangHandler gang-schedules pods that have the group label. It creates a // Fluence-owned PodGroup and // links every pod to it via spec.schedulingGroup.podGroupName, which is the -// field the scheduler gangs by. It knows nothing about quantum — a purely +// field the scheduler gangs by. It knows nothing about quantum, a purely // classical gang is fully handled here, with no sidecar. type gangHandler struct{} @@ -28,7 +28,7 @@ func (h *gangHandler) Name() string { return "gang" } func (h *gangHandler) Applies(ctx context.Context, m webhook.MutatorAPI, pod *corev1.Pod) bool { // Classical gangs only. A pod that requests the quantum resource is gang- // scheduled by the quantum handler, which owns the scout/worker split and - // creates both the -scout and PodGroups itself; handling it + // creates both the -scout and PodGroups itself. Handling it // here too would create a second, conflicting PodGroup for the group. if spec.PodRequestsResource(pod, QuantumResource) { return false @@ -40,25 +40,25 @@ func (h *gangHandler) Mutate(ctx context.Context, m webhook.MutatorAPI, pod *cor g := webhook.GroupName(pod) // Ensure the group's PodGroup exists with the resolved gang size, and link // this pod to it. EnsurePodGroup is idempotent (no-ops if the PodGroup - // already exists — e.g. created by an earlier, more specific handler), so we + // already exists, e.g. created by an earlier, more specific handler), so we // call it unconditionally. The gang handler knows nothing about quantum or - // submitters; that is the quantum handler's concern. + // submitters. That is the quantum handler's concern. // minCount = full gang size N (group-size annotation, else owner-derived); // see resolveMinCount. m.EnsurePodGroup(ctx, pod.Namespace, g, pod.Name, resolveMinCount(ctx, m, pod)) return schedulingGroupOps(pod, g) } -// resolveMinCount determines the gang's atomic-schedule size N: +// resolveMinCount determines the gang's atomic-schedule size N. // 1. explicit group-size annotation -> honor it verbatim. This is the override // for when minCount must differ from the parent's replica count (e.g. the // quantum leader/worker split, where the gang's N is expressed directly). -// 2. otherwise derive from the OWNING object: a Flux Operator MiniCluster pod +// 2. otherwise derive from the OWNING object. A Flux Operator MiniCluster pod // is owned by an indexed Job whose parallelism == completions == size == N. // (The operator sets Parallelism = Completions = MiniCluster.Spec.Size.) -// 3. otherwise default to 1, logged — never silently size a multi-pod gang to 1. +// 3. otherwise default to 1, logged, never silently size a multi-pod gang to 1. // -// The leader/worker (quantum) split is orthogonal and unchanged: it is driven by +// The leader/worker (quantum) split is orthogonal and unchanged. It is driven by // QuantumResource in the quantum handler. minCount is always the // FULL gang N regardless of which pods get gated. func resolveMinCount(ctx context.Context, m webhook.MutatorAPI, pod *corev1.Pod) int32 { @@ -74,7 +74,7 @@ func resolveMinCount(ctx context.Context, m webhook.MutatorAPI, pod *corev1.Pod) if n := ownerJobN(ctx, m, pod); n > 0 { return n } - // 3. no signal: a single-pod gang. Log so a missing size on a multi-pod + // 3. no signal. A single-pod gang. Log so a missing size on a multi-pod // workload is visible rather than a silent gang-of-1. log.Printf("[fluence-webhook] group %s: no group-size annotation and no owning Job parallelism; defaulting minCount=1", webhook.GroupName(pod)) return 1 diff --git a/pkg/webhook/handlers/gang_test.go b/pkg/webhook/handlers/gang_test.go index aeaa0c4..2ee600e 100644 --- a/pkg/webhook/handlers/gang_test.go +++ b/pkg/webhook/handlers/gang_test.go @@ -1,4 +1,4 @@ -// Tests for gang PodGroup minCount: the whole gang (full N) must schedule +// Tests for gang PodGroup minCount. The whole gang (full N) must schedule // atomically. Regression guard for the bug where every PodGroup was created // with minCount=1, so a multi-pod gang was "satisfied" by a single pod and the // rest were stranded (partial placement). @@ -130,9 +130,9 @@ func TestGangMinCountDefaultsToOne(t *testing.T) { } } -// group-size is the authoritative gang minCount: a workload that sets it to N +// group-size is the authoritative gang minCount. A workload that sets it to N // gets minCount=N (the whole gang schedules atomically), regardless of any owner -// replica count. In the gang+submitter model the full workload IS the gang — +// replica count. In the gang+submitter model the full workload IS the gang // there is no N-1 worker split. func TestGangMinCountHonorsGroupSize(t *testing.T) { pod := cpuPod("fluence") diff --git a/pkg/webhook/handlers/handlers_test.go b/pkg/webhook/handlers/handlers_test.go index 27b5983..c029dc9 100644 --- a/pkg/webhook/handlers/handlers_test.go +++ b/pkg/webhook/handlers/handlers_test.go @@ -138,7 +138,7 @@ func TestMutateSkipsNonFluxion(t *testing.T) { } } -// ── gang handler: scheduling group linkage ────────────────────────────────────── +// ── gang handler. Scheduling group linkage ────────────────────────────────────── func TestGangStampsSchedulingGroup(t *testing.T) { m := &webhook.Mutator{} diff --git a/pkg/webhook/handlers/quantum.go b/pkg/webhook/handlers/quantum.go index a64c5d7..c9fb518 100644 --- a/pkg/webhook/handlers/quantum.go +++ b/pkg/webhook/handlers/quantum.go @@ -47,24 +47,24 @@ const ( // without changing the mechanism. CoordinationAnnotation = "fluence.flux-framework.org/coordination" - // CoordinationShared: one real task; the scout (index 0) submits and the + // CoordinationShared. One real task. The scout (index 0) submits and the // other members are gated workers that fetch the scout's result. Each - // member is told its role via FLUENCE_COORDINATION_ROLE; a role-aware workload + // member is told its role via FLUENCE_COORDINATION_ROLE. A role-aware workload // branches on it (scout submits, worker fetches by FLUENCE_QUANTUM_JOB_ID). CoordinationShared = "shared" - // CoordinationIndependent (default): every member does its own quantum work; + // CoordinationIndependent (default). Every member does its own quantum work. // no coordination, no gating. Never invent coordination the user did not ask // for, and never dedup tasks meant to be distinct. CoordinationIndependent = "independent" - // ScoutGroupSuffix names the scout's own group-of-one: -scout + // ScoutGroupSuffix names the scout's own group-of-one. -scout // (minCount 1) so it schedules alone and never deadlocks against the gated // worker gang. ScoutGroupSuffix = "-scout" // CompletionIndexAnnotation is the indexed-Job completion index the Job - // controller stamps on each pod; index "0" is the scout (deterministic + // controller stamps on each pod. Index "0" is the scout (deterministic // election with no recorded state). CompletionIndexAnnotation = "batch.kubernetes.io/job-completion-index" @@ -110,7 +110,7 @@ func (h *quantumHandler) Admit(ctx context.Context, m webhook.MutatorAPI, pod *c return m.Admit(ctx, pod, key, want) } -// groupAdmitted reports whether the gang PodGroup already carries an admission record. +// groupAdmitted reports whether the gang PodGroup already has an admission record. func groupAdmitted(ctx context.Context, m webhook.MutatorAPI, namespace, group string) bool { c := m.Client() if c == nil { @@ -124,7 +124,7 @@ func groupAdmitted(ctx context.Context, m webhook.MutatorAPI, namespace, group s } // admissionOps marks a quantum pod protected and names its admission record. -// A standalone pod is its own record and carries the total. +// A standalone pod is its own record and has the total. func admissionOps(pod *corev1.Pod, record string, standaloneCores int) []spec.Op { ann := map[string]string{ webhook.ProtectedAnnotation: webhook.ProtectedValue, @@ -162,8 +162,8 @@ func (h *quantumHandler) Mutate(ctx context.Context, m webhook.MutatorAPI, pod * observe := spec.Label(pod, ObserveLabel) == "true" cores := placement.PodCores(pod) - // No coordination: a standalone quantum pod, or an explicitly independent - // member. The REAL submit happens in THIS pod; the sidecar is added only for + // No coordination. A standalone quantum pod, or an explicitly independent + // member. The REAL submit happens in THIS pod. The sidecar is added only for // observe-only telemetry. (independent mode routes every member here -> N // standalone scouts, each owning its task and its own queue wait.) if mode != CoordinationShared || g == "" || n <= 1 { @@ -180,7 +180,7 @@ func (h *quantumHandler) Mutate(ctx context.Context, m webhook.MutatorAPI, pod * return ops } - // shared mode: promote one member to scout; the rest are gated workers. + // shared mode. Promote one member to scout. The rest are gated workers. // the gang PodGroup is the admission record for scout and workers // alike, so make sure it exists and stamp the total on it once m.EnsurePodGroup(ctx, pod.Namespace, g, pod.Name, n-1) @@ -193,7 +193,7 @@ func (h *quantumHandler) Mutate(ctx context.Context, m webhook.MutatorAPI, pod * } // mutateScout wires the single scout member (indexed-Job completion index -// 0): its own group-of-one -scout (minCount 1) so it schedules alone +// 0). Its own group-of-one -scout (minCount 1) so it schedules alone // and runs the REAL submit, the interceptor in tag mode, RBAC, and the sidecar // told which worker group to ungate (FLUENCE_GANG_GROUP). The scout is one // of the N members, so the application is NOT run an extra time. Never gated. @@ -208,17 +208,17 @@ func (h *quantumHandler) mutateScout(ctx context.Context, m webhook.MutatorAPI, sc.EnsureRBAC(ctx, pod.Namespace) extra := []corev1.EnvVar{{Name: GangGroupEnv, Value: group}} ops = append(ops, sc.ContainerOps(pod, false, extra)...) - log.Printf("[fluence-webhook] quantum scout %s/%s — group %s (ungates workers %q)", + log.Printf("[fluence-webhook] quantum scout %s/%s, group %s (ungates workers %q)", pod.Namespace, pod.Name, pg, group) return ops } -// mutateWorker wires a non-scout member: it joins the worker gang +// mutateWorker wires a non-scout member. It joins the worker gang // (minCount N-1) and is gated until the scout's task is ready. It is told its // role (FLUENCE_COORDINATION_ROLE=worker) and handed the scout's task id // (FLUENCE_QUANTUM_JOB_ID, stamped on the pod by the sidecar at ungate). A // role-aware worker reads those and fetches the shared result instead of -// submitting — so the worker never calls the vendor submit, and needs neither +// submitting, so the worker never calls the vendor submit, and needs neither // the interceptor nor a faux flag. func (h *quantumHandler) mutateWorker(ctx context.Context, m webhook.MutatorAPI, pod *corev1.Pod, group string, n int32) []spec.Op { m.EnsurePodGroup(ctx, pod.Namespace, group, pod.Name, n-1) @@ -231,7 +231,7 @@ func (h *quantumHandler) mutateWorker(ctx context.Context, m webhook.MutatorAPI, // its quantum resource request is dropped. Otherwise Fluxion would allocate // a qpu per worker and a backend with one qpu could never seat the gang. ops = append(ops, dropQuantumResourceOps(pod)...) - log.Printf("[fluence-webhook] quantum worker %s/%s — group %s minCount=%d, gated (role=worker, qpu stripped)", + log.Printf("[fluence-webhook] quantum worker %s/%s, group %s minCount=%d, gated (role=worker, qpu stripped)", pod.Namespace, pod.Name, group, n-1) return ops } @@ -263,7 +263,7 @@ func dropQuantumResourceOps(pod *corev1.Pod) []spec.Op { return ops } -// coordinationMode reads the coordination annotation; default independent. +// coordinationMode reads the coordination annotation. Default independent. func coordinationMode(pod *corev1.Pod) string { if v := spec.Annotation(pod, CoordinationAnnotation); v != "" { return v @@ -272,9 +272,9 @@ func coordinationMode(pod *corev1.Pod) string { } // isScout decides whether THIS pod is the gang's single scout. Indexed Job -// (recommended): completion index 0 is the scout — deterministic, race-free, -// no recorded state. Otherwise: first arrival claims the scout slot by the -// absence of the scout PodGroup (best-effort under concurrent admission; +// (recommended). Completion index 0 is the scout, deterministic, race-free, +// no recorded state. Otherwise. First arrival claims the scout slot by the +// absence of the scout PodGroup (best-effort under concurrent admission. // prefer an indexed Job for determinism). Indexing a nil annotations map yields // ok=false, so the indexed branch is nil-safe. func isScout(ctx context.Context, m webhook.MutatorAPI, pod *corev1.Pod, group string) bool { @@ -292,8 +292,8 @@ func isScout(ctx context.Context, m webhook.MutatorAPI, pod *corev1.Pod, group s return true } -// resolveGroup returns the gang group identity: the explicit group label, else -// the owning controller's name (Job/ReplicaSet/StatefulSet — a Deployment's pods +// resolveGroup returns the gang group identity. The explicit group label, else +// the owning controller's name (Job/ReplicaSet/StatefulSet, a Deployment's pods // are owned by a ReplicaSet), else "" (a loose quantum pod with no group, which // is treated as a standalone group of one). func resolveGroup(pod *corev1.Pod) string { @@ -309,10 +309,10 @@ func resolveGroup(pod *corev1.Pod) string { return "" } -// resolveGangSize returns the full gang size N: the explicit group-size +// resolveGangSize returns the full gang size N. The explicit group-size // annotation (authoritative override), else the owner's replica count (Job // parallelism/completions, ReplicaSet replicas), else a count of pods already -// carrying the group label (best-effort for loose grouped pods; admission-order +// with the group label (best-effort for loose grouped pods. Admission-order // dependent, which is why the annotation is preferred), else 1. func resolveGangSize(ctx context.Context, m webhook.MutatorAPI, pod *corev1.Pod, group string) int32 { if pod.Annotations != nil { @@ -356,9 +356,9 @@ func ownerReplicaSetN(ctx context.Context, m webhook.MutatorAPI, pod *corev1.Pod return 0 } -// countGroupPods counts pods already carrying the group label (best-effort gang +// countGroupPods counts pods that already have the group label (best-effort gang // size for loose grouped pods that have neither a group-size annotation nor an -// owning controller). Admission-order dependent — prefer the group-size +// owning controller). Admission-order dependent, prefer the group-size // annotation when the exact size must be guaranteed. func countGroupPods(ctx context.Context, m webhook.MutatorAPI, namespace, group string) int32 { c := m.Client() @@ -374,7 +374,7 @@ func countGroupPods(ctx context.Context, m webhook.MutatorAPI, namespace, group return int32(len(list.Items)) } -// linkGroupOps ensures the gang pod carries the group label (so the scout's +// linkGroupOps ensures the gang pod has the group label (so the scout's // sidecar can list it) and is linked to the gang PodGroup via // spec.schedulingGroup.podGroupName. Idempotent. func linkGroupOps(pod *corev1.Pod, group string) []spec.Op { @@ -409,19 +409,19 @@ const QuantumClassicalPriorityClass = "fluence-quantum-classical" // ── coordination role (scout / worker) ───────────────────────────────────── // // In a shared gang each member is told its role positively, so the application -// branches on it instead of relying on any submit-interception magic: -// scout submits the one real task (and is tagged so the sidecar finds it); -// worker does NOT submit — it reads the scout's task id and fetches the +// branches on it instead of relying on any submit-interception magic. +// scout submits the one real task (and is tagged so the sidecar finds it). +// worker does NOT submit, it reads the scout's task id and fetches the // shared result (e.g. via the vendor's S3-backed result API). // The role is decided at admission by isScout (completion index 0, else the // scout-group claim) and surfaced as FLUENCE_COORDINATION_ROLE. Because the -// election is the webhook's, this env is the single source of truth — the +// election is the webhook's, this env is the single source of truth, the // container never re-derives its role from the Job index (which loose, non-Job // pods don't even have). const ( - // CoordinationRoleEnv carries the pod's role in a shared gang. A role-aware - // workload branches on it: RoleScout submits, RoleWorker fetches the + // CoordinationRoleEnv has the pod's role in a shared gang. A role-aware + // workload branches on it. RoleScout submits, RoleWorker fetches the // shared result by id. Unset for standalone/independent pods (they all submit). CoordinationRoleEnv = "FLUENCE_COORDINATION_ROLE" RoleScout = "scout" @@ -478,19 +478,19 @@ func setContainerEnvOps(pod *corev1.Pod, e corev1.EnvVar) []spec.Op { return ops } -// Sidecar implementation — quantum-owned, NOT core. +// Sidecar implementation, quantum-owned, NOT core. // // The fluence coordination sidecar (its container, name, RBAC, image, and the -// Python interceptor staging) is specific to the quantum integration: it polls a +// Python interceptor staging) is specific to the quantum integration. It polls a // vendor queue and ungates workers. None of this belongs on the webhook core, // which stays domain-agnostic and only exposes generic primitives (Client, -// InjectedEnv, EnsurePodGroup). The core invokes each handler's generic Mutate; -// a handler does its own create/edit side-effects (here: RBAC, ConfigMaps, +// InjectedEnv, EnsurePodGroup). The core invokes each handler's generic Mutate. +// a handler does its own create/edit side-effects (here. RBAC, ConfigMaps, // container injection) through the generic client. // // These are package-level functions (not methods on the core *Mutator) operating // on the generic webhook.MutatorAPI. coreSidecar (see sidecar.go) delegates to -// them; a future non-quantum handler that needs a different sidecar supplies its +// them. A future non-quantum handler that needs a different sidecar supplies its // own Sidecar implementation and its own container name/image. const ( @@ -503,16 +503,16 @@ const ( SidecarServiceAccount = "fluence-sidecar" // defaultSidecarImage is used when FLUENCE_SIDECAR_IMAGE is not set. Owned by - // the quantum integration; the deployment may override it via the env var. + // the quantum integration. The deployment may override it via the env var. defaultSidecarImage = "vanessa/fluence-sidecar:latest" - // StageVolumeName / StageMountPath: the shared emptyDir the init container + // StageVolumeName / StageMountPath. The shared emptyDir the init container // stages the fluence Python package into, mounted into workload containers // and prepended to PYTHONPATH (Model C delivery). StageVolumeName = "fluence-pkg" StageMountPath = "/opt/fluence-staged" - // HandoffVolumeName / HandoffMountPath / HandoffDirEnv: the shared emptyDir + // HandoffVolumeName / HandoffMountPath / HandoffDirEnv. The shared emptyDir // a submitting pod's interceptor writes its task id and session to, and // its sidecar reads (python/fluence/handoff.py). Workers never get it. HandoffVolumeName = "fluence-handoff" @@ -520,7 +520,7 @@ const ( HandoffDirEnv = "FLUENCE_HANDOFF_DIR" ) -// sidecarImage resolves the sidecar image: the FLUENCE_SIDECAR_IMAGE override +// sidecarImage resolves the sidecar image. The FLUENCE_SIDECAR_IMAGE override // (deployment config) or the quantum default. Read here so image config is owned // by the integration that uses it, not the core. func sidecarImage() string { @@ -569,12 +569,12 @@ func ensureSidecarRBAC(ctx context.Context, m webhook.MutatorAPI, namespace stri } } -// interceptorOps stages the fluence Python package (Model C): an init container +// interceptorOps stages the fluence Python package (Model C). An init container // copies it into a shared emptyDir, mounted into every workload container // (skipping the sidecar) with PYTHONPATH + FLUENCE_POD_UID, so Python auto-imports // the interceptor via sitecustomize, which tags the vendor submit so the sidecar // can find the task. Added to scouts and standalone/independent pods (the ones -// that actually submit); workers don't submit, so they don't get it. +// that actually submit). Workers don't submit, so they don't get it. func interceptorOps(pod *corev1.Pod) []spec.Op { var ops []spec.Op @@ -682,8 +682,8 @@ func sidecarContainerOps(m webhook.MutatorAPI, pod *corev1.Pod, observe bool, ex } env = append(env, extraEnv...) // Copy the workload container's secret/configMap-sourced env onto the sidecar - // so it can talk to the same backend (domain-agnostic: we propagate whatever - // the workload pulls from a secret/configMap; existing FLUENCE_/FLUXION_ names + // so it can talk to the same backend (domain-agnostic. We propagate whatever + // the workload pulls from a secret/configMap. Existing FLUENCE_/FLUXION_ names // are not overwritten). if len(pod.Spec.Containers) > 0 { have := map[string]bool{} diff --git a/pkg/webhook/handlers/quantum_test.go b/pkg/webhook/handlers/quantum_test.go index d4faf0d..0817048 100644 --- a/pkg/webhook/handlers/quantum_test.go +++ b/pkg/webhook/handlers/quantum_test.go @@ -1,4 +1,4 @@ -// quantum_test.go — all tests for the quantum handler: the scout/worker +// quantum_test.go, all tests for the quantum handler. The scout/worker // shared-coordination split (no separate submitter pod), independent mode, // the coordination role + job-id handoff, the sidecar wiring, the Dependency primitive, and the // standalone/observe paths. Shared fixtures (qpuPod, cpuPod, op helpers) live in @@ -51,12 +51,12 @@ func TestObserveLabelInjectsSidecar(t *testing.T) { } } -// ── shared coordination: scout / worker split ────────────────────────────── +// ── shared coordination. Scout / worker split ────────────────────────────── // sharedQPUPod is a quantum workload pod (requests the resource) in a group, // owned by a Job of parallelism N, with coordination=shared and a completion -// index. Index "0" is the scout; any other index is a worker. This is the -// real shape: an indexed Job whose identical template yields differentiated +// index. Index "0" is the scout. Any other index is a worker. This is the +// real shape. An indexed Job whose identical template yields differentiated // roles purely from the completion index. func sharedQPUPod(ns, group, name, job, index string) *corev1.Pod { p := qpuPod("fluence") @@ -72,7 +72,7 @@ func sharedQPUPod(ns, group, name, job, index string) *corev1.Pod { } // gangQPUPod is a quantum workload pod in a group owned by a Job, with NO -// coordination annotation — i.e. the default (independent) mode. +// coordination annotation, i.e. the default (independent) mode. func gangQPUPod(ns, group, name, job string) *corev1.Pod { p := qpuPod("fluence") p.Name = name @@ -95,7 +95,7 @@ func mincount(t *testing.T, cs *fake.Clientset, ns, group string) (int32, bool) // A shared-mode WORKER (completion index != 0, owned by Job parallelism=3) is // gated, told its role (FLUENCE_COORDINATION_ROLE=worker), joins the // worker gang at minCount N-1 (the split), and gets NO sidecar (it is gated). -// No separate submitter pod is ever created — the scout is one of the N members. +// No separate submitter pod is ever created, the scout is one of the N members. func TestSharedWorkerGatedRoleAndSplit(t *testing.T) { ns, group, job := "default", "qg", "qg-job" par := int32(3) @@ -129,10 +129,10 @@ func TestSharedWorkerGatedRoleAndSplit(t *testing.T) { } } -// The shared-mode SCOUT (completion index 0) is wired as the real coordinator: +// The shared-mode SCOUT (completion index 0) is wired as the real coordinator. // its own group-of-one -scout at minCount 1, the real sidecar, not // gated, role=scout, and told which worker group to ungate via -// FLUENCE_GANG_GROUP. It is one of the N members — no extra pod is created. +// FLUENCE_GANG_GROUP. It is one of the N members, no extra pod is created. func TestSharedScoutWiredAsRealSidecar(t *testing.T) { ns, group, job := "default", "qg2", "qg2-job" par := int32(2) @@ -189,7 +189,7 @@ func TestSharedScoutWiredAsRealSidecar(t *testing.T) { } } -// Shared mode never creates an extra pod: a full gang (scout index 0 + +// Shared mode never creates an extra pod. A full gang (scout index 0 + // workers) is N members, so the application runs exactly N times (not N+1 as // the old submitter-pod model did). func TestSharedGangNoSeparateSubmitterPod(t *testing.T) { @@ -218,8 +218,8 @@ func TestSharedGangNoSeparateSubmitterPod(t *testing.T) { // ── independent mode (default) ────────────────────────────────────────────────── -// A grouped quantum pod with no coordination annotation is INDEPENDENT (default): -// it does its own real submit, is not gated, carries no coordination role, and +// A grouped quantum pod with no coordination annotation is INDEPENDENT (default). +// it does its own real submit, is not gated, has no coordination role, and // triggers no group split and no submitter pod. func TestIndependentGroupedQuantumIsStandalone(t *testing.T) { ns, group, job := "default", "indep", "indep-job" @@ -247,7 +247,7 @@ func TestIndependentGroupedQuantumIsStandalone(t *testing.T) { } // A standalone quantum pod (no group, no owner → group of one) does its own real -// submit: interceptor staged, but no gating, no coordination role, no submitter. +// submit interceptor staged, but no gating, no coordination role, no submitter. func TestStandaloneQuantumIsReal(t *testing.T) { ns := "default" cs := fake.NewSimpleClientset() @@ -315,7 +315,7 @@ func envOp(ops []spec.Op, name string) (corev1.EnvVar, bool) { func annotationOps(ops []spec.Op) map[string]string { out := map[string]string{} for _, op := range ops { - // whole-map add: /metadata/annotations + // whole-map add. /metadata/annotations if op.Path == "/metadata/annotations" { if m, ok := op.Value.(map[string]string); ok { for k, v := range m { @@ -358,7 +358,7 @@ func unescapeJSONPointer(s string) string { return out } -// A shared-mode worker is expressed as a general Dependency: gated, stamped +// A shared-mode worker is expressed as a general Dependency. Gated, stamped // with depends-on-{kind,scout,gate}, and the scout is the -scout // group. func TestQuantumWorkerIsGeneralDependency(t *testing.T) { @@ -439,7 +439,7 @@ func TestQuantumWorkerStagedWithRole(t *testing.T) { // Classical override below the replica count: group-size=2 on a gang owned by a // Job(parallelism=5) must yield minCount=2 (the override), not 5. With a cluster -// sized to 2, the gang reaches quorum and runs; if the override were dropped the +// sized to 2, the gang reaches quorum and runs. If the override were dropped the // gang would wait forever for 5 (the e2e hang that fails CI). func TestClassicalOverrideBelowReplicaCount(t *testing.T) { ns, group, job := "default", "ovr2", "ovr2-job" @@ -459,7 +459,7 @@ func TestClassicalOverrideBelowReplicaCount(t *testing.T) { // The sidecar inherits the workload's secret/configMap-sourced credentials so it // can talk to the same backend, but NOT plain-value env. (Moved from the core -// webhook package: sidecar construction is now quantum-owned.) +// webhook package. Sidecar construction is now quantum-owned.) func TestSidecarInheritsWorkloadSecretEnv(t *testing.T) { m := &webhook.Mutator{Clientset: fake.NewSimpleClientset()} pod := &corev1.Pod{ diff --git a/pkg/webhook/handlers/registry_test.go b/pkg/webhook/handlers/registry_test.go index 08ed644..5e84f85 100644 --- a/pkg/webhook/handlers/registry_test.go +++ b/pkg/webhook/handlers/registry_test.go @@ -1,4 +1,4 @@ -// Registry behavior: dispatch order comes from the active handler list (not a +// Registry behavior. Dispatch order comes from the active handler list (not a // per-handler Order), and the list both selects and orders handlers. package handlers @@ -36,7 +36,7 @@ func TestDefaultOrderGangLast(t *testing.T) { } } -// The active list IS the order: passing a custom order reorders dispatch, and +// The active list IS the order. Passing a custom order reorders dispatch, and // unknown names are reported, not silently kept. func TestActiveListSetsOrderAndReportsUnknown(t *testing.T) { defer webhook.SetActiveHandlers(nil) @@ -49,7 +49,7 @@ func TestActiveListSetsOrderAndReportsUnknown(t *testing.T) { } } -// Dropping a handler from the list disables it: a quantum pod with quantum +// Dropping a handler from the list disables it. A quantum pod with quantum // omitted gets no interceptor ops (only fluxion/gang act). func TestOmittedHandlerDoesNotDispatch(t *testing.T) { defer webhook.SetActiveHandlers(nil) diff --git a/pkg/webhook/handlers/sidecar.go b/pkg/webhook/handlers/sidecar.go index d105a7c..bad103f 100644 --- a/pkg/webhook/handlers/sidecar.go +++ b/pkg/webhook/handlers/sidecar.go @@ -10,13 +10,13 @@ import ( ) // Sidecar is the capability a handler uses to attach a coordination sidecar to a -// pod. It is NOT part of the webhook core's MutatorAPI: only handlers that need +// pod. It is NOT part of the webhook core's MutatorAPI. Only handlers that need // a sidecar (today, quantum) depend on it, and a handler may supply its own // implementation to customize delivery. The default implementation // (coreSidecar) delegates to the webhook core's interceptor/sidecar ops, which // remain the staging mechanism shared by any sidecar-using handler. // -// This is the seam your design calls for: "a general sidecar interface that can +// This is the seam your design calls for. "a general sidecar interface that can // be used across handlers and customized by the quantum [handler]". A future // custom-resource handler can implement Sidecar differently (different image, // env, gating) without touching the core or other handlers. @@ -28,16 +28,16 @@ type Sidecar interface { // containers (init container + shared volume on PYTHONPATH). InterceptorOps(pod *corev1.Pod) []spec.Op // ContainerOps adds the sidecar container. observe=true selects observe-only - // telemetry mode (no ungating). extraEnv carries handler-computed, + // telemetry mode (no ungating). extraEnv has handler-computed, // domain-specific env (e.g. the quantum handler's FLUENCE_EXPECTED_WORKERS = // N-1 and FLUENCE_WORKER_GROUP_BASE) so the core never has to know about - // leader/worker concepts — the handler that owns the split owns those values. + // leader/worker concepts, the handler that owns the split owns those values. ContainerOps(pod *corev1.Pod, observe bool, extraEnv []corev1.EnvVar) []spec.Op } // coreSidecar is the default Sidecar. It delegates to the quantum-owned sidecar // implementation (see sidecar_impl.go), which uses only the generic MutatorAPI -// (Client, InjectedEnv). The webhook core no longer carries any sidecar logic; a +// (Client, InjectedEnv). The webhook core no longer has any sidecar logic. A // custom handler could supply its own Sidecar with a different container/image. type coreSidecar struct{ m webhook.MutatorAPI } diff --git a/pkg/webhook/spec/spec.go b/pkg/webhook/spec/spec.go index 18aa780..9ffad48 100644 --- a/pkg/webhook/spec/spec.go +++ b/pkg/webhook/spec/spec.go @@ -1,6 +1,6 @@ // Package spec holds the JSON patch type and pure pod-inspection / patch-op // builders shared by the webhook core and its handlers. Everything here is a -// pure function of its inputs — no Kubernetes client, no Mutator — so both the +// pure function of its inputs, no Kubernetes client, no Mutator, so both the // core package and the handlers subpackage can import it without an import // cycle. package spec @@ -26,7 +26,7 @@ type Op struct { EmitNull bool `json:"-"` } -// MarshalJSON honors EmitNull; otherwise it omits value when nil and includes it +// MarshalJSON honors EmitNull. Otherwise it omits value when nil and includes it // when present, matching the previous struct-tag behavior. func (o Op) MarshalJSON() ([]byte, error) { m := map[string]any{"op": o.Op, "path": o.Path} diff --git a/pkg/webhook/webhook.go b/pkg/webhook/webhook.go index b81fcba..03137f3 100644 --- a/pkg/webhook/webhook.go +++ b/pkg/webhook/webhook.go @@ -1,9 +1,9 @@ // Package webhook is fluence's mutating admission webhook. // -// The core here is domain-agnostic plumbing: it owns the Mutator, the handler +// The core here is domain-agnostic plumbing. It owns the Mutator, the handler // dispatcher, per-namespace PodGroup provisioning, the HTTP entrypoint, and // self-managed TLS. It knows nothing about quantum, Braket, gate names, sidecars, -// RBAC, or interceptor staging — that policy and machinery lives entirely in the +// RBAC, or interceptor staging, that policy and machinery lives entirely in the // handlers (pkg/webhook/handlers), which self-register via Register() and perform // their own create/edit side-effects through the generic MutatorAPI. // @@ -41,18 +41,18 @@ import ( // ── Constants ────────────────────────────────────────────────────────────────── const ( - // SchedulerName gates the whole webhook: only pods scheduled by Fluence are + // SchedulerName gates the whole webhook. Only pods scheduled by Fluence are // considered. This is the one cross-cutting constant the core owns. SchedulerName = "fluence" // GroupLabel marks a pod's gang membership. The core treats its value as an - // opaque group identity for PodGroup management; it ascribes no quantum + // opaque group identity for PodGroup management. It ascribes no quantum // meaning to it (a handler decides what a group means). GroupLabel = "fluence.flux-framework.org/group" // GroupSizeAnnotation is the gang member count N, set by the workload on each // pod. It is the authoritative override for the PodGroup gang minCount when - // the size cannot (or should not) be derived from the owning controller — and + // the size cannot (or should not) be derived from the owning controller, and // for loose grouped pods where counting at admission is unreliable. The core // treats it as an opaque integer string. GroupSizeAnnotation = "fluence.flux-framework.org/group-size" @@ -76,9 +76,9 @@ var _ MutatorAPI = (*Mutator)(nil) // GroupName returns the value of GroupLabel on the pod, or "". func GroupName(pod *corev1.Pod) string { return spec.Label(pod, GroupLabel) } -// ── MutatorAPI: capabilities exposed to handlers ──────────────────────────────── +// ── MutatorAPI. Capabilities exposed to handlers ──────────────────────────────── -// Client implements MutatorAPI: returns the Kubernetes client (nil in tests). +// Client implements MutatorAPI. Returns the Kubernetes client (nil in tests). func (m *Mutator) Client() kubernetes.Interface { return m.Clientset } // InjectedEnv is the FLUXION_* env contract sourced from scheduler annotations. @@ -119,10 +119,10 @@ func (m *Mutator) EnsurePodGroup(ctx context.Context, namespace, group, leaderPo ObjectMeta: metav1.ObjectMeta{ Name: group, Namespace: namespace, Labels: map[string]string{"app": "fluence", GroupLabel: group}, - // Ownership claim: the reconciler in pkg/fluence deletes a completed + // Ownership claim. The reconciler in pkg/fluence deletes a completed // gang's PodGroup (to free its Fluxion allocation) ONLY if this // annotation is present. A user-created PodGroup (e.g. a native gang - // not created by this webhook) never carries it and is never touched. + // not created by this webhook) never has it and is never touched. Annotations: map[string]string{placement.CreatedByAnnotation: placement.CreatedByValue}, }, Spec: schedulingv1alpha2.PodGroupSpec{ diff --git a/python/Dockerfile b/python/Dockerfile index 03bf153..b1e598b 100644 --- a/python/Dockerfile +++ b/python/Dockerfile @@ -1,6 +1,6 @@ # Fluence quantum coordination sidecar image. # TODO organize into subdirectories when we have >1 image -#sitecustomize; no install required in the user image +#sitecustomize. No install required in the user image FROM python:3.11-slim LABEL org.opencontainers.image.source="https://github.com/converged-computing/fluence" @@ -18,7 +18,7 @@ COPY . /app # Install the package with the vendor SDKs the SIDECAR needs for its own API # calls (task discovery / queue polling). The interceptor staged into the user -# container carries NONE of these — it patches whatever SDK the user already has. +# container has NONE of these, it patches whatever SDK the user already has. RUN pip install --no-cache-dir ".[all]" && ln -s $(which python3) /usr/bin/python ENV FLUENCE_TASK_DISCOVERY_TIMEOUT=300000 diff --git a/python/README.md b/python/README.md index e0231c4..576a17c 100644 --- a/python/README.md +++ b/python/README.md @@ -1,40 +1,40 @@ # fluence (Python) Quantum-classical scheduling coordination library for the Fluence Kubernetes -scheduler. Import name `fluence`; distributed on PyPI as `fluence-hpc`. +scheduler. Import name `fluence`. Distributed on PyPI as `fluence-hpc`. This package is **built into the Fluence sidecar image** and **staged into user -application containers at admission time** — users never install it. +application containers at admission time**, users never install it. ## What it does -A hybrid quantum-classical workflow submits work to two queues: the Kubernetes +A hybrid quantum-classical workflow submits work to two queues. The Kubernetes scheduler (classical) and a QPU vendor API (quantum). Classical worker pods would idle while the QPU queue drains. Fluence gates the workers until the quantum task -is about to run, then releases them. This library is the runtime half: +is about to run, then releases them. This library is the runtime half. -- **interceptor** (`fluence.interceptor`) — runs inside the user container, +- **interceptor** (`fluence.interceptor`), runs inside the user container, monkey-patches the vendor SDK submit call to tag each task with the pod UID. -- **sidecar** (`fluence.sidecar`) — runs in a sidecar container, discovers the +- **sidecar** (`fluence.sidecar`), runs in a sidecar container, discovers the tagged task, polls queue position, and ungates the classical workers when the task is ready (or, in observe-only mode, just records the queue position). -- **providers** (`fluence.providers`) — per-vendor plug-ins implementing both +- **providers** (`fluence.providers`), per-vendor plug-ins implementing both halves. Providers self-register on import. ## Delivery (Model C) The interceptor must run in the user's container, which does **not** have this package installed. Rather than require a user install or concatenate a text -snippet, the Fluence webhook: +snippet, the Fluence webhook. 1. injects an **init container** (the sidecar image) running `python -m fluence.stage `, which copies the pure-Python `fluence` - package plus a `sitecustomize.py` into a shared `emptyDir`; + package plus a `sitecustomize.py` into a shared `emptyDir`. 2. mounts that volume into the user container and prepends `` to `PYTHONPATH`. Python imports `sitecustomize` automatically on every interpreter start -(`python app.py` included — unlike `PYTHONSTARTUP`, which only fires for +(`python app.py` included, unlike `PYTHONSTARTUP`, which only fires for interactive sessions), so `import fluence.interceptor` runs before user code. The interceptor patches whichever vendor SDK is present and fail-soft skips the rest. No user code changes, no vendor SDKs added to the user image. @@ -58,7 +58,7 @@ moves the package version and the image tag together. ### Build the distributions -From `python/`: +From the `python/` directory, run pip install --upgrade build twine python -m build @@ -83,14 +83,14 @@ After this, `pip install fluence-hpc` works anywhere and imports as `fluence`. Bump `version` in `pyproject.toml` and `__version__` in `fluence/__init__.py` together (PyPI refuses to overwrite an existing version). Because the package is version-locked into the sidecar image, tag the release so the image and the -package share a version — e.g. a `v0.1.1` git tag triggers both the +package share a version, e.g. a `v0.1.1` git tag triggers both the `sidecar-build-deploy` workflow (image) and a PyPI publish. ### Automated release (recommended) Prefer GitHub Actions with PyPI Trusted Publishing (OIDC) over manual token -uploads: register the repo + workflow once on PyPI, then a release workflow +uploads register the repo + workflow once on PyPI, then a release workflow triggered by a version tag builds with `python -m build` and uploads with -`pypa/gh-action-pypi-publish` — no stored secret. The Docker image is built by +`pypa/gh-action-pypi-publish`, no stored secret. The Docker image is built by `.github/workflows/sidecar-build-deploy.yaml` on the same tag, keeping the package version and image tag in lockstep. diff --git a/python/fluence/__init__.py b/python/fluence/__init__.py index 26af3a1..f0fca4a 100644 --- a/python/fluence/__init__.py +++ b/python/fluence/__init__.py @@ -1,5 +1,5 @@ """ -fluence — quantum-classical scheduling coordination for the Fluence Kubernetes +fluence, quantum-classical scheduling coordination for the Fluence Kubernetes scheduler. This package is built into the Fluence sidecar image and staged into user diff --git a/python/fluence/interceptor.py b/python/fluence/interceptor.py index 51b1368..67399d7 100644 --- a/python/fluence/interceptor.py +++ b/python/fluence/interceptor.py @@ -1,12 +1,12 @@ """ -fluence.interceptor — installs every registered provider's submit-time tag hook. +fluence.interceptor, installs every registered provider's submit-time tag hook. Runs inside the user's application container, triggered automatically by a sitecustomize.py on PYTHONPATH (staged there by the Fluence init container). On -import it asks every registered provider to install its interceptor; each +import it asks every registered provider to install its interceptor. Each provider fail-soft skips if its vendor SDK is not present in this container. -This module's import must never raise — sitecustomize guards it, but we also +This module's import must never raise, sitecustomize guards it, but we also guard here so a single provider bug cannot affect the user application. """ diff --git a/python/fluence/providers/__init__.py b/python/fluence/providers/__init__.py index ce9c0f9..84670a4 100644 --- a/python/fluence/providers/__init__.py +++ b/python/fluence/providers/__init__.py @@ -1,13 +1,13 @@ """ -fluence.providers — provider registry. +fluence.providers, provider registry. Importing this package imports every provider submodule, each of which calls fluence.providers.base.register() at import time. This is the single extension -point: to add a vendor, drop a new module here that defines a Provider subclass -and calls register() — nothing else in the codebase needs to change. +point to add a vendor, drop a new module here that defines a Provider subclass +and calls register(), nothing else in the codebase needs to change. Provider discovery is by explicit submodule import below (simple and debuggable). -Importing a provider module never fails on a missing vendor SDK: the SDK is only +Importing a provider module never fails on a missing vendor SDK. The SDK is only imported lazily inside the methods that need it. """ diff --git a/python/fluence/providers/base.py b/python/fluence/providers/base.py index 9546784..9422f98 100644 --- a/python/fluence/providers/base.py +++ b/python/fluence/providers/base.py @@ -1,21 +1,21 @@ """ -fluence.providers.base — the provider interface and registration machinery. +fluence.providers.base, the provider interface and registration machinery. -A provider is a per-vendor plug-in (AWS Braket, IBM Qiskit Runtime, ...) that -implements two halves of the quantum-coordination mechanism: +A provider is a per-vendor plug-in (AWS Braket, IBM Qiskit Runtime...) that +implements two halves of the quantum-coordination mechanism. - INTERCEPTOR hook (`install_interceptor`): runs inside the user's application - container; monkey-patches the vendor SDK's submit call to stamp the shared + container. monkey-patches the vendor SDK's submit call to stamp the shared `fluence-pod-uid` tag on every task. Must fail-soft if the vendor SDK is not importable in that container. - SIDECAR methods (`matches`, `find_my_task`, `is_ready_to_ungate`, - `queue_position`, `job_id`): run inside the Fluence sidecar container; find + `queue_position`, `job_id`). Run inside the Fluence sidecar container. Find the tagged task, poll readiness, and yield a vendor-neutral job id. Providers self-register by calling `register()` at import time. The package imports every provider submodule (see fluence.providers.__init__) so importing -the package registers them all. Registration is the single extension point: +the package registers them all. Registration is the single extension point. adding a vendor is one new module that calls register(). """ @@ -25,7 +25,7 @@ from datetime import datetime, timezone # Shared convention between every interceptor hook and every find_my_task. -# The interceptor stamps this tag key with the pod UID; the sidecar searches +# The interceptor stamps this tag key with the pod UID. The sidecar searches # for it. Changing it is a coordinated change across all providers. TAG_KEY = "fluence-pod-uid" @@ -58,7 +58,7 @@ def log(msg: str) -> None: class Task: """ Opaque handle to a vendor quantum task. A provider returns its own subclass - from find_my_task; the framework treats it opaquely and only passes it back + from find_my_task. The framework treats it opaquely and only passes it back to that provider. Vendor identifiers (ARN, job id) live inside. """ @@ -85,11 +85,11 @@ def matches(self, attrs: "dict[str, str]") -> bool: """True if this provider handles the backend Fluxion selected. attrs is the full set of FLUXION_* values surfaced from the matched - qdevice's properties (backend, vendor, qrmi_type, arn, region, ...), so a + qdevice's properties (backend, vendor, qrmi_type, arn, region...), so a provider can route on whichever attribute is authoritative for it. For - Braket the routing key is qrmi_type (e.g. "braket-ahs", "braket-gate"): + Braket the routing key is qrmi_type (e.g. "braket-ahs", "braket-gate"). the device is reached through the Braket SDK regardless of which company - (amazon, quera, rigetti, iqm, ...) built the hardware. + (amazon, quera, rigetti, iqm...) built the hardware. """ raise NotImplementedError @@ -163,7 +163,7 @@ def resolve(attrs: "dict[str, str] | None" = None, **legacy) -> "Provider | None def resolve_from_env() -> "Provider | None": # Gather every FLUXION_* var the webhook injected (backend, vendor, - # qrmi_type, arn, region, ...) into a lowercased-key attribute dict. + # qrmi_type, arn, region...) into a lowercased-key attribute dict. attrs = {} for k, v in os.environ.items(): if k.startswith("FLUXION_"): diff --git a/python/fluence/providers/braket.py b/python/fluence/providers/braket.py index 87a671d..ac0edbc 100644 --- a/python/fluence/providers/braket.py +++ b/python/fluence/providers/braket.py @@ -1,14 +1,14 @@ """ -fluence.providers.braket — AWS Braket provider. +fluence.providers.braket, AWS Braket provider. -Holds both halves of the Braket coordination mechanism: +Holds both halves of the Braket coordination mechanism. - install_interceptor: patches AwsDevice.run() to stamp the pod-uid tag - (runs in the user container; fail-soft if amazon-braket-sdk is absent). - - sidecar methods: discover the tagged task, poll queue position, yield the + (runs in the user container. fail-soft if amazon-braket-sdk is absent). + - sidecar methods. Discover the tagged task, poll queue position, yield the task ARN as the (vendor-neutral-typed) job id. Self-registers via register(PROVIDER) at import. Importing this module never -requires the braket SDK; SDK imports are lazy, inside the methods. +requires the braket SDK. SDK imports are lazy, inside the methods. """ from __future__ import annotations @@ -38,7 +38,7 @@ def _region_from_arn(arn: str) -> str: # Prefer the region the scheduler chose for this backend (FLUXION_REGION, # injected from the resource graph's `region` attribute) over the pod's # generic AWS_DEFAULT_REGION. A QPU like IQM Emerald lives in eu-north-1, - # while the pod's AWS_DEFAULT_REGION is often us-east-1 — searching the wrong + # while the pod's AWS_DEFAULT_REGION is often us-east-1, searching the wrong # region means the task is never found and workers never ungate. return ( region @@ -56,7 +56,7 @@ def install_interceptor(self, pod_uid: str) -> bool: try: from braket.aws import AwsDevice except ImportError: - return False # braket SDK not in this container — fail-soft + return False # braket SDK not in this container, fail-soft original_run = AwsDevice.run @@ -75,10 +75,10 @@ def patched_run(self, task_specification, *args, **kwargs): # ── sidecar half ─────────────────────────────────────────────────────────── def matches(self, attrs: "dict[str, str]") -> bool: - # Authoritative routing key: qrmi_type. Every Braket-accessed device — + # Authoritative routing key. qrmi_type. Every Braket-accessed device # gate simulators (sv1/tn1/dm1), gate QPUs (rigetti/iqm/ionq), and AHS - # (aquila, ahs_local) — carries a "braket-*" qrmi_type, regardless of - # which company (amazon, quera, rigetti, iqm, ...) made the hardware. + # (aquila, ahs_local), has a "braket-*" qrmi_type, regardless of + # which company (amazon, quera, rigetti, iqm...) made the hardware. qrmi = (attrs.get("qrmi_type") or "").lower() if qrmi.startswith("braket"): return True @@ -115,13 +115,13 @@ def _client(self, backend: str): def _candidate_regions(self, backend: str): """Regions to search for the task, best guess first. If the backend is an - ARN or FLUXION_REGION is set, try that first; then fall back to the + ARN or FLUXION_REGION is set, try that first. Then fall back to the regions Braket operates in, because the QPU's region (e.g. eu-north-1 for IQM) frequently differs from the pod's AWS_DEFAULT_REGION (us-east-1) and a task only appears in SearchQuantumTasks in the region it was created. Set FLUENCE_BRAKET_REGIONS (comma-separated) to restrict the search to - regions the account can actually access — useful when a service control + regions the account can actually access, useful when a service control policy explicitly denies braket:SearchQuantumTasks in some regions (those raise AccessDeniedException and just add noise/latency).""" override = os.environ.get("FLUENCE_BRAKET_REGIONS", "").strip() @@ -160,7 +160,7 @@ def find_my_task(self, pod_uid, backend, timeout): f"across regions {regions}" ) deadline = time.time() + timeout - # SearchQuantumTasks has NO tags filter and is REGION-SCOPED: a task only + # SearchQuantumTasks has NO tags filter and is REGION-SCOPED. A task only # shows up in the region it was created in. We don't reliably know that # region up front (the backend may be a name, and the pod's default region # often differs from the QPU's), so we search each candidate region and @@ -174,7 +174,7 @@ def find_my_task(self, pod_uid, backend, timeout): clients = {} denied = ( set() - ) # regions that returned AccessDenied — skip them after the first hit + ) # regions that returned AccessDenied, skip them after the first hit while time.time() < deadline: for region in regions: if region in denied: @@ -209,8 +209,8 @@ def find_my_task(self, pod_uid, backend, timeout): arn = match["quantumTaskArn"] log(f"[braket] found task by tag in {region}: {arn}") return BraketTask(arn) - # Diagnostic: how many tasks did this region have, and how many - # carried OUR tag key at all? Distinguishes "task not here yet" + # Diagnostic. How many tasks did this region have, and how many + # had OUR tag key at all? Distinguishes "task not here yet" # from "task here but tag not landing". if seen: log( @@ -230,13 +230,13 @@ def find_my_task(self, pod_uid, backend, timeout): denied.add(region) log( f"[braket] region {region} not accessible " - f"(access denied) — skipping it for the rest of this search" + f"(access denied), skipping it for the rest of this search" ) else: log(f"[braket] search error in {region} (will retry): {e}") if len(denied) == len(regions): log( - "[braket] ERROR: every candidate region was access-denied — " + "[braket] ERROR: every candidate region was access-denied, " "set FLUENCE_BRAKET_REGIONS to a region this account can search" ) return None diff --git a/python/fluence/sidecar.py b/python/fluence/sidecar.py index 7a2a287..1356434 100644 --- a/python/fluence/sidecar.py +++ b/python/fluence/sidecar.py @@ -149,10 +149,10 @@ def main(): session_id = provider.session_id(task) n_ok = ungate_pods(gated_pods, job_id, namespace, session_id=session_id) if n_ok == len(gated_pods): - log(f"done — {n_ok} gang pod(s) ungated") + log(f"done, {n_ok} gang pod(s) ungated") else: log( - f"WARNING: ungated only {n_ok}/{len(gated_pods)} gang pod(s) — see errors above" + f"WARNING: ungated only {n_ok}/{len(gated_pods)} gang pod(s), see errors above" ) _hold_for_workers(namespace, gang_group, pod_name) diff --git a/python/fluence/sitecustomize.py b/python/fluence/sitecustomize.py index b7db9b7..2cae572 100644 --- a/python/fluence/sitecustomize.py +++ b/python/fluence/sitecustomize.py @@ -1,4 +1,4 @@ -# fluence sitecustomize — staged onto the user container's PYTHONPATH by the +# fluence sitecustomize, staged onto the user container's PYTHONPATH by the # Fluence init container. Python imports `sitecustomize` automatically on every # interpreter start (interactive OR script), so this runs the interceptor with # zero user code changes and without relying on PYTHONSTARTUP (which only fires diff --git a/python/fluence/stage.py b/python/fluence/stage.py index 16358a8..60bdc33 100644 --- a/python/fluence/stage.py +++ b/python/fluence/stage.py @@ -1,16 +1,16 @@ """ -fluence.stage — init-container entrypoint for Model C delivery. +fluence.stage, init-container entrypoint for Model C delivery. The Fluence webhook injects an init container (the sidecar image, which has `fluence` installed) that runs `python -m fluence.stage `. This copies the installed `fluence` package plus a `sitecustomize.py` into , a shared emptyDir volume. The webhook mounts that volume into the user's application container and prepends to PYTHONPATH. Python then auto-imports -sitecustomize on startup, which imports fluence.interceptor — tagging the user's +sitecustomize on startup, which imports fluence.interceptor, tagging the user's quantum tasks with zero user code changes and no vendor SDK requirement on our side (the interceptor patches whatever SDK the user already has). -Replaces the old build-interceptor.sh: assembly is real package staging, not +Replaces the old build-interceptor.sh. Assembly is real package staging, not text concatenation. Usage: @@ -30,8 +30,8 @@ def stage(dest: str) -> None: os.makedirs(dest, exist_ok=True) # Copy the installed `fluence` package into /fluence so it is importable - # when is on PYTHONPATH. We copy only the pure-Python package — no - # vendor SDKs — so we never perturb the user container's own dependencies. + # when is on PYTHONPATH. We copy only the pure-Python package, no + # vendor SDKs, so we never perturb the user container's own dependencies. pkg_src = os.path.dirname(os.path.abspath(fluence.__file__)) pkg_dst = os.path.join(dest, "fluence") if os.path.exists(pkg_dst): diff --git a/python/fluence/ungate.py b/python/fluence/ungate.py index 8cdfd5b..2be39bc 100644 --- a/python/fluence/ungate.py +++ b/python/fluence/ungate.py @@ -1,10 +1,10 @@ """ -fluence.ungate — generic worker ungating (Kubernetes side). +fluence.ungate, generic worker ungating (Kubernetes side). Once the sidecar determines the quantum task is ready, it ungates the gated -classical worker pods: stamp the vendor-neutral job-id annotation, set the +classical worker pods. Stamp the vendor-neutral job-id annotation, set the high-priority class, and remove the scheduling gate atomically. This is pure -Kubernetes plumbing — no vendor specifics. +Kubernetes plumbing, no vendor specifics. """ from __future__ import annotations @@ -31,13 +31,13 @@ def kubectl(args): def ungate_pods(gated_pods, job_id, namespace, session_id=None): """ - For each gated worker pod: + For each gated worker pod. 1. Stamp the job id annotation so the worker can locate the quantum result, and the session id when the scout has one, so the worker can submit into the same session. 2. Remove the scheduling gate so the pod can be scheduled. - NOTE: priorityClassName is NOT set here — it is immutable after pod creation + Note that priorityClassName is NOT set here, it is immutable after pod creation (the API server forbids changing any spec field but image/tolerations/ activeDeadlineSeconds/terminationGracePeriodSeconds on an existing pod). If a priority class is wanted on the classical gang, the webhook must set it at @@ -113,7 +113,7 @@ def gated_pods_from_env(): def discover_gated_pods(namespace, group, exclude=""): """ - Find the names of pods in the same group that still carry the quantum + Find the names of pods in the same group that still have the quantum scheduling gate (i.e. the gang pods this submitter must ungate). The submitter is created alongside the gang, so the gated set is discovered diff --git a/python/tests/test_fluence.py b/python/tests/test_fluence.py index 4e6c4e5..6843382 100644 --- a/python/tests/test_fluence.py +++ b/python/tests/test_fluence.py @@ -1,6 +1,6 @@ """ -Tests for the fluence package. Run: python3 -m pytest python/tests/ -None of these require a vendor SDK — they exercise registration, resolution, +Tests for the fluence package. Run. python3 -m pytest python/tests/ +None of these require a vendor SDK, they exercise registration, resolution, fail-soft interceptor behavior, and the staging mechanism. """ @@ -73,7 +73,7 @@ def test_stage_produces_importable_package(): def test_staged_sitecustomize_runs_interceptor(): # Stage, then run a subprocess with ONLY the staged dir on PYTHONPATH and a - # fake braket SDK; the interceptor must patch AwsDevice.run to inject the tag. + # fake braket SDK. The interceptor must patch AwsDevice.run to inject the tag. with tempfile.TemporaryDirectory() as d: staged = os.path.join(d, "staged") subprocess.run([sys.executable, "-m", "fluence.stage", staged], check=True) @@ -120,7 +120,7 @@ def test_braket_matches_by_qrmi_type_and_vendor(): def test_find_my_task_matches_tag_client_side(): - # SearchQuantumTasks has no tags filter; the provider must match the tag + # SearchQuantumTasks has no tags filter. The provider must match the tag # client-side. It calls search_quantum_tasks DIRECTLY (not the paginator, # which rejects empty filters) and pages via nextToken. Fake that here. from fluence.providers.braket import BraketProvider, BraketTask diff --git a/python/tests/test_qrmi.py b/python/tests/test_qrmi.py index ff9e4d0..dc7116e 100644 --- a/python/tests/test_qrmi.py +++ b/python/tests/test_qrmi.py @@ -1,10 +1,10 @@ """ Tests for the QRMI provider. Like test_fluence.py, none of these require the -real qrmi SDK — they exercise registration, routing, the resource-type mapping, +real qrmi SDK, they exercise registration, routing, the resource-type mapping, the handoff channel, fail-soft interceptor behavior, the status->ready mapping, and the free/paid (acquire) session capture using small fakes. -The provider is vendor-agnostic: the tests deliberately cover more than one QRMI +The provider is vendor-agnostic. The tests deliberately cover more than one QRMI vendor (IBM's qiskit-runtime-service AND Pasqal's pasqal-cloud) to prove nothing is IBM-specific. @@ -41,7 +41,7 @@ def test_resolve_qrmi_ibm_quantum_system_and_legacy(): def test_resolve_qrmi_pasqal_proves_vendor_agnostic(): # A DIFFERENT vendor (Pasqal) must route to the same provider with no code - # change — routing is data-driven from qrmi_type. + # change, routing is data-driven from qrmi_type. from fluence.providers import resolve assert resolve({"qrmi_type": "pasqal-cloud"}).name == "qrmi" @@ -62,7 +62,7 @@ def test_resolve_qrmi_by_explicit_transport_marker(): def test_qrmi_and_braket_do_not_collide(): from fluence.providers import resolve - # Braket-reached devices carry a braket-* qrmi_type and must go to braket. + # Braket-reached devices have a braket-* qrmi_type and must go to braket. assert resolve({"qrmi_type": "braket-gate", "vendor": "amazon"}).name == "braket" assert resolve({"qrmi_type": "braket-ahs", "vendor": "quera"}).name == "braket" # QRMI-reached devices must go to qrmi. @@ -161,7 +161,7 @@ def test_handoff_write_is_atomic_no_partial_file(): assert leftovers == [], leftovers -# ── interceptor: fail-soft + free/paid session capture ──────────────────────── +# ── interceptor. fail-soft + free/paid session capture ──────────────────────── def test_qrmi_install_interceptor_failsoft_without_qrmi(): diff --git a/test/e2e/03-restart-recovery.sh b/test/e2e/03-restart-recovery.sh index c26980f..e53950e 100644 --- a/test/e2e/03-restart-recovery.sh +++ b/test/e2e/03-restart-recovery.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -# Restart recovery: after the scheduler restarts, it must replay the existing allocation +# Restart recovery. After the scheduler restarts, it must replay the existing allocation # (via reapi update_allocate) and NOT double-book an exclusive qpu backend. set -euo pipefail HERE="$(cd "$(dirname "$0")" && pwd)"; . "${HERE}/lib.sh" @@ -15,7 +15,7 @@ backend="$(kubectl get pod sampler-mock -n "$NS" -o jsonpath="{.metadata.annotat [ -n "$backend" ] || fail "first pod has no backend annotation" log "first pod holds backend: $backend" -# 2. Restart the scheduler. Its in-memory Fluxion graph is rebuilt empty; recovery must +# 2. Restart the scheduler. Its in-memory Fluxion graph is rebuilt empty. Recovery must # replay the persisted allocation so the backend stays occupied. log "restarting fluence scheduler" kubectl rollout restart -n "${NS_KUBE}" deployment/fluence diff --git a/test/e2e/gang/01-classical-gang.sh b/test/e2e/gang/01-classical-gang.sh index 1ebfc64..727bf26 100755 --- a/test/e2e/gang/01-classical-gang.sh +++ b/test/e2e/gang/01-classical-gang.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -# Classical gang scheduling: a PodGroup of 2 must be placed all-or-nothing on real nodes. +# Classical gang scheduling. A PodGroup of 2 must be placed all-or-nothing on real nodes. set -euo pipefail HERE="$(cd "$(dirname "$0")" && pwd)"; . "${HERE%/test/e2e/*}/test/e2e/lib.sh" @@ -7,7 +7,7 @@ log "TEST 1: classical gang scheduling" kubectl apply -f examples/test/e2e/gang/single-podgroup.yaml # All pods in the 'training' deployment must reach Running (scheduled + started). -# Wait for the pod to EXIST before waiting for Ready — kubectl wait errors out +# Wait for the pod to EXIST before waiting for Ready, kubectl wait errors out # immediately if the Deployment's pod hasn't been registered yet (a race that # fails the test at 0s with "no matching resources found"). log "waiting for both training pods to schedule" @@ -27,7 +27,7 @@ count="$(kubectl get pods -l app=training --no-headers | wc -l | tr -d ' ')" log "PASS: classical gang placed all $count pods via fluence" kubectl delete -f examples/test/e2e/gang/single-podgroup.yaml --wait=false || true kubectl patch podgroup training --type=merge -p '{"metadata":{"finalizers":null}}' 2>/dev/null || true -# Wait for the pods to actually be gone before the next test runs — otherwise a +# Wait for the pods to actually be gone before the next test runs, otherwise a # terminating 'training' pod (same name/labels reused by other scenarios) can be # misread as the next test's placement. kubectl wait --for=delete pod -l app=training --timeout=60s 2>/dev/null || true diff --git a/test/e2e/gang/02-postfilter-rematch.sh b/test/e2e/gang/02-postfilter-rematch.sh index f74c87b..9282cd9 100755 --- a/test/e2e/gang/02-postfilter-rematch.sh +++ b/test/e2e/gang/02-postfilter-rematch.sh @@ -1,10 +1,10 @@ #!/usr/bin/env bash -# PostFilter re-match: when another scheduler plugin (TaintToleration) rejects a +# PostFilter re-match. When another scheduler plugin (TaintToleration) rejects a # node Fluxion allocated, Fluence must abandon that allocation, exclude the node, -# and re-match onto an untainted node. Safety: the gang's RUNNING pod must NEVER +# and re-match onto an untainted node. Safety. The gang's RUNNING pod must NEVER # bind to the tainted node. # -# This test is self-isolating: it uses its own workload name (pf-rematch) and +# This test is self-isolating. It uses its own workload name (pf-rematch) and # labels, distinct from the other e2e scenarios, and ensures a clean slate first, # so a pod left over (terminating) from a previous test can never be mistaken for # this test's placement. It also ignores terminating pods when asserting. @@ -22,7 +22,7 @@ kubectl delete podgroup "$NAME" --ignore-not-found >/dev/null 2>&1 || true kubectl patch podgroup "$NAME" --type=merge \ -p '{"metadata":{"finalizers":null}}' >/dev/null 2>&1 || true kubectl wait --for=delete pod -l "$SEL" --timeout=60s >/dev/null 2>&1 || true -# Defensive: a prior test's workload left running would occupy the only +# Defensive. A prior test's workload left running would occupy the only # untainted worker and make this test fail with a (correct) fluxion # allocate -1 for lack of capacity. Ensure none lingers. kubectl delete deployment training --ignore-not-found --wait=false >/dev/null 2>&1 || true @@ -80,9 +80,9 @@ YAML log "waiting for the gang to schedule (must avoid the tainted node)" wait_pods_ready "$SEL" 1 180 \ - || fail "gang never became Ready — PostFilter re-match did not recover (likely stuck on the taint-rejected allocation)" + || fail "gang never became Ready, PostFilter re-match did not recover (likely stuck on the taint-rejected allocation)" -# SAFETY: among NON-terminating (Running, no deletionTimestamp) pods, none may be +# SAFETY. Among NON-terminating (Running, no deletionTimestamp) pods, none may be # on the tainted node. Terminating leftovers are ignored by construction (we use # a unique name and cleaned the slate), but we still filter defensively. checked=0 @@ -103,7 +103,7 @@ done < <(kubectl get pods -l "$SEL" \ [ "$checked" -ge 1 ] || fail "no running ${NAME} pod found to check" -# Informational: did PostFilter actually fire (Fluxion picked the tainted node +# Informational. Did PostFilter actually fire (Fluxion picked the tainted node # first and we re-matched), or did Fluxion place on the good node directly? POD="$(kubectl -n kube-system get pods -l app=fluence \ -o jsonpath='{.items[0].metadata.name}' 2>/dev/null || true)" diff --git a/test/e2e/gang/03-multi-gang.sh b/test/e2e/gang/03-multi-gang.sh index 9f01ae5..d31d222 100755 --- a/test/e2e/gang/03-multi-gang.sh +++ b/test/e2e/gang/03-multi-gang.sh @@ -2,7 +2,7 @@ # Multi-pod gang scheduling on real nodes. Guards the two failures that the # single-pod 01 test could NOT catch (and that shipped a minCount=1 bug): # A) a multi-pod gang must place ALL of them (minCount must equal the gang size, not 1) -# B) under contention, a gang that cannot fully fit stays ENTIRELY pending — +# B) under contention, a gang that cannot fully fit stays ENTIRELY pending # never partially placed (no stranded pods holding nodes). set -euo pipefail HERE="$(cd "$(dirname "$0")" && pwd)"; . "${HERE%/test/e2e/*}/test/e2e/lib.sh" @@ -36,10 +36,10 @@ kubectl patch podgroup gang3 --type=merge -p '{"metadata":{"finalizers":null}}' kubectl wait --for=delete pod -l app=gang3 --timeout=60s 2>/dev/null || true # ---- B) contention: the gang that can't fully fit stays ENTIRELY pending -------- -log "TEST 6B: contention — a gang that cannot fully fit must NOT partially place" +log "TEST 6B: contention, a gang that cannot fully fit must NOT partially place" kubectl apply -f examples/test/e2e/gang/multi-gang-contention.yaml -# wait until the cluster settles. Three possible outcomes: +# wait until the cluster settles. Three possible outcomes. # - one gang fully Running, other fully Pending -> contention; assert no partial # - BOTH fully Running -> runner big enough, no contention to test (skip) # - any partial (1 of 2 in a gang scheduled) -> the bug, fail @@ -55,15 +55,15 @@ for i in $(seq 1 90); do done if [ -n "$both" ]; then - log "SKIP 6B: cluster placed both gangs (>=4 schedulable cores) — no contention on this runner" + log "SKIP 6B: cluster placed both gangs (>=4 schedulable cores), no contention on this runner" else [ -n "$winner" ] || fail "no clean settle: gang-a=$ra gang-b=$rb running (possible PARTIAL placement)" log "winner=$winner (2 running), loser=$loser (expected 0 running)" - # the loser must have ZERO pods scheduled to a node — the all-or-nothing guarantee. + # the loser must have ZERO pods scheduled to a node, the all-or-nothing guarantee. # A single scheduled loser pod = partial placement = the bug. scheduled_loser="$(kubectl get pods -l app=$loser -o jsonpath='{range .items[*]}{.spec.nodeName}{"\n"}{end}' | grep -c . || true)" - [ "$scheduled_loser" = "0" ] || fail "$loser has $scheduled_loser pod(s) on a node — PARTIAL placement (gang violated)" - log "PASS 6B: $loser stayed entirely pending — no partial placement under contention" + [ "$scheduled_loser" = "0" ] || fail "$loser has $scheduled_loser pod(s) on a node, PARTIAL placement (gang violated)" + log "PASS 6B: $loser stayed entirely pending, no partial placement under contention" fi kubectl delete -f examples/test/e2e/gang/multi-gang-contention.yaml --wait=false || true diff --git a/test/e2e/gang/04-requeue-on-capacity.sh b/test/e2e/gang/04-requeue-on-capacity.sh index f41aa71..984eefd 100755 --- a/test/e2e/gang/04-requeue-on-capacity.sh +++ b/test/e2e/gang/04-requeue-on-capacity.sh @@ -2,19 +2,19 @@ # Requeue-on-capacity + gang atomicity under contention. # # Two 2-pod gangs contend for a cluster that can only run one at a time. This -# guards two invariants that the GKE contention runs exposed: -# 1. ALL-OR-NOTHING: each gang places ALL its pods or NONE — never a partial -# (e.g. 1-of-2 scheduled). The winner must be a clean 2/2; the loser a clean +# guards two invariants that the GKE contention runs exposed. +# 1. ALL-OR-NOTHING. Each gang places ALL its pods or NONE, never a partial +# (e.g. 1-of-2 scheduled). The winner must be a clean 2/2. The loser a clean # 0/2 while it waits. -# 2. REQUEUE: when the winner completes and frees its nodes, the loser is +# 2. REQUEUE. When the winner completes and frees its nodes, the loser is # re-attempted on its own (no manual nudge) and then ALSO places atomically # (2/2), driven by the shortened --pod-max-in-unschedulable-pods-duration. # -# SCOPE / LIMITATION: this is a 3-node kind cluster with small (1-core) pods. It +# SCOPE / LIMITATION. This is a 3-node kind cluster with small (1-core) pods. It # verifies the INVARIANTS on a minimal contention case. It does NOT reproduce the -# GKE-scale dynamics where the bug was first seen — one-pod-per-node (~80-core) +# GKE-scale dynamics where the bug was first seen, one-pod-per-node (~80-core) # saturation and ~20 simultaneous mixed-size gangs draining in sequence. That -# scale behavior is validated on the real cluster, not in CI; a pass here means +# scale behavior is validated on the real cluster, not in CI. A pass here means # the invariants hold on the simple case, not that large-scale draining is proven. set -euo pipefail HERE="$(cd "$(dirname "$0")" && pwd)"; . "${HERE%/test/e2e/*}/test/e2e/lib.sh" @@ -34,7 +34,7 @@ for i in $(seq 1 60); do rw="$(running gang-win)"; ra="$(running gang-wait)" if [ "$rw" = "2" ] && [ "$ra" = "0" ]; then winner=gang-win; loser=gang-wait; break; fi if [ "$ra" = "2" ] && [ "$rw" = "0" ]; then winner=gang-wait; loser=gang-win; break; fi - # a 1/x or x/1 state that persists is a PARTIAL gang — fail fast on it + # a 1/x or x/1 state that persists is a PARTIAL gang, fail fast on it if [ "$rw" = "1" ] || [ "$ra" = "1" ]; then sleep 6 # allow a transient mid-bind moment to resolve rw="$(running gang-win)"; ra="$(running gang-wait)" @@ -46,10 +46,10 @@ done [ -n "$winner" ] || fail "no clean 2/0 split (gang-win=$(running gang-win) gang-wait=$(running gang-wait))" log " winner=$winner (2/2 running), loser=$loser" -# loser must have ZERO pods on any node — not even one (that would be a partial) +# loser must have ZERO pods on any node, not even one (that would be a partial) sl="$(on_nodes "$loser")" -[ "$sl" = "0" ] || fail "$loser has $sl pod(s) bound while it should be entirely pending — PARTIAL placement" -log " $loser entirely pending (0 pods bound) — all-or-nothing holds" +[ "$sl" = "0" ] || fail "$loser has $sl pod(s) bound while it should be entirely pending, PARTIAL placement" +log " $loser entirely pending (0 pods bound), all-or-nothing holds" # ---- 2. winner completes -> loser is requeued AND places atomically ------------ log "waiting for winner=$winner to complete and free its nodes" @@ -57,7 +57,7 @@ kubectl wait --for=condition=complete job/$winner --timeout=120s || fail "$winne log " $winner completed; capacity freed" # The loser must now place ALL its pods (2/2), on its own, within a window above -# the 30s recheck flush but below the 5m default — proving the shortened timeout +# the 30s recheck flush but below the 5m default, proving the shortened timeout # is in effect AND that the requeued gang is still atomic (not a partial). log "asserting $loser requeues and places ATOMICALLY (2/2) within ~75s" ok="" @@ -70,11 +70,11 @@ for i in $(seq 1 38); do # ~75s if [ "$((rl + dl))" = "1" ]; then sleep 6 rl="$(running $loser)"; dl="$(kubectl get pods -l job-name=$loser --field-selector=status.phase=Succeeded --no-headers 2>/dev/null | wc -l | tr -d ' ')" - [ "$((rl + dl))" = "1" ] && fail "$loser placed 1 of 2 pods — PARTIAL placement of the requeued gang" + [ "$((rl + dl))" = "1" ] && fail "$loser placed 1 of 2 pods, PARTIAL placement of the requeued gang" fi sleep 2 done -[ -n "$ok" ] || fail "$loser did NOT place both pods within 75s of capacity freeing — \ +[ -n "$ok" ] || fail "$loser did NOT place both pods within 75s of capacity freeing, \ either the shortened --pod-max-in-unschedulable-pods-duration is not taking effect \ (gang stuck) or the requeued gang did not assemble" log "PASS 9: $loser requeued and placed atomically (2/2) after $winner freed capacity" diff --git a/test/e2e/lib.sh b/test/e2e/lib.sh index 13390c9..da9d44d 100644 --- a/test/e2e/lib.sh +++ b/test/e2e/lib.sh @@ -10,7 +10,7 @@ log() { echo "=== $*"; } # Wait until at least N pods matching a label selector EXIST, then wait for them # to be Ready. `kubectl wait` errors immediately ("no matching resources found") # if the pods are not yet registered, so a Deployment that has only just been -# applied races it — wait for existence first. +# applied races it, wait for existence first. wait_pods_ready() { local selector="$1" want="${2:-1}" timeout="${3:-180}" local deadline=$(( $(date +%s) + timeout )) diff --git a/test/e2e/quantum/01-quantum-placement.sh b/test/e2e/quantum/01-quantum-placement.sh index 8f5c475..b46a116 100755 --- a/test/e2e/quantum/01-quantum-placement.sh +++ b/test/e2e/quantum/01-quantum-placement.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -# Quantum placement: a qpu pod is matched to a backend and the webhook injects FLUXION_BACKEND. +# Quantum placement. A qpu pod is matched to a backend and the webhook injects FLUXION_BACKEND. set -euo pipefail HERE="$(cd "$(dirname "$0")" && pwd)"; . "${HERE%/test/e2e/*}/test/e2e/lib.sh" ANN="fluence.flux-framework.org/backend" @@ -17,7 +17,7 @@ log "fluence chose backend: $backend" # The webhook must have surfaced it as FLUXION_BACKEND inside the container. out="$(kubectl logs sampler-mock || true)" if ! echo "$out" | grep -q "BACKEND=${backend}"; then - # Diagnostic (CI has no interactive shell): show whether the env var is ABSENT + # Diagnostic (CI has no interactive shell). Show whether the env var is ABSENT # (not injected -> webhook issue) or PRESENT-BUT-EMPTY (annotation not resolved # at container start -> delivery/timing issue), and what the container actually got. log "--- diagnostic: container env spec ---" diff --git a/test/e2e/quantum/02-sidecar-ungate.sh b/test/e2e/quantum/02-sidecar-ungate.sh index a4ae79f..7084afa 100755 --- a/test/e2e/quantum/02-sidecar-ungate.sh +++ b/test/e2e/quantum/02-sidecar-ungate.sh @@ -1,26 +1,26 @@ #!/usr/bin/env bash -# Shared-coordination webhook test (producer/consumer, no submitter pod). +# Shared-coordination webhook test (scout/worker, no submitter pod). # # When a shared quantum gang (coordination=shared, N pods all requesting QPU) is -# submitted, the webhook must: +# submitted, the webhook must. # 1. create the fluence-sidecar RBAC in the namespace automatically -# 2. gate every CONSUMER pod with quantum.braket/ready -# 3. raise every CONSUMER pod to the fluence-quantum-classical priority class -# 4. leave the PRODUCER (completion index 0) UNGATED, as a real member (NOT a +# 2. gate every WORKER pod with quantum.braket/ready +# 3. raise every WORKER pod to the fluence-quantum-classical priority class +# 4. leave the SCOUT (completion index 0) UNGATED, as a real member (NOT a # separate spawned pod) # 5. inject the fluence-stage init container + the sidecar container into the -# producer (Model C staging + the real coordinator) +# scout (Model C staging + the real coordinator) # # Does NOT test the sidecar runtime (task discovery, interceptor, queue polling) -# — that needs real AWS creds (sidecars/providers/braket/test/integration.sh). +#, that needs real AWS creds (sidecars/providers/braket/test/integration.sh). set -euo pipefail HERE="$(cd "$(dirname "$0")" && pwd)"; . "${HERE%/test/e2e/*}/test/e2e/lib.sh" GROUP=qgang -PRODUCER=${GROUP}-0 # completion index 0 -CONSUMER=${GROUP}-1 # completion index 1 +SCOUT=${GROUP}-0 # completion index 0 +WORKER=${GROUP}-1 # completion index 1 -log "TEST 4: shared-gang webhook — RBAC, consumer gating, priority, producer wiring" +log "TEST 4: shared-gang webhook, RBAC, worker gating, priority, scout wiring" kubectl apply -f examples/test/e2e/quantum/quantum-gang-pods.yaml sleep 3 @@ -39,38 +39,38 @@ kubectl get role fluence-sidecar -n default || fail "no fluence-sidec kubectl get rolebinding fluence-sidecar -n default || fail "no fluence-sidecar RoleBinding" log " RBAC present" -# 2 + 3. The CONSUMER is gated and at the preempting priority class. -gate="$(kubectl get pod "$CONSUMER" -o jsonpath='{.spec.schedulingGates[0].name}' 2>/dev/null || true)" -[ "$gate" = "quantum.braket/ready" ] || fail "$CONSUMER not gated (gate=$gate)" -pc="$(kubectl get pod "$CONSUMER" -o jsonpath='{.spec.priorityClassName}' 2>/dev/null || true)" -[ "$pc" = "fluence-quantum-classical" ] || fail "$CONSUMER priorityClass=$pc, want fluence-quantum-classical" -log " consumer gated + fluence-quantum-classical priority" +# 2 + 3. The WORKER is gated and at the preempting priority class. +gate="$(kubectl get pod "$WORKER" -o jsonpath='{.spec.schedulingGates[0].name}' 2>/dev/null || true)" +[ "$gate" = "quantum.braket/ready" ] || fail "$WORKER not gated (gate=$gate)" +pc="$(kubectl get pod "$WORKER" -o jsonpath='{.spec.priorityClassName}' 2>/dev/null || true)" +[ "$pc" = "fluence-quantum-classical" ] || fail "$WORKER priorityClass=$pc, want fluence-quantum-classical" +log " worker gated + fluence-quantum-classical priority" -# 4. The PRODUCER is NOT a separate spawned pod and is NOT gated. No -submitter. +# 4. The SCOUT is NOT a separate spawned pod and is NOT gated. No -submitter. if kubectl get pod "${GROUP}-submitter" -n default >/dev/null 2>&1; then - fail "found ${GROUP}-submitter pod — the obsolete separate-submitter model must not exist" + fail "found ${GROUP}-submitter pod, the obsolete separate-submitter model must not exist" fi -pgate="$(kubectl get pod "$PRODUCER" -o jsonpath='{.spec.schedulingGates[0].name}' 2>/dev/null || true)" -[ -z "$pgate" ] || fail "producer must NOT be gated (gate=$pgate)" -log " producer is a real member, not gated; no separate submitter pod" +pgate="$(kubectl get pod "$SCOUT" -o jsonpath='{.spec.schedulingGates[0].name}' 2>/dev/null || true)" +[ -z "$pgate" ] || fail "scout must NOT be gated (gate=$pgate)" +log " scout is a real member, not gated; no separate submitter pod" -# 5. Producer has the staging init container + the sidecar container. -wait_pod_phase "$PRODUCER" Running 120 \ - || { kubectl describe pod "$PRODUCER"; fail "$PRODUCER did not reach Running"; } -initc="$(kubectl get pod "$PRODUCER" -o jsonpath='{.spec.initContainers[*].name}')" +# 5. Scout has the staging init container + the sidecar container. +wait_pod_phase "$SCOUT" Running 120 \ + || { kubectl describe pod "$SCOUT"; fail "$SCOUT did not reach Running"; } +initc="$(kubectl get pod "$SCOUT" -o jsonpath='{.spec.initContainers[*].name}')" echo "$initc" | grep -q fluence-stage || fail "fluence-stage init container not injected (init: $initc)" -conts="$(kubectl get pod "$PRODUCER" -o jsonpath='{.spec.containers[*].name}')" +conts="$(kubectl get pod "$SCOUT" -o jsonpath='{.spec.containers[*].name}')" echo "$conts" | grep -q fluence-sidecar || fail "fluence-sidecar container not injected (containers: $conts)" -log " producer has fluence-stage + fluence-sidecar" +log " scout has fluence-stage + fluence-sidecar" -log "PASS: webhook gated the consumers, set priority, created RBAC + wired the producer" +log "PASS: webhook gated the workers, set priority, created RBAC + wired the scout" log "NOTE: priority is set at admission (immutable post-creation)" log "NOTE: braket sidecar runtime (SDK intercept, tag discovery, queue polling)" log " is in sidecars/providers/braket/test/integration.sh" -# Clean up pods + PodGroups; RBAC is namespace infra and persists. +# Clean up pods + PodGroups. RBAC is namespace infra and persists. kubectl delete -f examples/test/e2e/quantum/quantum-gang-pods.yaml --wait=false || true -for g in "$GROUP" "${GROUP}-producer"; do +for g in "$GROUP" "${GROUP}-scout"; do kubectl patch podgroup "$g" --type=merge -p '{"metadata":{"finalizers":null}}' 2>/dev/null || true done kubectl wait --for=delete pod -l app="$GROUP" --timeout=60s 2>/dev/null || true diff --git a/test/e2e/quantum/03-gang-producer.sh b/test/e2e/quantum/03-gang-producer.sh deleted file mode 100644 index fce4248..0000000 --- a/test/e2e/quantum/03-gang-producer.sh +++ /dev/null @@ -1,70 +0,0 @@ -#!/usr/bin/env bash -# Producer/consumer structure (replaces the old leader/worker and submitter-pod -# models). -# -# The structural guarantee the ungate path depends on: a shared quantum gang of -# size N is split, by completion index, into the CONSUMER gang -# (minCount N-1, gated) and the PRODUCER's group-of-one -producer -# (minCount 1, not gated). The producer is a real member of the user's workload — -# there is NO separate -submitter pod, NO -workers subgroup, and no -# leader among the user's pods. (The runtime ungate is covered by the braket -# integration test; here we prove the shape.) -set -euo pipefail -HERE="$(cd "$(dirname "$0")" && pwd)"; . "${HERE%/test/e2e/*}/test/e2e/lib.sh" - -GROUP=qgang -PRODUCER_GROUP=${GROUP}-producer -PRODUCER=${GROUP}-0 # completion index 0 -CONSUMER=${GROUP}-1 # completion index 1 - -log "TEST 7: consumer gang(N-1, gated) + producer(1, member) structure" -kubectl apply -f examples/test/e2e/quantum/quantum-gang-pods.yaml - -# Consumer PodGroup exists with minCount N-1 = 1 (the split). -log "checking consumer group '$GROUP' minCount == 1 (N-1)" -for i in $(seq 1 30); do - gc="$(kubectl get podgroup "$GROUP" -o jsonpath='{.spec.schedulingPolicy.gang.minCount}' 2>/dev/null || true)" - [ -n "$gc" ] && break; sleep 2 -done -[ "$gc" = "1" ] || fail "consumer group $GROUP minCount=$gc, want 1 (N-1)" - -# There must be NO -workers subgroup and NO -submitter pod. -if kubectl get podgroup "${GROUP}-workers" >/dev/null 2>&1; then - fail "found ${GROUP}-workers PodGroup — the obsolete leader/worker split must not exist" -fi -if kubectl get pod "${GROUP}-submitter" >/dev/null 2>&1; then - fail "found ${GROUP}-submitter pod — the obsolete separate-submitter model must not exist" -fi -log " consumer group minCount=1, no -workers subgroup, no -submitter pod" - -# Producer PodGroup -producer exists with minCount 1 (schedules alone). -log "checking producer group '$PRODUCER_GROUP' minCount == 1" -for i in $(seq 1 30); do - sc="$(kubectl get podgroup "$PRODUCER_GROUP" -o jsonpath='{.spec.schedulingPolicy.gang.minCount}' 2>/dev/null || true)" - [ -n "$sc" ] && break; sleep 2 -done -[ "$sc" = "1" ] || fail "producer group $PRODUCER_GROUP minCount=$sc, want 1" - -# Producer pod (index 0) is relinked into its own group-of-one and is NOT gated. -pl="$(kubectl get pod "$PRODUCER" -o jsonpath='{.metadata.labels.fluence\.flux-framework\.org/group}' 2>/dev/null || true)" -[ "$pl" = "$PRODUCER_GROUP" ] || fail "producer group label=$pl, want $PRODUCER_GROUP" -pgate="$(kubectl get pod "$PRODUCER" -o jsonpath='{.spec.schedulingGates[0].name}' 2>/dev/null || true)" -[ -z "$pgate" ] || fail "producer must NOT be gated (gate=$pgate)" -log " producer in '$PRODUCER_GROUP' (minCount 1), not gated" - -# Consumer pod (index 1+) stays in and is gated. -g="$(kubectl get pod "$CONSUMER" -o jsonpath='{.metadata.labels.fluence\.flux-framework\.org/group}' 2>/dev/null || true)" -[ "$g" = "$GROUP" ] || fail "$CONSUMER group label=$g, want $GROUP" -gate="$(kubectl get pod "$CONSUMER" -o jsonpath='{.spec.schedulingGates[0].name}' 2>/dev/null || true)" -[ "$gate" = "quantum.braket/ready" ] || fail "$CONSUMER not gated (gate=$gate)" -# The consumer's dependency points at the producer group. -dp="$(kubectl get pod "$CONSUMER" -o jsonpath='{.metadata.annotations.fluence\.flux-framework\.org/depends-on-producer}' 2>/dev/null || true)" -[ "$dp" = "$PRODUCER_GROUP" ] || fail "consumer depends-on-producer=$dp, want $PRODUCER_GROUP" -log " consumer in '$GROUP', gated, depends on '$PRODUCER_GROUP'" - -log "PASS 7: consumer gang(N-1, gated) + producer(1, member, ungates gang), no submitter/leader/worker" -kubectl delete -f examples/test/e2e/quantum/quantum-gang-pods.yaml --wait=false || true -for g in "$GROUP" "$PRODUCER_GROUP"; do - kubectl patch podgroup "$g" --type=merge -p '{"metadata":{"finalizers":null}}' 2>/dev/null || true -done -kubectl wait --for=delete pod -l app="$GROUP" --timeout=60s 2>/dev/null || true diff --git a/test/e2e/quantum/03-gang-scout.sh b/test/e2e/quantum/03-gang-scout.sh new file mode 100644 index 0000000..d762c08 --- /dev/null +++ b/test/e2e/quantum/03-gang-scout.sh @@ -0,0 +1,70 @@ +#!/usr/bin/env bash +# Scout/worker structure (replaces the old leader/worker and submitter-pod +# models). +# +# The structural guarantee the ungate path depends on. A shared quantum gang of +# size N is split, by completion index, into the WORKER gang +# (minCount N-1, gated) and the SCOUT's group-of-one -scout +# (minCount 1, not gated). The scout is a real member of the user's workload +# there is NO separate -submitter pod, NO -workers subgroup, and no +# leader among the user's pods. (The runtime ungate is covered by the braket +# integration test. Here we prove the shape.) +set -euo pipefail +HERE="$(cd "$(dirname "$0")" && pwd)"; . "${HERE%/test/e2e/*}/test/e2e/lib.sh" + +GROUP=qgang +SCOUT_GROUP=${GROUP}-scout +SCOUT=${GROUP}-0 # completion index 0 +WORKER=${GROUP}-1 # completion index 1 + +log "TEST 7: worker gang(N-1, gated) + scout(1, member) structure" +kubectl apply -f examples/test/e2e/quantum/quantum-gang-pods.yaml + +# Worker PodGroup exists with minCount N-1 = 1 (the split). +log "checking worker group '$GROUP' minCount == 1 (N-1)" +for i in $(seq 1 30); do + gc="$(kubectl get podgroup "$GROUP" -o jsonpath='{.spec.schedulingPolicy.gang.minCount}' 2>/dev/null || true)" + [ -n "$gc" ] && break; sleep 2 +done +[ "$gc" = "1" ] || fail "worker group $GROUP minCount=$gc, want 1 (N-1)" + +# There must be NO -workers subgroup and NO -submitter pod. +if kubectl get podgroup "${GROUP}-workers" >/dev/null 2>&1; then + fail "found ${GROUP}-workers PodGroup, the obsolete leader/worker split must not exist" +fi +if kubectl get pod "${GROUP}-submitter" >/dev/null 2>&1; then + fail "found ${GROUP}-submitter pod, the obsolete separate-submitter model must not exist" +fi +log " worker group minCount=1, no -workers subgroup, no -submitter pod" + +# Scout PodGroup -scout exists with minCount 1 (schedules alone). +log "checking scout group '$SCOUT_GROUP' minCount == 1" +for i in $(seq 1 30); do + sc="$(kubectl get podgroup "$SCOUT_GROUP" -o jsonpath='{.spec.schedulingPolicy.gang.minCount}' 2>/dev/null || true)" + [ -n "$sc" ] && break; sleep 2 +done +[ "$sc" = "1" ] || fail "scout group $SCOUT_GROUP minCount=$sc, want 1" + +# Scout pod (index 0) is relinked into its own group-of-one and is NOT gated. +pl="$(kubectl get pod "$SCOUT" -o jsonpath='{.metadata.labels.fluence\.flux-framework\.org/group}' 2>/dev/null || true)" +[ "$pl" = "$SCOUT_GROUP" ] || fail "scout group label=$pl, want $SCOUT_GROUP" +pgate="$(kubectl get pod "$SCOUT" -o jsonpath='{.spec.schedulingGates[0].name}' 2>/dev/null || true)" +[ -z "$pgate" ] || fail "scout must NOT be gated (gate=$pgate)" +log " scout in '$SCOUT_GROUP' (minCount 1), not gated" + +# Worker pod (index 1+) stays in and is gated. +g="$(kubectl get pod "$WORKER" -o jsonpath='{.metadata.labels.fluence\.flux-framework\.org/group}' 2>/dev/null || true)" +[ "$g" = "$GROUP" ] || fail "$WORKER group label=$g, want $GROUP" +gate="$(kubectl get pod "$WORKER" -o jsonpath='{.spec.schedulingGates[0].name}' 2>/dev/null || true)" +[ "$gate" = "quantum.braket/ready" ] || fail "$WORKER not gated (gate=$gate)" +# The worker's dependency points at the scout group. +dp="$(kubectl get pod "$WORKER" -o jsonpath='{.metadata.annotations.fluence\.flux-framework\.org/depends-on-scout}' 2>/dev/null || true)" +[ "$dp" = "$SCOUT_GROUP" ] || fail "worker depends-on-scout=$dp, want $SCOUT_GROUP" +log " worker in '$GROUP', gated, depends on '$SCOUT_GROUP'" + +log "PASS 7: worker gang(N-1, gated) + scout(1, member, ungates gang), no submitter/leader/worker" +kubectl delete -f examples/test/e2e/quantum/quantum-gang-pods.yaml --wait=false || true +for g in "$GROUP" "$SCOUT_GROUP"; do + kubectl patch podgroup "$g" --type=merge -p '{"metadata":{"finalizers":null}}' 2>/dev/null || true +done +kubectl wait --for=delete pod -l app="$GROUP" --timeout=60s 2>/dev/null || true diff --git a/test/e2e/quantum/04-gang-env-contract.sh b/test/e2e/quantum/04-gang-env-contract.sh index 157f78b..b58532b 100755 --- a/test/e2e/quantum/04-gang-env-contract.sh +++ b/test/e2e/quantum/04-gang-env-contract.sh @@ -1,23 +1,23 @@ #!/usr/bin/env bash -# Env-contract e2e (producer/consumer): verify the webhook injects, at admission, -# the env the runtime depends on — IN-CLUSTER, on the real pod specs, with no +# Env-contract e2e (scout/worker). Verify the webhook injects, at admission, +# the env the runtime depends on, IN-CLUSTER, on the real pod specs, with no # Braket/AWS and WITHOUT requiring scheduling. Guards the seam that, if broken, # makes a gang schedule then hang or double-submit. # # Spec layer only (these are downward-API valueFrom refs whose VALUES resolve at # placement, but whose PRESENCE is deterministic at admission), so no scheduling, -# no qpu capacity, no logs — it cannot flake on capacity. Contract: -# consumer (role): FLUENCE_COORDINATION_ROLE=consumer, FLUENCE_QUANTUM_JOB_ID, FLUXION_BACKEND -# (NO interceptor/PYTHONPATH — a consumer never submits) -# producer (role): FLUENCE_COORDINATION_ROLE=producer + FLUENCE_GANG_GROUP on the sidecar +# no qpu capacity, no logs, it cannot flake on capacity. Contract. +# worker (role): FLUENCE_COORDINATION_ROLE=worker, FLUENCE_QUANTUM_JOB_ID, FLUXION_BACKEND +# (no interceptor, its submits go into the scout session) +# scout (role): FLUENCE_COORDINATION_ROLE=scout + FLUENCE_GANG_GROUP on the sidecar set -euo pipefail HERE="$(cd "$(dirname "$0")" && pwd)"; . "${HERE%/test/e2e/*}/test/e2e/lib.sh" GROUP=qgang -PRODUCER=${GROUP}-0 # completion index 0 -CONSUMER=${GROUP}-1 # completion index 1 +SCOUT=${GROUP}-0 # completion index 0 +WORKER=${GROUP}-1 # completion index 1 -log "TEST 8: producer/consumer env contract — spec layer" +log "TEST 8: scout/worker env contract, spec layer" kubectl apply -f examples/test/e2e/quantum/quantum-gang-pods.yaml # does container $2 of pod $1 have an env entry named $3 ? (spec-level only) @@ -31,44 +31,44 @@ env_val() { 2>/dev/null || true } -log "checking the webhook wired the consumer role contract" -for i in $(seq 1 15); do has_env "$CONSUMER" app FLUENCE_COORDINATION_ROLE && break; sleep 2; done -# Present: the role (=consumer), the producer's task id, and the backend. +log "checking the webhook wired the worker role contract" +for i in $(seq 1 15); do has_env "$WORKER" app FLUENCE_COORDINATION_ROLE && break; sleep 2; done +# Present: the role (=worker), the scout's task id, and the backend. for v in FLUENCE_COORDINATION_ROLE FLUENCE_QUANTUM_JOB_ID FLUXION_BACKEND; do - has_env "$CONSUMER" app "$v" \ - || { kubectl get pod "$CONSUMER" -o yaml | sed -n '/containers:/,/status:/p'; \ - fail "consumer 'app' container missing env '$v'"; } - log " consumer has env: $v" + has_env "$WORKER" app "$v" \ + || { kubectl get pod "$WORKER" -o yaml | sed -n '/containers:/,/status:/p'; \ + fail "worker 'app' container missing env '$v'"; } + log " worker has env: $v" done -role="$(env_val "$CONSUMER" app FLUENCE_COORDINATION_ROLE)" -[ "$role" = "consumer" ] || fail "consumer role=$role, want consumer" -# Absent: a consumer never submits, so no interceptor staging and no faux flag. +role="$(env_val "$WORKER" app FLUENCE_COORDINATION_ROLE)" +[ "$role" = "worker" ] || fail "worker role=$role, want worker" +# Absent. No interceptor staging, a worker submits into the scout session if at all. for v in PYTHONPATH FLUENCE_FAUX_SUBMIT; do - ! has_env "$CONSUMER" app "$v" || fail "consumer must NOT carry '$v' (it does not submit)" + ! has_env "$WORKER" app "$v" || fail "worker must NOT have '$v' (it is not staged with the interceptor)" done -log " consumer role=consumer, no interceptor/faux" +log " worker role=worker, no interceptor/faux" -# The producer's sidecar must know which consumer group to ungate. -log "checking the producer sidecar has FLUENCE_GANG_GROUP=$GROUP" -for i in $(seq 1 30); do kubectl get pod "$PRODUCER" >/dev/null 2>&1 && break; sleep 2; done -gg="$(kubectl get pod "$PRODUCER" \ +# The scout's sidecar must know which worker group to ungate. +log "checking the scout sidecar has FLUENCE_GANG_GROUP=$GROUP" +for i in $(seq 1 30); do kubectl get pod "$SCOUT" >/dev/null 2>&1 && break; sleep 2; done +gg="$(kubectl get pod "$SCOUT" \ -o jsonpath="{.spec.containers[?(@.name=='fluence-sidecar')].env[?(@.name=='FLUENCE_GANG_GROUP')].value}" \ 2>/dev/null || true)" -[ "$gg" = "$GROUP" ] || fail "producer sidecar FLUENCE_GANG_GROUP=$gg, want $GROUP" -log " producer sidecar has FLUENCE_GANG_GROUP=$gg" +[ "$gg" = "$GROUP" ] || fail "scout sidecar FLUENCE_GANG_GROUP=$gg, want $GROUP" +log " scout sidecar has FLUENCE_GANG_GROUP=$gg" -# The producer carries role=producer and is the real submitter (no consumer id). -prole="$(env_val "$PRODUCER" app FLUENCE_COORDINATION_ROLE)" -[ "$prole" = "producer" ] || fail "producer role=$prole, want producer" -if has_env "$PRODUCER" app FLUENCE_QUANTUM_JOB_ID; then - fail "producer must NOT carry FLUENCE_QUANTUM_JOB_ID (it submits its own task)" +# The scout has role=scout and is the real submitter (no worker id). +prole="$(env_val "$SCOUT" app FLUENCE_COORDINATION_ROLE)" +[ "$prole" = "scout" ] || fail "scout role=$prole, want scout" +if has_env "$SCOUT" app FLUENCE_QUANTUM_JOB_ID; then + fail "scout must NOT have FLUENCE_QUANTUM_JOB_ID (it submits its own task)" fi -log " producer role=producer, submits its own task" +log " scout role=scout, submits its own task" -log "PASS 8: webhook injects the consumer(role) + producer(role) env contract at admission" +log "PASS 8: webhook injects the worker(role) + scout(role) env contract at admission" kubectl delete -f examples/test/e2e/quantum/quantum-gang-pods.yaml --wait=false || true -for g in "$GROUP" "${GROUP}-producer"; do +for g in "$GROUP" "${GROUP}-scout"; do kubectl patch podgroup "$g" --type=merge -p '{"metadata":{"finalizers":null}}' 2>/dev/null || true done kubectl wait --for=delete pod -l app="$GROUP" --timeout=60s 2>/dev/null || true diff --git a/test/e2e/quantum/setup.sh b/test/e2e/quantum/setup.sh index 984c689..fbdd69f 100644 --- a/test/e2e/quantum/setup.sh +++ b/test/e2e/quantum/setup.sh @@ -1,17 +1,17 @@ #!/usr/bin/env bash # Quantum suite setup (run by the e2e-suite workflow before the NN-*.sh tests). # -# Installs the qpu add-on so nodes advertise fluxion.flux-framework.org/qpu — +# Installs the qpu add-on so nodes advertise fluxion.flux-framework.org/qpu # without it every quantum pod stays Pending (fluence matches in its own graph, # but the default NodeResourcesFit plugin rejects each node because the extended # resource is not in allocatable, so the match is rolled back). The base deploy -# (deploy/fluence-test.yaml) does NOT include this; it is quantum-only. +# (deploy/fluence-test.yaml) does NOT include this. It is quantum-only. # -# Also points the webhook-injected sidecar/stage image at the CI-loaded image: +# Also points the webhook-injected sidecar/stage image at the CI-loaded image. # the default sidecar image (ghcr.io/.../fluence-sidecar:latest) is not loaded in -# kind, so the producer's containers could not pull. The fluence-stage init is +# kind, so the scout's containers could not pull. The fluence-stage init is # fail-soft (no python in this image -> it logs and exits 0), which is fine for -# the structural assertions; the producer still schedules and runs. +# the structural assertions. The scout still schedules and runs. set -euo pipefail HERE="$(cd "$(dirname "$0")" && pwd)"; . "${HERE%/test/e2e/*}/test/e2e/lib.sh" IMAGE="${IMAGE:-vanessa/fluence:test}"