update: use fluxion with bugfixes

.github/workflows/e2e-suite.yaml

@@ -26,6 +26,21 @@ jobs:
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v3
 
+      # build the base here so a PR is tested with its own flux-sched
+      - name: Build base into a local OCI layout
+        uses: docker/build-push-action@v6
+        with:
+          context: ./docker
+          file: ./docker/Dockerfile
+          provenance: false
+          outputs: type=oci,tar=false,dest=${{ runner.temp }}/base-oci
+          cache-from: type=gha,scope=base-linux-amd64
+          cache-to: type=gha,mode=max,scope=base-linux-amd64
+
+      - name: Read the base digest from the layout
+        id: layout
+        run: echo "digest=$(jq -r '.manifests[0].digest' ${{ runner.temp }}/base-oci/index.json)" >> $GITHUB_OUTPUT
+
       - name: Build fluence image
         uses: docker/build-push-action@v6
         with:
@@ -34,6 +49,9 @@ jobs:
           push: false
           load: true
           tags: ${{ env.IMAGE }}
+          # the Dockerfile's FROM is redirected to the base built above
+          build-contexts: |
+            ghcr.io/converged-computing/fluence-base:latest=oci-layout://${{ runner.temp }}/base-oci@${{ steps.layout.outputs.digest }}
           cache-from: type=gha
           cache-to: type=gha,mode=max
 

.github/workflows/unit-tests.yaml

@@ -11,24 +11,29 @@ concurrency:
 jobs:
   test:
     runs-on: ubuntu-latest
-    # fluence-base has flux-sched 0.52.0 (installed to /usr) + Go, so the
-    # cgo matcher (pkg/graph) links and `make test` can build every package.
-    container:
-      image: ghcr.io/converged-computing/fluence-base:latest
-    env:
-      FLUX_SCHED_ROOT: /opt/flux-sched
-      LD_LIBRARY_PATH: /usr/lib:/opt/flux-sched/resource:/opt/flux-sched/resource/reapi/bindings:/opt/flux-sched/resource/libjobspec
     steps:
       - name: Checkout
         uses: actions/checkout@v4
 
-      - name: Versions
-        run: |
-          export PATH=$PATH:/usr/local/go/bin
-          go version
-          ls /opt/flux-sched >/dev/null && echo "flux-sched present"
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      # the base has flux-sched and Go. Build it here so a PR is tested with its own
+      # flux-sched. Cached, so it only rebuilds when it changes.
+      - name: Build base
+        uses: docker/build-push-action@v6
+        with:
+          context: ./docker
+          file: ./docker/Dockerfile
+          provenance: false
+          load: true
+          tags: fluence-base:ci
+          cache-from: type=gha,scope=base-linux-amd64
+          cache-to: type=gha,mode=max,scope=base-linux-amd64
 
       - name: Go test
         run: |
-          export PATH=$PATH:/usr/local/go/bin
-          make test
+          docker run --rm -v "$PWD":/src -w /src \
+            -e FLUX_SCHED_ROOT=/opt/flux-sched \
+            -e LD_LIBRARY_PATH=/usr/lib:/opt/flux-sched/resource:/opt/flux-sched/resource/reapi/bindings:/opt/flux-sched/resource/libjobspec \
+            fluence-base:ci bash -c 'export PATH=$PATH:/usr/local/go/bin && go version && make test'

README.md

@@ -39,7 +39,7 @@ Everything in the graph that can be allocated is a node vertex. That includes th
 
 Real nodes have the attribute virtual=false and virtual resources have virtual=true. A full set of attributes might include virtual=true, class=qpu, and region=us-east-1, because RFC 31 only checks whether a key is present. You select a resource by its class, which means any level of the tree can be asked for on its own. Classes are inherited so a nested type is not pruned at its parent on the way down.
 
-A vertex is the unit Fluxion allocates. Anything under a slot is taken whole, so a node modeled as one core vertex of size 64 could hold a single slot of a gang, and two pods of one gang could only land on two nodes. Real nodes are built the way Flux builds them, one vertex per core and per GPU. Memory is chunked, which is how flux-sched's own test graphs and Slurm's memory per CPU work. Each node gets one memory chunk per core, sized as the node's memory split over its cores in MiB, and a pod's request in MiB takes however many whole chunks cover it. The vertex count stays at about twice the cores, and the kubelet's own fit check still admits the pod with the exact bytes.
+A vertex is the unit Fluxion allocates. Anything under a slot is taken whole, so a node modeled as one core vertex of size 64 could hold a single slot of a gang, and two pods of one gang could only land on two nodes. Real nodes are built the way Flux builds them, one vertex per core and per GPU. Memory is chunked, which is how flux-sched's own test graphs and Slurm's memory per CPU work. Each node gets one memory chunk per core, sized as the node's memory split over its cores in MiB, and a pod's request in MiB takes however many whole chunks cover it. The vertex count stays at about twice the cores, and the kubelet's own fit check still admits the pod with the exact bytes. This needs flux-sched 0.53 or newer. Older versions only match a partial chunk in a single slot.
 
 ## Resources
 

docker/Dockerfile

@@ -21,8 +21,8 @@ RUN arch="${TARGETARCH:-$(dpkg --print-architecture)}" \
  && tar -C /usr/local -xzf "go1.26.0.linux-${arch}.tar.gz" && rm "go1.26.0.linux-${arch}.tar.gz"
 ENV PATH=$PATH:/usr/local/go/bin
 
-RUN wget https://github.com/flux-framework/flux-sched/releases/download/v0.52.0/flux-sched-0.52.0.tar.gz && \
-    tar -xzvf flux-sched-0.52.0.tar.gz && mv flux-sched-0.52.0 /opt/flux-sched \
+RUN wget https://github.com/flux-framework/flux-sched/releases/download/v0.56.0/flux-sched-0.56.0.tar.gz && \
+    tar -xzvf flux-sched-0.56.0.tar.gz && mv flux-sched-0.56.0 /opt/flux-sched \
  && cd /opt/flux-sched && export WITH_GO=yes && ./configure --prefix=/usr \
  && mkdir build && cd build && cmake ../ && cd ../ && make -j"$(nproc)" && make install
 ENV FLUX_SCHED_ROOT=/opt/flux-sched

docs/coordination-handler-design.md

@@ -50,7 +50,7 @@ That partitions a quantum gang into two kinds, decided per pod.
 
 Identical pod templates (a Job/Deployment) are genuinely ambiguous between "one
 shared task, fan the result out to N pods" and "N independent tasks." Fluence
-cannot infer this. The user declares it with one annotation on the pod template.
+cannot infer this. The user says which with one annotation on the pod template.
 
 ```yaml
 metadata:

examples/test/e2e/gang/gang-one-node.yaml

@@ -0,0 +1,34 @@
+# The tutorial's gang. Two pods, a core and some memory each, on one node.
+apiVersion: scheduling.k8s.io/v1alpha2
+kind: PodGroup
+metadata:
+  name: onenode
+spec:
+  schedulingPolicy:
+    gang:
+      minCount: 2
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: onenode
+spec:
+  replicas: 2
+  selector:
+    matchLabels: {app: onenode}
+  template:
+    metadata:
+      labels:
+        app: onenode
+    spec:
+      schedulerName: fluence
+      schedulingGroup:
+        podGroupName: onenode
+      containers:
+        - name: worker
+          image: busybox
+          command: ["sleep", "3600"]
+          resources:
+            requests:
+              cpu: "1"
+              memory: 128Mi

examples/test/e2e/gang/multi-gang-contention.yaml

@@ -1,7 +1,5 @@
-# Two gangs that cannot both place. GANG_CPU is filled in by the test with the
-# free cores on a worker, so one pod fills one worker and a gang of two takes
-# both. One gang places entirely. The loser stays FULLY pending (all-or-nothing),
-# never partial.
+# Two gangs that cannot both place. The test fills in GANG_CPU so one pod fills
+# a worker. One gang places entirely, the loser stays FULLY pending, never partial.
 apiVersion: apps/v1
 kind: Deployment
 metadata:

examples/test/e2e/gang/multi-gang-requeue.yaml

@@ -4,8 +4,7 @@
 # gang-wait. A 2-pod gang needing the same nodes. Loses the initial race and sits
 #            Unschedulable. When gang-win completes, gang-wait must be re-attempted
 #            (via the shortened unschedulable-recheck timeout) and place atomically.
-# GANG_CPU is filled in by the test with the free cores on a worker, so one pod
-# fills one worker and the two gangs cannot co-run.
+# The test fills in GANG_CPU so one pod fills a worker and the gangs cannot co-run.
 apiVersion: batch/v1
 kind: Job
 metadata:

go.mod

@@ -3,7 +3,7 @@ module github.com/converged-computing/fluence
 go 1.26.0
 
 require (
-	github.com/flux-framework/flux-sched/resource/reapi/bindings/go v0.0.0-20260526195258-f0e815f1f354
+	github.com/flux-framework/flux-sched/resource/reapi/bindings/go v0.0.0-20261006164720-2ae3f6cdf839
 	google.golang.org/grpc v1.79.3
 	k8s.io/api v0.36.0
 	k8s.io/apimachinery v0.36.0

go.sum

@@ -34,8 +34,8 @@ github.com/emicklei/go-restful/v3 v3.13.0 h1:C4Bl2xDndpU6nJ4bc1jXd+uTmYPVUwkD6bF
 github.com/emicklei/go-restful/v3 v3.13.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc=
 github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg=
 github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U=
-github.com/flux-framework/flux-sched/resource/reapi/bindings/go v0.0.0-20260526195258-f0e815f1f354 h1:9Z0aLiKYuqFV9BkUHIdmB2LUCL5/RRB6YkLVc8f0Ge0=
-github.com/flux-framework/flux-sched/resource/reapi/bindings/go v0.0.0-20260526195258-f0e815f1f354/go.mod h1:yhmzNyn45YhoxEohh1Sl3h3izLMqL7qpcvmYTRpv7eY=
+github.com/flux-framework/flux-sched/resource/reapi/bindings/go v0.0.0-20261006164720-2ae3f6cdf839 h1:esDDhc7wsHfKm5LF4dhlmRAlWLUT4Pq2z0gUQAmMFqw=
+github.com/flux-framework/flux-sched/resource/reapi/bindings/go v0.0.0-20261006164720-2ae3f6cdf839/go.mod h1:yhmzNyn45YhoxEohh1Sl3h3izLMqL7qpcvmYTRpv7eY=
 github.com/fsnotify/fsnotify v1.9.0 h1:2Ml+OJNzbYCTzsxtv8vKSFD9PbJjmhYF14k/jKC7S9k=
 github.com/fsnotify/fsnotify v1.9.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0=
 github.com/fxamacker/cbor/v2 v2.9.0 h1:NpKPmjDBgUfBms6tr6JZkTHtfFGcMKsw3eGcmD/sapM=

pkg/cluster/cluster.go

@@ -85,12 +85,8 @@ func BuildGraph(nodes []corev1.Node, opts Options) ([]byte, error) {
 		})
 		parents[n.Name] = nodeV
 
-		// A vertex is the unit Fluxion allocates. Anything under a slot is taken
-		// whole, so a node built as one sized vertex per type can host one slot
-		// of a gang. Cores and GPUs are one vertex each, as in Flux. Memory is
-		// chunked the way flux-sched's own graphs and Slurm's memory per CPU
-		// do it, one chunk per core sized in MiB, so a request takes whole
-		// chunks and the vertex count stays at twice the cores.
+		// Fluxion gives a slot whole vertices, so a node needs many. One per
+		// core and GPU, and memory as one chunk per core, like Slurm does it.
 		cores := count(n, corev1.ResourceCPU)
 		for i := int64(0); i < cores; i++ {
 			b.AddChild(nodeV, "core", "core", jgf.Options{})
@@ -140,9 +136,7 @@ func count(n *corev1.Node, name corev1.ResourceName) int64 {
 	return 0
 }
 
-// memoryChunkMiB is the size of one memory chunk, the node's memory split
-// evenly over its cores, in whole MiB rounded down. Zero when there is no
-// memory or no cores to split it over.
+// memoryChunkMiB is the node's memory split over its cores, in whole MiB.
 func memoryChunkMiB(n *corev1.Node, cores int64) int64 {
 	if cores <= 0 {
 		return 0

pkg/cluster/cluster_test.go

@@ -41,10 +41,7 @@ func TestIsControlPlane(t *testing.T) {
 	}
 }
 
-// A real node is built with one vertex per core and per GPU, and memory as one
-// chunk per core, each chunk the node's memory split over its cores in whole
-// MiB. Fluxion hands a vertex to a slot and never splits it, so a node built
-// as one sized vertex per type could host one slot of a gang.
+// A node is one vertex per core and GPU, and one memory chunk per core.
 func TestBuildGraphOneVertexPerUnit(t *testing.T) {
 	n := corev1.Node{}
 	n.Name = "worker"

pkg/graph/graph.go

@@ -146,7 +146,9 @@ func (f *FluxionGraph) Satisfy(specFile string) (bool, error) {
 		return false, errors.New("Error reading jobspec")
 	}
 
-	satisfied, overhead, err := f.cli.MatchSatisfy(string(spec))
+	// 0 is satisfiable, 1 is not, -1 is an error
+	rc, overhead, err := f.cli.MatchSatisfy(string(spec))
+	satisfied := err == nil && rc == 0
 	fmt.Printf("  Satisfied : %t\n", satisfied)
 	fmt.Printf("  Overhead : %.6f seconds\n", overhead)
 

pkg/placement/placement.go

@@ -144,8 +144,7 @@ func requireConstraintsFromPods(pods []corev1.Pod) []string {
 // fluxion.flux-framework.org/<type> prefix is a virtual device type instead.
 var computeTypes = map[string]bool{"core": true, "memory": true, "gpu": true}
 
-// mib is the unit memory is requested in. The graph sizes its memory chunks
-// in MiB, so a request in MiB takes however many whole chunks cover it.
+// memory is requested in MiB, the unit the graph sizes its chunks in
 const mib = 1024 * 1024
 
 // podResources distills a pod's container requests into Fluxion resource counts
@@ -160,7 +159,7 @@ func podResources(p *corev1.Pod) map[string]int {
 			case name == corev1.ResourceCPU:
 				counts["core"] += int(q.Value()) // rounds millicores up to whole cores
 			case name == corev1.ResourceMemory:
-				// whole MiB rounded up, the unit the memory chunks are sized in
+				// whole MiB, rounded up
 				counts["memory"] += int((q.Value() + mib - 1) / mib)
 			case name == "nvidia.com/gpu":
 				counts["gpu"] += int(q.Value())

pkg/placement/placement_test.go

@@ -440,8 +440,7 @@ func computeConstraints(t *testing.T, spec *jobspec.Jobspec) map[string]interfac
 	return cons
 }
 
-// Memory requests are counted in whole MiB rounded up, the unit the graph
-// sizes its memory chunks in, so Fluxion takes however many chunks cover it.
+// Memory requests are whole MiB, rounded up.
 func TestMemoryRequestsCountInMiB(t *testing.T) {
 	cases := []struct {
 		req  string

pkg/webhook/handlers/dependency.go

@@ -33,7 +33,7 @@ import (
 //
 // The webhook PRODUCES a Dependency (gates the workers, stamps the
 // annotations). REMOVING the gate is owned by the observer best placed to see
-// the scout's readiness. That split, declare here, observe elsewhere, is
+// the scout's readiness. That split, set here, watched elsewhere, is
 // what keeps the primitive general. A new resource type adds a Kind and an
 // observer and reuses the gating/annotation machinery unchanged.
 type Dependency struct {
@@ -67,7 +67,7 @@ func (d Dependency) applyOps(pod *corev1.Pod) []spec.Op {
 	return ops
 }
 
-// DependencyOf reads a dependent pod's declared Dependency, or ok=false if it
+// DependencyOf reads the Dependency set on a pod, or ok=false if it
 // has none. The scheduler's reconcile loop and the sidecar use this to learn
 // what a gated pod is waiting for without hardcoding a kind.
 func DependencyOf(pod *corev1.Pod) (Dependency, bool) {

test/e2e/gang/03-multi-gang.sh

@@ -37,7 +37,7 @@ kubectl wait --for=delete pod -l app=gang3 --timeout=60s 2>/dev/null || true
 
 # ---- B) contention: the gang that can't fully fit stays ENTIRELY pending --------
 log "TEST 6B: contention, a gang that cannot fully fit must NOT partially place"
-log "each pod asks for $(free_cores_per_worker) cores, the free cores on a worker, so a gang of two takes both workers"
+log "cpu per pod is $(free_cores_per_worker), the free cores on a worker"
 render_gang_manifest examples/test/e2e/gang/multi-gang-contention.yaml | kubectl apply -f -
 
 # wait until the cluster settles. Three possible outcomes.

test/e2e/gang/04-requeue-on-capacity.sh

@@ -10,9 +10,8 @@
 #      re-attempted on its own (no manual nudge) and then ALSO places atomically
 #      (2/2), driven by the shortened --pod-max-in-unschedulable-pods-duration.
 #
-# SCOPE / LIMITATION. This is a kind cluster with two workers, and each pod's cpu
-# request is raised to fill a worker, so one gang fits at a time. It verifies
-# the INVARIANTS on a minimal contention case. It does NOT reproduce the
+# SCOPE / LIMITATION. Two kind workers, and each pod asks for a whole worker, so
+# one gang fits at a time. It verifies the INVARIANTS on a minimal case. It does NOT reproduce the
 # GKE-scale dynamics where the bug was first seen, one-pod-per-node (~80-core)
 # saturation and ~20 simultaneous mixed-size gangs draining in sequence. That
 # scale behavior is validated on the real cluster, not in CI. A pass here means
@@ -26,7 +25,7 @@ running() { kubectl get pods -l job-name="$1" --field-selector=status.phase=Runn
 on_nodes() { kubectl get pods -l job-name="$1" -o jsonpath='{range .items[*]}{.spec.nodeName}{"\n"}{end}' 2>/dev/null | grep -c . || true; }
 
 log "TEST 9: contended gangs stay all-or-nothing, loser requeues when capacity frees"
-log "each pod asks for $(free_cores_per_worker) cores, the free cores on a worker, so one gang fits at a time"
+log "cpu per pod is $(free_cores_per_worker), the free cores on a worker"
 render_gang_manifest examples/test/e2e/gang/multi-gang-requeue.yaml | kubectl apply -f -
 
 # ---- 1. one gang wins CLEANLY (2/2); the other places NOTHING (0/2) ------------

test/e2e/gang/05-gang-on-one-node.sh

@@ -0,0 +1,43 @@
+#!/usr/bin/env bash
+# The tutorial's gang, two pods with cpu and memory, on one node. The other
+# gang tests let each slot take its own worker, so they never hit this.
+set -euo pipefail
+HERE="$(cd "$(dirname "$0")" && pwd)"; . "${HERE%/test/e2e/*}/test/e2e/lib.sh"
+
+NAME=onenode
+FIXTURE=examples/test/e2e/gang/gang-one-node.yaml
+
+log "TEST 10: a two pod gang with memory places on one node"
+
+workers=( $(kubectl get nodes -l '!node-role.kubernetes.io/control-plane' -o jsonpath='{.items[*].metadata.name}') )
+[ "${#workers[@]}" -ge 1 ] || fail "no worker node found"
+keep="${workers[0]}"
+others=( "${workers[@]:1}" )
+
+cleanup() {
+  kubectl delete -f "$FIXTURE" --ignore-not-found --wait=false >/dev/null 2>&1 || true
+  kubectl patch podgroup "$NAME" --type=merge -p '{"metadata":{"finalizers":null}}' >/dev/null 2>&1 || true
+  kubectl wait --for=delete pod -l "app=$NAME" --timeout=60s >/dev/null 2>&1 || true
+  for n in "${others[@]}"; do kubectl uncordon "$n" >/dev/null 2>&1 || true; done
+  # give the graph its nodes back
+  kubectl rollout restart -n "${NS_KUBE}" deployment/fluence >/dev/null 2>&1 || true
+  kubectl rollout status -n "${NS_KUBE}" deployment/fluence --timeout="${TIMEOUT}" >/dev/null 2>&1 || true
+}
+trap cleanup EXIT
+
+# cordon the rest and restart fluence so its graph is that one node
+for n in "${others[@]}"; do
+  log "cordoning $n"
+  kubectl cordon "$n"
+done
+kubectl rollout restart -n "${NS_KUBE}" deployment/fluence
+wait_fluence_ready
+
+kubectl apply -f "$FIXTURE"
+log "waiting for both pods on $keep"
+wait_pods_ready "app=$NAME" 2 180 \
+  || fail "the gang did not place on one node"
+
+placed="$(kubectl get pods -l "app=$NAME" -o jsonpath='{range .items[*]}{.spec.nodeName}{"\n"}{end}' | sort -u | tr '\n' ' ')"
+[ "$placed" = "$keep " ] || fail "pods landed on '$placed', want only $keep"
+log "PASS 10: both pods of the gang run on $keep"

test/e2e/lib.sh

@@ -89,10 +89,8 @@ assert_stays_pending() {
 # cpu quantities to millicores, so "4" and "250m" add up
 millicores() { awk '{ if ($0 ~ /m$/) { sub(/m$/, ""); t += $0 } else if ($0 != "") { t += $0 * 1000 } } END { print t + 0 }'; }
 
-# Whole cores free on the fullest worker. Allocatable less what the pods already
-# on it request, which is the arithmetic the kube-scheduler's fit check does. A
-# pod that asks for this fills a worker, so a two pod gang takes two workers and
-# two gangs contend, on any runner, with nothing assumed about the system pods.
+# Whole cores free on the fullest worker, allocatable less what its pods
+# request. A pod that asks for this fills a worker.
 free_cores_per_worker() {
   local min="" node alloc used free
   for node in $(kubectl get nodes -l '!node-role.kubernetes.io/control-plane' -o jsonpath='{.items[*].metadata.name}'); do
@@ -106,8 +104,7 @@ free_cores_per_worker() {
   echo "$min"
 }
 
-# Render a manifest whose cpu request is ${GANG_CPU} with the free cores on a
-# worker. Pipe it to kubectl apply or delete.
+# Fill in ${GANG_CPU} with the free cores on a worker. Pipe to kubectl.
 render_gang_manifest() {
   local cpu line
   cpu="$(free_cores_per_worker)"