From ef5087aa07c7d7ac4ede280f1ace2f15f3228a36 Mon Sep 17 00:00:00 2001 From: vsoch Date: Sun, 18 Oct 2026 18:02:52 -0700 Subject: [PATCH] update: use fluxion with bugfixes Signed-off-by: vsoch --- .github/workflows/e2e-suite.yaml | 18 ++++++++ .github/workflows/unit-tests.yaml | 33 ++++++++------ README.md | 2 +- docker/Dockerfile | 4 +- docs/coordination-handler-design.md | 2 +- examples/test/e2e/gang/gang-one-node.yaml | 34 +++++++++++++++ .../test/e2e/gang/multi-gang-contention.yaml | 6 +-- .../test/e2e/gang/multi-gang-requeue.yaml | 3 +- go.mod | 2 +- go.sum | 4 +- pkg/cluster/cluster.go | 12 ++---- pkg/cluster/cluster_test.go | 5 +-- pkg/graph/graph.go | 4 +- pkg/placement/placement.go | 5 +-- pkg/placement/placement_test.go | 3 +- pkg/webhook/handlers/dependency.go | 4 +- test/e2e/gang/03-multi-gang.sh | 2 +- test/e2e/gang/04-requeue-on-capacity.sh | 7 ++- test/e2e/gang/05-gang-on-one-node.sh | 43 +++++++++++++++++++ test/e2e/lib.sh | 9 ++-- 20 files changed, 143 insertions(+), 59 deletions(-) create mode 100644 examples/test/e2e/gang/gang-one-node.yaml create mode 100755 test/e2e/gang/05-gang-on-one-node.sh diff --git a/.github/workflows/e2e-suite.yaml b/.github/workflows/e2e-suite.yaml index 2c2b0c7..8f36c93 100644 --- a/.github/workflows/e2e-suite.yaml +++ b/.github/workflows/e2e-suite.yaml @@ -26,6 +26,21 @@ jobs: - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 + # build the base here so a PR is tested with its own flux-sched + - name: Build base into a local OCI layout + uses: docker/build-push-action@v6 + with: + context: ./docker + file: ./docker/Dockerfile + provenance: false + outputs: type=oci,tar=false,dest=${{ runner.temp }}/base-oci + cache-from: type=gha,scope=base-linux-amd64 + cache-to: type=gha,mode=max,scope=base-linux-amd64 + + - name: Read the base digest from the layout + id: layout + run: echo "digest=$(jq -r '.manifests[0].digest' ${{ runner.temp }}/base-oci/index.json)" >> $GITHUB_OUTPUT + - name: Build fluence image uses: docker/build-push-action@v6 with: @@ -34,6 +49,9 @@ jobs: push: false load: true tags: ${{ env.IMAGE }} + # the Dockerfile's FROM is redirected to the base built above + build-contexts: | + ghcr.io/converged-computing/fluence-base:latest=oci-layout://${{ runner.temp }}/base-oci@${{ steps.layout.outputs.digest }} cache-from: type=gha cache-to: type=gha,mode=max diff --git a/.github/workflows/unit-tests.yaml b/.github/workflows/unit-tests.yaml index 11a7758..f47dc37 100644 --- a/.github/workflows/unit-tests.yaml +++ b/.github/workflows/unit-tests.yaml @@ -11,24 +11,29 @@ concurrency: jobs: test: runs-on: ubuntu-latest - # fluence-base has flux-sched 0.52.0 (installed to /usr) + Go, so the - # cgo matcher (pkg/graph) links and `make test` can build every package. - container: - image: ghcr.io/converged-computing/fluence-base:latest - env: - FLUX_SCHED_ROOT: /opt/flux-sched - LD_LIBRARY_PATH: /usr/lib:/opt/flux-sched/resource:/opt/flux-sched/resource/reapi/bindings:/opt/flux-sched/resource/libjobspec steps: - name: Checkout uses: actions/checkout@v4 - - name: Versions - run: | - export PATH=$PATH:/usr/local/go/bin - go version - ls /opt/flux-sched >/dev/null && echo "flux-sched present" + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + # the base has flux-sched and Go. Build it here so a PR is tested with its own + # flux-sched. Cached, so it only rebuilds when it changes. + - name: Build base + uses: docker/build-push-action@v6 + with: + context: ./docker + file: ./docker/Dockerfile + provenance: false + load: true + tags: fluence-base:ci + cache-from: type=gha,scope=base-linux-amd64 + cache-to: type=gha,mode=max,scope=base-linux-amd64 - name: Go test run: | - export PATH=$PATH:/usr/local/go/bin - make test + docker run --rm -v "$PWD":/src -w /src \ + -e FLUX_SCHED_ROOT=/opt/flux-sched \ + -e LD_LIBRARY_PATH=/usr/lib:/opt/flux-sched/resource:/opt/flux-sched/resource/reapi/bindings:/opt/flux-sched/resource/libjobspec \ + fluence-base:ci bash -c 'export PATH=$PATH:/usr/local/go/bin && go version && make test' diff --git a/README.md b/README.md index 1a4427d..6bbeaf8 100644 --- a/README.md +++ b/README.md @@ -39,7 +39,7 @@ Everything in the graph that can be allocated is a node vertex. That includes th Real nodes have the attribute virtual=false and virtual resources have virtual=true. A full set of attributes might include virtual=true, class=qpu, and region=us-east-1, because RFC 31 only checks whether a key is present. You select a resource by its class, which means any level of the tree can be asked for on its own. Classes are inherited so a nested type is not pruned at its parent on the way down. -A vertex is the unit Fluxion allocates. Anything under a slot is taken whole, so a node modeled as one core vertex of size 64 could hold a single slot of a gang, and two pods of one gang could only land on two nodes. Real nodes are built the way Flux builds them, one vertex per core and per GPU. Memory is chunked, which is how flux-sched's own test graphs and Slurm's memory per CPU work. Each node gets one memory chunk per core, sized as the node's memory split over its cores in MiB, and a pod's request in MiB takes however many whole chunks cover it. The vertex count stays at about twice the cores, and the kubelet's own fit check still admits the pod with the exact bytes. +A vertex is the unit Fluxion allocates. Anything under a slot is taken whole, so a node modeled as one core vertex of size 64 could hold a single slot of a gang, and two pods of one gang could only land on two nodes. Real nodes are built the way Flux builds them, one vertex per core and per GPU. Memory is chunked, which is how flux-sched's own test graphs and Slurm's memory per CPU work. Each node gets one memory chunk per core, sized as the node's memory split over its cores in MiB, and a pod's request in MiB takes however many whole chunks cover it. The vertex count stays at about twice the cores, and the kubelet's own fit check still admits the pod with the exact bytes. This needs flux-sched 0.53 or newer. Older versions only match a partial chunk in a single slot. ## Resources diff --git a/docker/Dockerfile b/docker/Dockerfile index c28bee1..927304c 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -21,8 +21,8 @@ RUN arch="${TARGETARCH:-$(dpkg --print-architecture)}" \ && tar -C /usr/local -xzf "go1.26.0.linux-${arch}.tar.gz" && rm "go1.26.0.linux-${arch}.tar.gz" ENV PATH=$PATH:/usr/local/go/bin -RUN wget https://github.com/flux-framework/flux-sched/releases/download/v0.52.0/flux-sched-0.52.0.tar.gz && \ - tar -xzvf flux-sched-0.52.0.tar.gz && mv flux-sched-0.52.0 /opt/flux-sched \ +RUN wget https://github.com/flux-framework/flux-sched/releases/download/v0.56.0/flux-sched-0.56.0.tar.gz && \ + tar -xzvf flux-sched-0.56.0.tar.gz && mv flux-sched-0.56.0 /opt/flux-sched \ && cd /opt/flux-sched && export WITH_GO=yes && ./configure --prefix=/usr \ && mkdir build && cd build && cmake ../ && cd ../ && make -j"$(nproc)" && make install ENV FLUX_SCHED_ROOT=/opt/flux-sched diff --git a/docs/coordination-handler-design.md b/docs/coordination-handler-design.md index 21da503..7e697c9 100644 --- a/docs/coordination-handler-design.md +++ b/docs/coordination-handler-design.md @@ -50,7 +50,7 @@ That partitions a quantum gang into two kinds, decided per pod. Identical pod templates (a Job/Deployment) are genuinely ambiguous between "one shared task, fan the result out to N pods" and "N independent tasks." Fluence -cannot infer this. The user declares it with one annotation on the pod template. +cannot infer this. The user says which with one annotation on the pod template. ```yaml metadata: diff --git a/examples/test/e2e/gang/gang-one-node.yaml b/examples/test/e2e/gang/gang-one-node.yaml new file mode 100644 index 0000000..0812c21 --- /dev/null +++ b/examples/test/e2e/gang/gang-one-node.yaml @@ -0,0 +1,34 @@ +# The tutorial's gang. Two pods, a core and some memory each, on one node. +apiVersion: scheduling.k8s.io/v1alpha2 +kind: PodGroup +metadata: + name: onenode +spec: + schedulingPolicy: + gang: + minCount: 2 +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: onenode +spec: + replicas: 2 + selector: + matchLabels: {app: onenode} + template: + metadata: + labels: + app: onenode + spec: + schedulerName: fluence + schedulingGroup: + podGroupName: onenode + containers: + - name: worker + image: busybox + command: ["sleep", "3600"] + resources: + requests: + cpu: "1" + memory: 128Mi diff --git a/examples/test/e2e/gang/multi-gang-contention.yaml b/examples/test/e2e/gang/multi-gang-contention.yaml index ec8bd6a..ae1d8ec 100644 --- a/examples/test/e2e/gang/multi-gang-contention.yaml +++ b/examples/test/e2e/gang/multi-gang-contention.yaml @@ -1,7 +1,5 @@ -# Two gangs that cannot both place. GANG_CPU is filled in by the test with the -# free cores on a worker, so one pod fills one worker and a gang of two takes -# both. One gang places entirely. The loser stays FULLY pending (all-or-nothing), -# never partial. +# Two gangs that cannot both place. The test fills in GANG_CPU so one pod fills +# a worker. One gang places entirely, the loser stays FULLY pending, never partial. apiVersion: apps/v1 kind: Deployment metadata: diff --git a/examples/test/e2e/gang/multi-gang-requeue.yaml b/examples/test/e2e/gang/multi-gang-requeue.yaml index 4bf3026..beff727 100644 --- a/examples/test/e2e/gang/multi-gang-requeue.yaml +++ b/examples/test/e2e/gang/multi-gang-requeue.yaml @@ -4,8 +4,7 @@ # gang-wait. A 2-pod gang needing the same nodes. Loses the initial race and sits # Unschedulable. When gang-win completes, gang-wait must be re-attempted # (via the shortened unschedulable-recheck timeout) and place atomically. -# GANG_CPU is filled in by the test with the free cores on a worker, so one pod -# fills one worker and the two gangs cannot co-run. +# The test fills in GANG_CPU so one pod fills a worker and the gangs cannot co-run. apiVersion: batch/v1 kind: Job metadata: diff --git a/go.mod b/go.mod index 6b072c0..2940179 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/converged-computing/fluence go 1.26.0 require ( - github.com/flux-framework/flux-sched/resource/reapi/bindings/go v0.0.0-20260526195258-f0e815f1f354 + github.com/flux-framework/flux-sched/resource/reapi/bindings/go v0.0.0-20261006164720-2ae3f6cdf839 google.golang.org/grpc v1.79.3 k8s.io/api v0.36.0 k8s.io/apimachinery v0.36.0 diff --git a/go.sum b/go.sum index a67f060..0c83cbb 100644 --- a/go.sum +++ b/go.sum @@ -34,8 +34,8 @@ github.com/emicklei/go-restful/v3 v3.13.0 h1:C4Bl2xDndpU6nJ4bc1jXd+uTmYPVUwkD6bF github.com/emicklei/go-restful/v3 v3.13.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= -github.com/flux-framework/flux-sched/resource/reapi/bindings/go v0.0.0-20260526195258-f0e815f1f354 h1:9Z0aLiKYuqFV9BkUHIdmB2LUCL5/RRB6YkLVc8f0Ge0= -github.com/flux-framework/flux-sched/resource/reapi/bindings/go v0.0.0-20260526195258-f0e815f1f354/go.mod h1:yhmzNyn45YhoxEohh1Sl3h3izLMqL7qpcvmYTRpv7eY= +github.com/flux-framework/flux-sched/resource/reapi/bindings/go v0.0.0-20261006164720-2ae3f6cdf839 h1:esDDhc7wsHfKm5LF4dhlmRAlWLUT4Pq2z0gUQAmMFqw= +github.com/flux-framework/flux-sched/resource/reapi/bindings/go v0.0.0-20261006164720-2ae3f6cdf839/go.mod h1:yhmzNyn45YhoxEohh1Sl3h3izLMqL7qpcvmYTRpv7eY= github.com/fsnotify/fsnotify v1.9.0 h1:2Ml+OJNzbYCTzsxtv8vKSFD9PbJjmhYF14k/jKC7S9k= github.com/fsnotify/fsnotify v1.9.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0= github.com/fxamacker/cbor/v2 v2.9.0 h1:NpKPmjDBgUfBms6tr6JZkTHtfFGcMKsw3eGcmD/sapM= diff --git a/pkg/cluster/cluster.go b/pkg/cluster/cluster.go index b4b2637..4cdad4d 100644 --- a/pkg/cluster/cluster.go +++ b/pkg/cluster/cluster.go @@ -85,12 +85,8 @@ func BuildGraph(nodes []corev1.Node, opts Options) ([]byte, error) { }) parents[n.Name] = nodeV - // A vertex is the unit Fluxion allocates. Anything under a slot is taken - // whole, so a node built as one sized vertex per type can host one slot - // of a gang. Cores and GPUs are one vertex each, as in Flux. Memory is - // chunked the way flux-sched's own graphs and Slurm's memory per CPU - // do it, one chunk per core sized in MiB, so a request takes whole - // chunks and the vertex count stays at twice the cores. + // Fluxion gives a slot whole vertices, so a node needs many. One per + // core and GPU, and memory as one chunk per core, like Slurm does it. cores := count(n, corev1.ResourceCPU) for i := int64(0); i < cores; i++ { b.AddChild(nodeV, "core", "core", jgf.Options{}) @@ -140,9 +136,7 @@ func count(n *corev1.Node, name corev1.ResourceName) int64 { return 0 } -// memoryChunkMiB is the size of one memory chunk, the node's memory split -// evenly over its cores, in whole MiB rounded down. Zero when there is no -// memory or no cores to split it over. +// memoryChunkMiB is the node's memory split over its cores, in whole MiB. func memoryChunkMiB(n *corev1.Node, cores int64) int64 { if cores <= 0 { return 0 diff --git a/pkg/cluster/cluster_test.go b/pkg/cluster/cluster_test.go index 7f56ef1..2b7e071 100644 --- a/pkg/cluster/cluster_test.go +++ b/pkg/cluster/cluster_test.go @@ -41,10 +41,7 @@ func TestIsControlPlane(t *testing.T) { } } -// A real node is built with one vertex per core and per GPU, and memory as one -// chunk per core, each chunk the node's memory split over its cores in whole -// MiB. Fluxion hands a vertex to a slot and never splits it, so a node built -// as one sized vertex per type could host one slot of a gang. +// A node is one vertex per core and GPU, and one memory chunk per core. func TestBuildGraphOneVertexPerUnit(t *testing.T) { n := corev1.Node{} n.Name = "worker" diff --git a/pkg/graph/graph.go b/pkg/graph/graph.go index 7c0b6db..1d4eb6a 100644 --- a/pkg/graph/graph.go +++ b/pkg/graph/graph.go @@ -146,7 +146,9 @@ func (f *FluxionGraph) Satisfy(specFile string) (bool, error) { return false, errors.New("Error reading jobspec") } - satisfied, overhead, err := f.cli.MatchSatisfy(string(spec)) + // 0 is satisfiable, 1 is not, -1 is an error + rc, overhead, err := f.cli.MatchSatisfy(string(spec)) + satisfied := err == nil && rc == 0 fmt.Printf(" Satisfied : %t\n", satisfied) fmt.Printf(" Overhead : %.6f seconds\n", overhead) diff --git a/pkg/placement/placement.go b/pkg/placement/placement.go index 7538ec9..86691e0 100644 --- a/pkg/placement/placement.go +++ b/pkg/placement/placement.go @@ -144,8 +144,7 @@ func requireConstraintsFromPods(pods []corev1.Pod) []string { // fluxion.flux-framework.org/ prefix is a virtual device type instead. var computeTypes = map[string]bool{"core": true, "memory": true, "gpu": true} -// mib is the unit memory is requested in. The graph sizes its memory chunks -// in MiB, so a request in MiB takes however many whole chunks cover it. +// memory is requested in MiB, the unit the graph sizes its chunks in const mib = 1024 * 1024 // podResources distills a pod's container requests into Fluxion resource counts @@ -160,7 +159,7 @@ func podResources(p *corev1.Pod) map[string]int { case name == corev1.ResourceCPU: counts["core"] += int(q.Value()) // rounds millicores up to whole cores case name == corev1.ResourceMemory: - // whole MiB rounded up, the unit the memory chunks are sized in + // whole MiB, rounded up counts["memory"] += int((q.Value() + mib - 1) / mib) case name == "nvidia.com/gpu": counts["gpu"] += int(q.Value()) diff --git a/pkg/placement/placement_test.go b/pkg/placement/placement_test.go index 278f872..07030c0 100644 --- a/pkg/placement/placement_test.go +++ b/pkg/placement/placement_test.go @@ -440,8 +440,7 @@ func computeConstraints(t *testing.T, spec *jobspec.Jobspec) map[string]interfac return cons } -// Memory requests are counted in whole MiB rounded up, the unit the graph -// sizes its memory chunks in, so Fluxion takes however many chunks cover it. +// Memory requests are whole MiB, rounded up. func TestMemoryRequestsCountInMiB(t *testing.T) { cases := []struct { req string diff --git a/pkg/webhook/handlers/dependency.go b/pkg/webhook/handlers/dependency.go index a0a8a4b..32d0951 100644 --- a/pkg/webhook/handlers/dependency.go +++ b/pkg/webhook/handlers/dependency.go @@ -33,7 +33,7 @@ import ( // // The webhook PRODUCES a Dependency (gates the workers, stamps the // annotations). REMOVING the gate is owned by the observer best placed to see -// the scout's readiness. That split, declare here, observe elsewhere, is +// the scout's readiness. That split, set here, watched elsewhere, is // what keeps the primitive general. A new resource type adds a Kind and an // observer and reuses the gating/annotation machinery unchanged. type Dependency struct { @@ -67,7 +67,7 @@ func (d Dependency) applyOps(pod *corev1.Pod) []spec.Op { return ops } -// DependencyOf reads a dependent pod's declared Dependency, or ok=false if it +// DependencyOf reads the Dependency set on a pod, or ok=false if it // has none. The scheduler's reconcile loop and the sidecar use this to learn // what a gated pod is waiting for without hardcoding a kind. func DependencyOf(pod *corev1.Pod) (Dependency, bool) { diff --git a/test/e2e/gang/03-multi-gang.sh b/test/e2e/gang/03-multi-gang.sh index 94f56bc..8dd4485 100755 --- a/test/e2e/gang/03-multi-gang.sh +++ b/test/e2e/gang/03-multi-gang.sh @@ -37,7 +37,7 @@ kubectl wait --for=delete pod -l app=gang3 --timeout=60s 2>/dev/null || true # ---- B) contention: the gang that can't fully fit stays ENTIRELY pending -------- log "TEST 6B: contention, a gang that cannot fully fit must NOT partially place" -log "each pod asks for $(free_cores_per_worker) cores, the free cores on a worker, so a gang of two takes both workers" +log "cpu per pod is $(free_cores_per_worker), the free cores on a worker" render_gang_manifest examples/test/e2e/gang/multi-gang-contention.yaml | kubectl apply -f - # wait until the cluster settles. Three possible outcomes. diff --git a/test/e2e/gang/04-requeue-on-capacity.sh b/test/e2e/gang/04-requeue-on-capacity.sh index 511ac41..a2b3c29 100755 --- a/test/e2e/gang/04-requeue-on-capacity.sh +++ b/test/e2e/gang/04-requeue-on-capacity.sh @@ -10,9 +10,8 @@ # re-attempted on its own (no manual nudge) and then ALSO places atomically # (2/2), driven by the shortened --pod-max-in-unschedulable-pods-duration. # -# SCOPE / LIMITATION. This is a kind cluster with two workers, and each pod's cpu -# request is raised to fill a worker, so one gang fits at a time. It verifies -# the INVARIANTS on a minimal contention case. It does NOT reproduce the +# SCOPE / LIMITATION. Two kind workers, and each pod asks for a whole worker, so +# one gang fits at a time. It verifies the INVARIANTS on a minimal case. It does NOT reproduce the # GKE-scale dynamics where the bug was first seen, one-pod-per-node (~80-core) # saturation and ~20 simultaneous mixed-size gangs draining in sequence. That # scale behavior is validated on the real cluster, not in CI. A pass here means @@ -26,7 +25,7 @@ running() { kubectl get pods -l job-name="$1" --field-selector=status.phase=Runn on_nodes() { kubectl get pods -l job-name="$1" -o jsonpath='{range .items[*]}{.spec.nodeName}{"\n"}{end}' 2>/dev/null | grep -c . || true; } log "TEST 9: contended gangs stay all-or-nothing, loser requeues when capacity frees" -log "each pod asks for $(free_cores_per_worker) cores, the free cores on a worker, so one gang fits at a time" +log "cpu per pod is $(free_cores_per_worker), the free cores on a worker" render_gang_manifest examples/test/e2e/gang/multi-gang-requeue.yaml | kubectl apply -f - # ---- 1. one gang wins CLEANLY (2/2); the other places NOTHING (0/2) ------------ diff --git a/test/e2e/gang/05-gang-on-one-node.sh b/test/e2e/gang/05-gang-on-one-node.sh new file mode 100755 index 0000000..becbe1d --- /dev/null +++ b/test/e2e/gang/05-gang-on-one-node.sh @@ -0,0 +1,43 @@ +#!/usr/bin/env bash +# The tutorial's gang, two pods with cpu and memory, on one node. The other +# gang tests let each slot take its own worker, so they never hit this. +set -euo pipefail +HERE="$(cd "$(dirname "$0")" && pwd)"; . "${HERE%/test/e2e/*}/test/e2e/lib.sh" + +NAME=onenode +FIXTURE=examples/test/e2e/gang/gang-one-node.yaml + +log "TEST 10: a two pod gang with memory places on one node" + +workers=( $(kubectl get nodes -l '!node-role.kubernetes.io/control-plane' -o jsonpath='{.items[*].metadata.name}') ) +[ "${#workers[@]}" -ge 1 ] || fail "no worker node found" +keep="${workers[0]}" +others=( "${workers[@]:1}" ) + +cleanup() { + kubectl delete -f "$FIXTURE" --ignore-not-found --wait=false >/dev/null 2>&1 || true + kubectl patch podgroup "$NAME" --type=merge -p '{"metadata":{"finalizers":null}}' >/dev/null 2>&1 || true + kubectl wait --for=delete pod -l "app=$NAME" --timeout=60s >/dev/null 2>&1 || true + for n in "${others[@]}"; do kubectl uncordon "$n" >/dev/null 2>&1 || true; done + # give the graph its nodes back + kubectl rollout restart -n "${NS_KUBE}" deployment/fluence >/dev/null 2>&1 || true + kubectl rollout status -n "${NS_KUBE}" deployment/fluence --timeout="${TIMEOUT}" >/dev/null 2>&1 || true +} +trap cleanup EXIT + +# cordon the rest and restart fluence so its graph is that one node +for n in "${others[@]}"; do + log "cordoning $n" + kubectl cordon "$n" +done +kubectl rollout restart -n "${NS_KUBE}" deployment/fluence +wait_fluence_ready + +kubectl apply -f "$FIXTURE" +log "waiting for both pods on $keep" +wait_pods_ready "app=$NAME" 2 180 \ + || fail "the gang did not place on one node" + +placed="$(kubectl get pods -l "app=$NAME" -o jsonpath='{range .items[*]}{.spec.nodeName}{"\n"}{end}' | sort -u | tr '\n' ' ')" +[ "$placed" = "$keep " ] || fail "pods landed on '$placed', want only $keep" +log "PASS 10: both pods of the gang run on $keep" diff --git a/test/e2e/lib.sh b/test/e2e/lib.sh index fb26a9e..0f5c3f9 100644 --- a/test/e2e/lib.sh +++ b/test/e2e/lib.sh @@ -89,10 +89,8 @@ assert_stays_pending() { # cpu quantities to millicores, so "4" and "250m" add up millicores() { awk '{ if ($0 ~ /m$/) { sub(/m$/, ""); t += $0 } else if ($0 != "") { t += $0 * 1000 } } END { print t + 0 }'; } -# Whole cores free on the fullest worker. Allocatable less what the pods already -# on it request, which is the arithmetic the kube-scheduler's fit check does. A -# pod that asks for this fills a worker, so a two pod gang takes two workers and -# two gangs contend, on any runner, with nothing assumed about the system pods. +# Whole cores free on the fullest worker, allocatable less what its pods +# request. A pod that asks for this fills a worker. free_cores_per_worker() { local min="" node alloc used free for node in $(kubectl get nodes -l '!node-role.kubernetes.io/control-plane' -o jsonpath='{.items[*].metadata.name}'); do @@ -106,8 +104,7 @@ free_cores_per_worker() { echo "$min" } -# Render a manifest whose cpu request is ${GANG_CPU} with the free cores on a -# worker. Pipe it to kubectl apply or delete. +# Fill in ${GANG_CPU} with the free cores on a worker. Pipe to kubectl. render_gang_manifest() { local cpu line cpu="$(free_cores_per_worker)"