1818 FLUXION_BACKEND / FLUXION_VENDOR scheduler-chosen backend / vendor
1919 FLUENCE_TASK_DISCOVERY_TIMEOUT seconds to wait for discovery (default 300)
2020 FLUENCE_POLL_INTERVAL seconds between polls (default 30)
21+ FLUENCE_UNGATE_POSITION ungate at this queue position or closer
22+ (default 1, next in line)
2123"""
2224
2325from __future__ import annotations
2729import time
2830
2931from fluence .providers import resolve_from_env
30- from fluence .providers .base import log
32+ from fluence .providers .base import log , ungate_position
3133from fluence .ungate import ungate_pods , gated_pods_from_env , namespace_from_env , wait_for_gated_pods
3234
3335
3436
35- def _poll (provider , task , poll_interval , ungate ):
37+ def _poll (provider , task , poll_interval , ungate , position = 1 ):
38+ """Poll until the task is ready or has failed. True when ready."""
3639 mode = "gang" if ungate else "observe-only"
37- log (f"{ mode } mode: polling queue position" )
40+ log (f"{ mode } mode: polling queue position, ungating at { position } or closer " )
3841 last = object ()
3942 while True :
4043 try :
41- if provider .is_ready_to_ungate (task ):
44+ if provider .task_failed (task ):
45+ log ("ERROR: task reached a terminal state with no result" )
46+ return False
47+ if provider .is_ready_to_ungate (task , position ):
4248 log (f"task ready (position={ provider .queue_position (task )} )" )
43- return
49+ return True
4450 pos = provider .queue_position (task )
4551 if pos != last :
4652 log (f"queue position: { pos } " )
@@ -64,6 +70,7 @@ def main():
6470 discovery_timeout = int (os .environ .get ("FLUENCE_TASK_DISCOVERY_TIMEOUT" , 300 ))
6571 poll_interval = int (os .environ .get ("FLUENCE_POLL_INTERVAL" , 30 ))
6672 ungate_timeout = int (os .environ .get ("FLUENCE_UNGATE_TIMEOUT" , 120 ))
73+ ungate_at = ungate_position ()
6774
6875 namespace = namespace_from_env ()
6976
@@ -90,11 +97,22 @@ def main():
9097 job_id = provider .job_id (task )
9198 log (f"discovered task, job_id={ job_id } " )
9299
93- _poll (provider , task , poll_interval , ungate = not observe )
100+ ready = _poll (provider , task , poll_interval , ungate = not observe ,
101+ position = ungate_at )
94102
95103 if observe :
96104 log ("observe-only run complete" )
97- return
105+ sys .exit (0 if ready else 1 )
106+
107+ if not ready :
108+ # fail open like a discovery failure does, so the gang is not stranded.
109+ # The pods find no result and exit, and we exit non zero so it shows
110+ log ("ERROR: ungating anyway so the gang is not stranded, but the task "
111+ "produced no result" )
112+ ungate_pods (gated_pods_from_env () or wait_for_gated_pods (
113+ namespace , gang_group , exclude = pod_name , timeout = ungate_timeout ),
114+ job_id , namespace )
115+ sys .exit (1 )
98116
99117 # Ungate the gang: discover the gated pods in the gang group and remove their
100118 # gate, stamping the job-id so each can fetch results by id. The gang pods are
0 commit comments