@@ -185,6 +185,7 @@ def __del__(self):
185185 # Print performance info
186186 t_str = ""
187187 q_str = ""
188+ c_str = ""
188189 for t , q , c in zip (self .timings , self .q_len , self .exec_count ):
189190 if c > 0 :
190191 avg_time = t / c
@@ -194,7 +195,8 @@ def __del__(self):
194195 avg_q = 0.0
195196 t_str += " {:5.1f}" .format (avg_time )
196197 q_str += " {:5.1f}" .format (avg_q )
197- logging .info (f"{ self .tpu_name } per-segment timing (ms) & queue len:{ t_str } ;{ q_str } " )
198+ c_str += " {:5d}" .format (c )
199+ logging .info (f"{ self .tpu_name } timing, queue len, & count:{ t_str } ;{ q_str } ;{ c_str } " )
198200
199201 self .interpreter = None
200202 self .input_details = None
@@ -209,7 +211,6 @@ def __init__(self, tpu_list: list, fname_list: list):
209211 seg_count = len (fname_list )
210212 assert seg_count <= len (tpu_list ), "seg_count: {} tpu_list: {}" .format (seg_count , len (tpu_list ))
211213
212- self .max_idle_secs_before_recycle = MAX_WAIT_TIME
213214 self .max_pipeline_queue_length = MAX_PIPELINE_QUEUE_LEN
214215
215216 self .fname_list = fname_list
@@ -237,19 +238,31 @@ def __init__(self, tpu_list: list, fname_list: list):
237238 with open (fname , "rb" ) as fd :
238239 self .fbytes_list .append (fd .read ())
239240
241+ self ._init_interpreters ()
242+
243+ def _init_interpreters (self ):
244+
245+ start_boot_time = time .perf_counter_ns ()
246+
240247 # Fill TPUs with interpreters
241248 with self .balance_lock :
242- for i , tpu_name in enumerate (tpu_list ):
243- seg_idx = i % len (fname_list )
249+ for i , tpu_name in enumerate (self . tpu_list ):
250+ seg_idx = i % len (self . fname_list )
244251
245- i = DynamicInterpreter (fname_list , tpu_name , self .queues , self .rebalancing_lock )
252+ i = DynamicInterpreter (self . fname_list , tpu_name , self .queues , self .rebalancing_lock )
246253 i .start (seg_idx , self .fbytes_list [seg_idx ])
247254 self .interpreters [seg_idx ].append (i )
248255
249256 self .first_name = self .interpreters [0 ][0 ].input_details [0 ]['name' ]
257+
258+ boot_time = (time .perf_counter_ns () - start_boot_time ) / (1000.0 * 1000.0 )
259+ logging .info (f"Initialized pipeline interpreters in { boot_time :.1f} ms" )
250260
251261
252262 def enqueue (self , in_tensor , out_q : queue .Queue ):
263+ if not self .first_name :
264+ self ._init_interpreters ()
265+
253266 self .queues [0 ].put (({self .first_name : in_tensor }, out_q ))
254267
255268
@@ -367,12 +380,13 @@ def _halt_interpreters(self, seg_idx: int):
367380 for interpreter in self .interpreters [seg_idx ]:
368381 t = interpreter .thread
369382 logging .debug ("Joining thread {} for DynamicPipeline.delete()" .format (t .native_id ))
370- t .join (timeout = self . max_idle_secs_before_recycle )
383+ t .join (timeout = MAX_WAIT_TIME )
371384 if t .is_alive ():
372385 logging .warning ("Pipe thread didn't join!" )
373386
374387
375388 def delete (self ):
389+ # Kill interpreters. Maybe refresh later; maybe delete object.
376390 with self .balance_lock :
377391 # Insert EOF to each queue
378392 # Wait for threads to finish
@@ -384,7 +398,7 @@ def delete(self):
384398 if self .interpreters and len (self .interpreters ) > q_idx :
385399 self .interpreters [q_idx ] = []
386400
387- self .fbytes_list = None
401+ self .first_name = None
388402
389403
390404class TPURunner (object ):
@@ -396,8 +410,12 @@ def __init__(self, tpu_limit: int = -1):
396410 the registry.
397411 """
398412
413+ # Tricky because MAX_WAIT_TIME is intended to relatively quickly handle an error condition
414+ # before there are significant user-facing problems, whereas idling for N seconds isn't an
415+ # error condition.
416+ self .max_idle_secs_before_recycle = MAX_WAIT_TIME * 20
399417 self .watchdog_idle_secs = WATCHDOG_IDLE_SECS
400- self . max_idle_secs_before_recycle = MAX_WAIT_TIME
418+
401419 self .pipe_lifespan_secs = INTERPRETER_LIFESPAN_SECONDS
402420 self .warn_temperature_thresh_C = WARN_TEMPERATURE_THRESHOLD_CELSIUS
403421
@@ -446,12 +464,13 @@ def _watchdog(self):
446464 self .watchdog_time = time .time ()
447465 while not self .watchdog_shutdown :
448466 if self .pipe and \
449- time .time () - self .watchdog_time > self .max_idle_secs_before_recycle * 2 :
450- logging .warning ("No work in {} seconds, watchdog shutting down TPUs." .format (self .max_idle_secs_before_recycle * 2 ))
451- self .runner_lock .acquire (timeout = self .max_idle_secs_before_recycle )
452- self ._delete ()
453- self .runner_lock .release ()
454- # Pipeline will reinitialize itself as needed
467+ time .time () - self .watchdog_time > self .max_idle_secs_before_recycle :
468+ logging .warning ("No work in {} seconds, watchdog shutting down TPUs." .format (self .max_idle_secs_before_recycle ))
469+ self .runner_lock .acquire (timeout = MAX_WAIT_TIME )
470+ if self .pipe :
471+ self .pipe .delete ()
472+ self .runner_lock .release ()
473+ # Pipeline will reinitialize itself as needed
455474 time .sleep (self .watchdog_idle_secs )
456475
457476 logging .debug ("Watchdog caught shutdown in {}" .format (threading .get_ident ()))
@@ -779,7 +798,7 @@ def _process_image(self,
779798 # We may have to wait a few seconds at most, but I'd expect the
780799 # pipeline to clear fairly quickly.
781800 start_inference_time = time .perf_counter ()
782- result = rs_queue .get (timeout = self . max_idle_secs_before_recycle )
801+ result = rs_queue .get (timeout = MAX_WAIT_TIME )
783802 tot_inference_time += time .perf_counter () - start_inference_time
784803 assert result
785804
0 commit comments