Skip to content

Commit d35d6d0

Browse files
committed
Better watchdog handling
1 parent 3fd4747 commit d35d6d0

3 files changed

Lines changed: 45 additions & 26 deletions

File tree

‎src/modules/ObjectDetectionCoral/modulesettings.json‎

Lines changed: 9 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,7 @@
2525
"RuntimeLocation": "Local", // Can be Local, Shared or System
2626
"PostStartPauseSecs": 1, // Generally 1 if using GPU, 0 for CPU
2727
"Queue": "objectdetection_queue", // We make all Object detectors use the same queue.
28-
"Parallelism": 16 // 0 = Default = number of CPUs / 2
28+
"Parallelism": 16 // Should probably be TPU count * 2; I don't see harm in overprovisioning threads
2929
},
3030

3131
"ModelRequirements" : [{
@@ -36,7 +36,7 @@
3636

3737
"EnvironmentVariables": {
3838
"CPAI_CORAL_MULTI_TPU": "true",
39-
"CPAI_CORAL_MODEL_NAME": "MobileNet SSD", // "MobileNet SSD", "EfficientDet-Lite", "YOLOv5"
39+
"CPAI_CORAL_MODEL_NAME": "MobileNet SSD", // "MobileNet SSD", "EfficientDet-Lite", "YOLOv5", "YOLOv8"
4040

4141
"MODELS_DIR": "%CURRENT_MODULE_PATH%/assets",
4242
"MODEL_SIZE": "Small"
@@ -85,16 +85,16 @@
8585
],
8686
"DownloadableModels":[
8787

88-
{ "Name": "EfficientDet Large", "Filename": "objectdetection-efficientdet-large-edgetpu.zip", "Folder": "assets", "Description": "EfficientDet object detection, Large", "FileSizeKb": 275800, "PreInstall": false },
89-
{ "Name": "EfficientDet Medium", "Filename": "objectdetection-efficientdet-medium-edgetpu.zip", "Folder": "assets", "Description": "EfficientDet object detection, Medium", "FileSizeKb": 275800, "PreInstall": false },
90-
{ "Name": "EfficientDet Small", "Filename": "objectdetection-efficientdet-small-edgetpu.zip", "Folder": "assets", "Description": "EfficientDet object detection, Small", "FileSizeKb": 275800, "PreInstall": false },
91-
{ "Name": "EfficientDet Tiny", "Filename": "objectdetection-efficientdet-tiny-edgetpu.zip", "Folder": "assets", "Description": "EfficientDet object detection, Tiny", "FileSizeKb": 275800, "PreInstall": false },
92-
9388
{ "Name": "MobileNet Large", "Filename": "objectdetection-mobilenet-large-edgetpu.zip", "Folder": "assets", "Description": "MobileNet object detection, Large", "FileSizeKb": 275800, "PreInstall": true },
9489
{ "Name": "MobileNet Medium", "Filename": "objectdetection-mobilenet-medium-edgetpu.zip", "Folder": "assets", "Description": "MobileNet object detection, Medium", "FileSizeKb": 275800, "PreInstall": true },
9590
{ "Name": "MobileNet Small", "Filename": "objectdetection-mobilenet-small-edgetpu.zip", "Folder": "assets", "Description": "MobileNet object detection, Small", "FileSizeKb": 275800, "PreInstall": true },
9691
{ "Name": "MobileNet Tiny", "Filename": "objectdetection-mobilenet-tiny-edgetpu.zip", "Folder": "assets", "Description": "MobileNet object detection, Tiny", "FileSizeKb": 275800, "PreInstall": true },
9792

93+
{ "Name": "EfficientDet Large", "Filename": "objectdetection-efficientdet-large-edgetpu.zip", "Folder": "assets", "Description": "EfficientDet object detection, Large", "FileSizeKb": 275800, "PreInstall": false },
94+
{ "Name": "EfficientDet Medium", "Filename": "objectdetection-efficientdet-medium-edgetpu.zip", "Folder": "assets", "Description": "EfficientDet object detection, Medium", "FileSizeKb": 275800, "PreInstall": false },
95+
{ "Name": "EfficientDet Small", "Filename": "objectdetection-efficientdet-small-edgetpu.zip", "Folder": "assets", "Description": "EfficientDet object detection, Small", "FileSizeKb": 275800, "PreInstall": false },
96+
{ "Name": "EfficientDet Tiny", "Filename": "objectdetection-efficientdet-tiny-edgetpu.zip", "Folder": "assets", "Description": "EfficientDet object detection, Tiny", "FileSizeKb": 275800, "PreInstall": false },
97+
9898
{ "Name": "YOLOv5 Large", "Filename": "objectdetection-yolov5-large-edgetpu.zip", "Folder": "assets", "Description": "YOLOv5 object detection, Large", "FileSizeKb": 275800, "PreInstall": false },
9999
{ "Name": "YOLOv5 Medium", "Filename": "objectdetection-yolov5-medium-edgetpu.zip", "Folder": "assets", "Description": "YOLOv5 object detection, Medium", "FileSizeKb": 275800, "PreInstall": false },
100100
{ "Name": "YOLOv5 Small", "Filename": "objectdetection-yolov5-small-edgetpu.zip", "Folder": "assets", "Description": "YOLOv5 object detection, Small", "FileSizeKb": 275800, "PreInstall": false },
@@ -112,9 +112,9 @@
112112
"Label": "Model",
113113
"Options": [
114114
{ "Label": "MobileNet SSD", "Setting": "CPAI_CORAL_MODEL_NAME", "Value": "MobileNet SSD" },
115+
{ "Label": "EfficientDet Lite", "Setting": "CPAI_CORAL_MODEL_NAME", "Value": "EfficientDet-Lite" },
115116
{ "Label": "YOLOv5", "Setting": "CPAI_CORAL_MODEL_NAME", "Value": "YOLOv5" },
116-
{ "Label": "YOLOv8", "Setting": "CPAI_CORAL_MODEL_NAME", "Value": "YOLOv8" },
117-
{ "Label": "EfficientDet Lite", "Setting": "CPAI_CORAL_MODEL_NAME", "Value": "EfficientDet-Lite" }
117+
{ "Label": "YOLOv8", "Setting": "CPAI_CORAL_MODEL_NAME", "Value": "YOLOv8" }
118118
]
119119
},
120120
{

‎src/modules/ObjectDetectionCoral/objectdetection_coral_multitpu.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -257,15 +257,15 @@ def main():
257257
objs, infr_time, _ = _tpu_runner.process_image(options, copy.copy(image), args.threshold)
258258
tot_infr_time += infr_time
259259
wall_time = time.perf_counter() - start
260-
print('completed one run every %.2f ms for %d runs; %.2f ms wall time for a single run' %
260+
print('completed one run every %.2fms for %d runs; %.2fms wall time for a single run' %
261261
(wall_time * 1000 / args.count, args.count,
262262
(time.perf_counter() - start_one) * 1000))
263263

264264
# Optimizing the number of segments used for a model would result in the
265265
# lowest average time spent adjusted for number of TPUs used. At some point,
266266
# adding additional segments just removes from the pool of TPUs you can use
267267
# for parallelism.
268-
print('%.2f ms avg time blocked across %d threads; %.2f avg TPU * ms / run' %
268+
print('%.2fms avg time blocked across %d threads; %.2f avg TPU * ms / run' %
269269
(tot_infr_time / args.count, thread_cnt,
270270
len(_tpu_runner.pipe.tpu_list) * wall_time * 1000 / args.count))
271271

‎src/modules/ObjectDetectionCoral/tpu_runner.py‎

Lines changed: 34 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -185,6 +185,7 @@ def __del__(self):
185185
# Print performance info
186186
t_str = ""
187187
q_str = ""
188+
c_str = ""
188189
for t, q, c in zip(self.timings, self.q_len, self.exec_count):
189190
if c > 0:
190191
avg_time = t / c
@@ -194,7 +195,8 @@ def __del__(self):
194195
avg_q = 0.0
195196
t_str += " {:5.1f}".format(avg_time)
196197
q_str += " {:5.1f}".format(avg_q)
197-
logging.info(f"{self.tpu_name} per-segment timing (ms) & queue len:{t_str};{q_str}")
198+
c_str += " {:5d}".format(c)
199+
logging.info(f"{self.tpu_name} timing, queue len, & count:{t_str};{q_str};{c_str}")
198200

199201
self.interpreter = None
200202
self.input_details = None
@@ -209,7 +211,6 @@ def __init__(self, tpu_list: list, fname_list: list):
209211
seg_count = len(fname_list)
210212
assert seg_count <= len(tpu_list), "seg_count: {} tpu_list: {}".format(seg_count, len(tpu_list))
211213

212-
self.max_idle_secs_before_recycle = MAX_WAIT_TIME
213214
self.max_pipeline_queue_length = MAX_PIPELINE_QUEUE_LEN
214215

215216
self.fname_list = fname_list
@@ -237,19 +238,31 @@ def __init__(self, tpu_list: list, fname_list: list):
237238
with open(fname, "rb") as fd:
238239
self.fbytes_list.append(fd.read())
239240

241+
self._init_interpreters()
242+
243+
def _init_interpreters(self):
244+
245+
start_boot_time = time.perf_counter_ns()
246+
240247
# Fill TPUs with interpreters
241248
with self.balance_lock:
242-
for i, tpu_name in enumerate(tpu_list):
243-
seg_idx = i % len(fname_list)
249+
for i, tpu_name in enumerate(self.tpu_list):
250+
seg_idx = i % len(self.fname_list)
244251

245-
i = DynamicInterpreter(fname_list, tpu_name, self.queues, self.rebalancing_lock)
252+
i = DynamicInterpreter(self.fname_list, tpu_name, self.queues, self.rebalancing_lock)
246253
i.start(seg_idx, self.fbytes_list[seg_idx])
247254
self.interpreters[seg_idx].append(i)
248255

249256
self.first_name = self.interpreters[0][0].input_details[0]['name']
257+
258+
boot_time = (time.perf_counter_ns() - start_boot_time) / (1000.0 * 1000.0)
259+
logging.info(f"Initialized pipeline interpreters in {boot_time:.1f}ms")
250260

251261

252262
def enqueue(self, in_tensor, out_q: queue.Queue):
263+
if not self.first_name:
264+
self._init_interpreters()
265+
253266
self.queues[0].put(({self.first_name: in_tensor}, out_q))
254267

255268

@@ -367,12 +380,13 @@ def _halt_interpreters(self, seg_idx: int):
367380
for interpreter in self.interpreters[seg_idx]:
368381
t = interpreter.thread
369382
logging.debug("Joining thread {} for DynamicPipeline.delete()".format(t.native_id))
370-
t.join(timeout=self.max_idle_secs_before_recycle)
383+
t.join(timeout=MAX_WAIT_TIME)
371384
if t.is_alive():
372385
logging.warning("Pipe thread didn't join!")
373386

374387

375388
def delete(self):
389+
# Kill interpreters. Maybe refresh later; maybe delete object.
376390
with self.balance_lock:
377391
# Insert EOF to each queue
378392
# Wait for threads to finish
@@ -384,7 +398,7 @@ def delete(self):
384398
if self.interpreters and len(self.interpreters) > q_idx:
385399
self.interpreters[q_idx] = []
386400

387-
self.fbytes_list = None
401+
self.first_name = None
388402

389403

390404
class TPURunner(object):
@@ -396,8 +410,12 @@ def __init__(self, tpu_limit: int = -1):
396410
the registry.
397411
"""
398412

413+
# Tricky because MAX_WAIT_TIME is intended to relatively quickly handle an error condition
414+
# before there are significant user-facing problems, whereas idling for N seconds isn't an
415+
# error condition.
416+
self.max_idle_secs_before_recycle = MAX_WAIT_TIME * 20
399417
self.watchdog_idle_secs = WATCHDOG_IDLE_SECS
400-
self.max_idle_secs_before_recycle = MAX_WAIT_TIME
418+
401419
self.pipe_lifespan_secs = INTERPRETER_LIFESPAN_SECONDS
402420
self.warn_temperature_thresh_C = WARN_TEMPERATURE_THRESHOLD_CELSIUS
403421

@@ -446,12 +464,13 @@ def _watchdog(self):
446464
self.watchdog_time = time.time()
447465
while not self.watchdog_shutdown:
448466
if self.pipe and \
449-
time.time() - self.watchdog_time > self.max_idle_secs_before_recycle*2:
450-
logging.warning("No work in {} seconds, watchdog shutting down TPUs.".format(self.max_idle_secs_before_recycle*2))
451-
self.runner_lock.acquire(timeout=self.max_idle_secs_before_recycle)
452-
self._delete()
453-
self.runner_lock.release()
454-
# Pipeline will reinitialize itself as needed
467+
time.time() - self.watchdog_time > self.max_idle_secs_before_recycle:
468+
logging.warning("No work in {} seconds, watchdog shutting down TPUs.".format(self.max_idle_secs_before_recycle))
469+
self.runner_lock.acquire(timeout=MAX_WAIT_TIME)
470+
if self.pipe:
471+
self.pipe.delete()
472+
self.runner_lock.release()
473+
# Pipeline will reinitialize itself as needed
455474
time.sleep(self.watchdog_idle_secs)
456475

457476
logging.debug("Watchdog caught shutdown in {}".format(threading.get_ident()))
@@ -779,7 +798,7 @@ def _process_image(self,
779798
# We may have to wait a few seconds at most, but I'd expect the
780799
# pipeline to clear fairly quickly.
781800
start_inference_time = time.perf_counter()
782-
result = rs_queue.get(timeout=self.max_idle_secs_before_recycle)
801+
result = rs_queue.get(timeout=MAX_WAIT_TIME)
783802
tot_inference_time += time.perf_counter() - start_inference_time
784803
assert result
785804

0 commit comments

Comments
 (0)