Skip to content

Commit 80250e0

Browse files
committed
fewer memory ops
1 parent dc972c8 commit 80250e0

4 files changed

Lines changed: 50 additions & 44 deletions

File tree

‎src/modules/ObjectDetectionCoral/objectdetection_coral_multitpu.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -238,7 +238,7 @@ def main():
238238
fs = [executor.submit(_tpu_runner.process_image, options, copy.copy(image), args.threshold)
239239
for i in range(min(thread_cnt*8, args.count-1 - chunk_i))]
240240
for f in concurrent.futures.as_completed(fs):
241-
_, infr_time = f.result()
241+
_, infr_time, _ = f.result()
242242
tot_infr_time += infr_time
243243

244244
# Uncomment for testing
@@ -254,7 +254,7 @@ def main():
254254
# print(stat)
255255

256256
start_one = time.perf_counter()
257-
objs, infr_time = _tpu_runner.process_image(options, image, args.threshold)
257+
objs, infr_time, _ = _tpu_runner.process_image(options, copy.copy(image), args.threshold)
258258
tot_infr_time += infr_time
259259
wall_time = time.perf_counter() - start
260260
print('completed one run every %.2f ms for %d runs; %.2f ms wall time for a single run' %

‎src/modules/ObjectDetectionCoral/options.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@
77
print("Unable to import ModuleOptions, running with defaults")
88
class ModuleOptions:
99
module_path = '.'
10-
def getEnvVariable(self, a, b):
10+
def getEnvVariable(a, b):
1111
return b
1212

1313
class Settings:
@@ -169,7 +169,7 @@ def __init__(self):
169169
# ----------------------------------------------------------------------
170170
# Setup values
171171

172-
self._show_env_variables = True
172+
self._show_env_variables = False
173173

174174
self.module_path = ModuleOptions.module_path
175175
self.models_dir = os.path.normpath(ModuleOptions.getEnvVariable("MODELS_DIR", f"{self.module_path}/assets"))

‎src/modules/ObjectDetectionCoral/segment_and_test.py‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -11,12 +11,12 @@
1111
#'ssd_mobilenet_v1_coco_quant_postprocess',
1212
'tf2_ssd_mobilenet_v1_fpn_640x640_coco17_ptq',
1313
#'efficientdet_lite0_320_ptq',
14-
'efficientdet_lite1_384_ptq',
14+
#'efficientdet_lite1_384_ptq',
1515
'efficientdet_lite2_448_ptq',
1616
'efficientdet_lite3_512_ptq',
1717
'efficientdet_lite3x_640_ptq',
1818
#'yolov5n-int8',
19-
'yolov5s-int8',
19+
#'yolov5s-int8',
2020
'yolov5m-int8',
2121
'yolov5l-int8',
2222
#'yolov8n_416_640px', # lg 1st seg
@@ -192,7 +192,7 @@ def seg_exists(filename, segment_type, segment_count):
192192

193193
MAX_TPU_COUNT = 8
194194

195-
#'''
195+
'''
196196
# Generate segment files
197197
for sn in range(1,MAX_TPU_COUNT+1):
198198
for fn in fn_list:

‎src/modules/ObjectDetectionCoral/tpu_runner.py‎

Lines changed: 43 additions & 37 deletions
Original file line numberDiff line numberDiff line change
@@ -113,21 +113,25 @@ def start(self, seg_idx: int, fbytes: bytes):
113113

114114

115115
def _interpreter_runner(self, seg_idx: int):
116-
# Input tensors for this interpreter
117-
input_tensors = {}
118-
for details in self.input_details:
119-
input_tensors[details['name']] = self.interpreter.tensor(details['index'])
116+
in_names = [d['name'] for d in self.input_details ]
117+
out_names = [d['name'] for d in self.output_details]
118+
indices = [d['index'] for d in self.output_details]
119+
first_in_name = in_names.pop(0)
120120

121121
# Setup input/output queues
122122
in_q = self.queues[seg_idx]
123+
out_q = None
123124
if len(self.queues) > seg_idx+1:
124125
out_q = self.queues[seg_idx+1]
125-
else:
126-
out_q = None
127126

128-
in_names = [d['name'] for d in self.input_details ]
129-
out_names = [d['name'] for d in self.output_details]
130-
indices = [d['index'] for d in self.output_details]
127+
# Input tensors for this interpreter
128+
input_tensors = {}
129+
for details in self.input_details:
130+
input_tensors[details['name']] = self.interpreter.tensor(details['index'])
131+
output_tensors = []
132+
if not out_q:
133+
for details in self.output_details:
134+
output_tensors.append(self.interpreter.tensor(details['index']))
131135

132136
expected_input_size = np.prod(self.input_details[0]['shape'])
133137
interpreter_handle = self.interpreter._native_handle()
@@ -149,19 +153,15 @@ def _interpreter_runner(self, seg_idx: int):
149153

150154
start_inference_time = time.perf_counter_ns()
151155

152-
if len(in_names) == 1:
153-
# Invoke_with_membuffer() directly on numpy memory,
154-
# but only works with a single input
155-
edgetpu.invoke_with_membuffer(interpreter_handle,
156-
working_tensors[0][in_names[0]].ctypes.data,
157-
expected_input_size)
158-
else:
159-
# Set inputs
160-
for name in in_names:
161-
input_tensors[name]()[0] = working_tensors[0][name]
156+
# Set inputs beyond the first
157+
for name in in_names:
158+
input_tensors[name]()[0] = working_tensors[0][name]
162159

163-
# Run segment
164-
self.interpreter.invoke()
160+
# Invoke_with_membuffer() directly on numpy memory,
161+
# but only works with a single input
162+
edgetpu.invoke_with_membuffer(interpreter_handle,
163+
working_tensors[0][first_in_name].ctypes.data,
164+
expected_input_size)
165165

166166
if out_q:
167167
# Fetch results
@@ -171,9 +171,10 @@ def _interpreter_runner(self, seg_idx: int):
171171
# Deliver to next queue in pipeline
172172
out_q.put(working_tensors)
173173
else:
174-
# Fetch results
174+
# Fetch pointer to results
175+
# Copy and convert to float
175176
# Deliver to final results queue
176-
working_tensors[1].put([self.interpreter.get_tensor(index) for index in indices])
177+
working_tensors[1].put([t().astype(np.float32) for t in output_tensors])
177178

178179
# Convert elapsed time to double precision ms
179180
self.timings[seg_idx] += (time.perf_counter_ns() - start_inference_time) / (1000.0 * 1000.0)
@@ -474,9 +475,9 @@ def _get_tpu_devices(self):
474475
['usb:%d' % i for i in range(max(0, len(edge_tpus) - num_pci_devices))]
475476

476477
if self.tpu_limit > 0:
477-
return tpu_l[:self.tpu_limit]
478+
return tpu_l[4:self.tpu_limit]
478479
else:
479-
return tpu_l
480+
return tpu_l[4:]
480481

481482

482483
def _get_model_filenames(self, options: Options, tpu_list: list) -> list:
@@ -526,7 +527,7 @@ def _get_model_filenames(self, options: Options, tpu_list: list) -> list:
526527

527528

528529
# Should be called while holding runner_lock (if called at run time)
529-
def init_pipe(self, options: Options) -> tuple[str, str]:
530+
def init_pipe(self, options: Options) -> tuple:
530531
"""
531532
Initializes the pipe with the TFLite models.
532533
@@ -599,7 +600,7 @@ def init_pipe(self, options: Options) -> tuple[str, str]:
599600

600601

601602
def _periodic_check(self, options: Options, force: bool = False,
602-
check_temp: bool = True, check_refresh: bool = True) -> tuple[bool, str]:
603+
check_temp: bool = True, check_refresh: bool = True) -> tuple:
603604
"""
604605
Run a periodic check to ensure the temperatures are good and we don't
605606
need to (re)initialize the interpreters/workers/pipelines. The system
@@ -743,6 +744,9 @@ def _process_image(self,
743744
- Remove duplicate results.
744745
- Return results as Objects.
745746
- Return inference timing.
747+
748+
Note that the image object is modified in place to resize it
749+
for in input tensor.
746750
"""
747751

748752
all_objects = []
@@ -837,11 +841,11 @@ def _decode_result(self, result_list, score_threshold: float):
837841
# using a v8 or v5-based network.
838842
if j < k:
839843
rs = self._yolov8_non_max_suppression(
840-
(dict_values.astype('float32') - output_zero) * output_scale,
844+
(dict_values - output_zero) * output_scale,
841845
conf_thres=score_threshold)
842846
else:
843847
rs = self._yolov5_non_max_suppression(
844-
(dict_values.astype('float32') - output_zero) * output_scale,
848+
(dict_values - output_zero) * output_scale,
845849
conf_thres=score_threshold)
846850

847851
for a in rs:
@@ -957,7 +961,6 @@ def _yolov8_non_max_suppression(self, prediction, conf_thres=0.25, iou_thres=0.4
957961
if (time.time() - t) > time_limit:
958962
logging.warning(f'NMS time limit {time_limit}s exceeded')
959963
break # time limit exceeded
960-
961964
return output
962965

963966

@@ -1145,26 +1148,29 @@ def to_numpy(im, shape, typestr):
11451148
self.printed_shape_map[key] = True
11461149

11471150
# Chop & resize image piece
1148-
resamp_img = image.convert('RGB')
1149-
resamp_img.thumbnail((resamp_x, resamp_y), Image.LANCZOS)
1151+
if image.mode != 'RGB':
1152+
image = image.convert('RGB')
1153+
image.thumbnail((resamp_x, resamp_y), Image.LANCZOS)
11501154

11511155
# Do chunking
11521156
tiles = []
11531157
for x_off in range(0, resamp_x - options.tile_overlap, m_width - options.tile_overlap):
11541158
for y_off in range(0, resamp_y - options.tile_overlap, m_height - options.tile_overlap):
1155-
cropped_arr = to_numpy(resamp_img.crop((x_off,
1156-
y_off,
1157-
x_off + m_width,
1158-
y_off + m_height)), self.input_details['shape'][1:], np.uint8)
1159+
cropped_arr = to_numpy(image.crop((x_off,
1160+
y_off,
1161+
x_off + m_width,
1162+
y_off + m_height)), self.input_details['shape'][1:], np.uint8)
11591163

11601164
logging.debug("Resampled image tile {} at offset {}, {}".format(cropped_arr.shape, x_off, y_off))
1161-
resamp_info = (x_off, y_off, i_width/resamp_img.width, i_height/resamp_img.height)
1165+
resamp_info = (x_off, y_off, i_width/image.width, i_height/image.height)
11621166

11631167
# Normalize from 8-bit image to whatever the input is
11641168
if self.input_details['dtype'] == np.int8:
11651169
tiles.append(((cropped_arr - 128).astype(np.int8), resamp_info))
11661170
else:
11671171
tiles.append((cropped_arr, resamp_info))
1172+
1173+
#Image.fromarray((tiles[-1][0] + 128).astype(np.uint8)).save("test.png")
11681174
return tiles
11691175

11701176

0 commit comments

Comments
 (0)