Skip to content

Commit 68986e3

Browse files
chmjkbchmjkb
committed
wip
1 parent 8c1fbfc commit 68986e3

11 files changed

Lines changed: 181 additions & 114 deletions

File tree

‎apps/computer-vision/app/instance_segmentation/index.tsx‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -112,7 +112,7 @@ export default function InstanceSegmentationScreen() {
112112
const start = Date.now();
113113
const output = await forward(imageUri, {
114114
confidenceThreshold: 0.5,
115-
iouThreshold: 0.55,
115+
nms: { iouThreshold: 0.55 },
116116
maxInstances: 20,
117117
returnMaskAtOriginalResolution: true,
118118
inputSize,

‎apps/computer-vision/app/segment_anything/index.tsx‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -241,7 +241,7 @@ export default function SegmentAnythingScreen() {
241241
const start = Date.now();
242242
const output = await forward(imageUri, {
243243
confidenceThreshold: 0.4,
244-
iouThreshold: 0.9,
244+
nms: { iouThreshold: 0.9 },
245245
maxInstances: 50,
246246
returnMaskAtOriginalResolution: true,
247247
});

‎apps/computer-vision/components/vision_camera/tasks/InstanceSegmentationTask.tsx‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -133,7 +133,7 @@ export default function InstanceSegmentationTask({
133133
const ih = frame.width > frame.height ? frame.width : frame.height;
134134
const result = instSegRof(frame, isFrontCamera, {
135135
confidenceThreshold: 0.5,
136-
iouThreshold: 0.5,
136+
nms: { iouThreshold: 0.5 },
137137
maxInstances: 5,
138138
returnMaskAtOriginalResolution: false,
139139
...(activeModel === 'instanceSegmentationYolo26n' && {

‎packages/react-native-executorch/common/rnexecutorch/models/instance_segmentation/BaseInstanceSegmentation.cpp‎

Lines changed: 22 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -57,7 +57,8 @@ TensorPtr BaseInstanceSegmentation::buildInputTensor(const cv::Mat &image) {
5757
std::vector<types::Instance> BaseInstanceSegmentation::runInference(
5858
const cv::Mat &image, double confidenceThreshold, double iouThreshold,
5959
int32_t maxInstances, const std::vector<int32_t> &classIndices,
60-
bool returnMaskAtOriginalResolution, const std::string &methodName) {
60+
bool returnMaskAtOriginalResolution, const std::string &methodName,
61+
bool useWeightedNms) {
6162

6263
std::scoped_lock lock(inference_mutex_);
6364

@@ -86,34 +87,37 @@ std::vector<types::Instance> BaseInstanceSegmentation::runInference(
8687
auto instances = collectInstances(
8788
forwardResult.get(), originalSize, modelInputSize, confidenceThreshold,
8889
classIndices, returnMaskAtOriginalResolution);
89-
return finalizeInstances(std::move(instances), iouThreshold, maxInstances);
90+
return finalizeInstances(std::move(instances), iouThreshold, maxInstances,
91+
useWeightedNms);
9092
}
9193

9294
std::vector<types::Instance> BaseInstanceSegmentation::generateFromString(
9395
std::string imageSource, double confidenceThreshold, double iouThreshold,
9496
int32_t maxInstances, std::vector<int32_t> classIndices,
95-
bool returnMaskAtOriginalResolution, std::string methodName) {
97+
bool returnMaskAtOriginalResolution, std::string methodName,
98+
bool useWeightedNms) {
9699

97100
cv::Mat imageBGR = image_processing::readImage(imageSource);
98101
cv::Mat imageRGB;
99102
cv::cvtColor(imageBGR, imageRGB, cv::COLOR_BGR2RGB);
100103

101104
return runInference(imageRGB, confidenceThreshold, iouThreshold, maxInstances,
102-
classIndices, returnMaskAtOriginalResolution, methodName);
105+
classIndices, returnMaskAtOriginalResolution, methodName,
106+
useWeightedNms);
103107
}
104108

105109
std::vector<types::Instance> BaseInstanceSegmentation::generateFromFrame(
106110
jsi::Runtime &runtime, const jsi::Value &frameData,
107111
double confidenceThreshold, double iouThreshold, int32_t maxInstances,
108112
std::vector<int32_t> classIndices, bool returnMaskAtOriginalResolution,
109-
std::string methodName) {
113+
std::string methodName, bool useWeightedNms) {
110114

111115
auto orient = ::rnexecutorch::utils::readFrameOrientation(runtime, frameData);
112116
cv::Mat frame = extractFromFrame(runtime, frameData);
113117
cv::Mat rotated = utils::rotateFrameForModel(frame, orient);
114-
auto instances =
115-
runInference(rotated, confidenceThreshold, iouThreshold, maxInstances,
116-
classIndices, returnMaskAtOriginalResolution, methodName);
118+
auto instances = runInference(
119+
rotated, confidenceThreshold, iouThreshold, maxInstances, classIndices,
120+
returnMaskAtOriginalResolution, methodName, useWeightedNms);
117121
for (auto &inst : instances) {
118122
utils::inverseRotateBbox(inst.bbox, orient, rotated.size());
119123
// Inverse-rotate the mask to match the screen orientation
@@ -131,11 +135,13 @@ std::vector<types::Instance> BaseInstanceSegmentation::generateFromFrame(
131135
std::vector<types::Instance> BaseInstanceSegmentation::generateFromPixels(
132136
JSTensorViewIn tensorView, double confidenceThreshold, double iouThreshold,
133137
int32_t maxInstances, std::vector<int32_t> classIndices,
134-
bool returnMaskAtOriginalResolution, std::string methodName) {
138+
bool returnMaskAtOriginalResolution, std::string methodName,
139+
bool useWeightedNms) {
135140

136141
cv::Mat image = extractFromPixels(tensorView);
137142
return runInference(image, confidenceThreshold, iouThreshold, maxInstances,
138-
classIndices, returnMaskAtOriginalResolution, methodName);
143+
classIndices, returnMaskAtOriginalResolution, methodName,
144+
useWeightedNms);
139145
}
140146

141147
std::tuple<utils::computer_vision::BBox, float, int32_t>
@@ -296,11 +302,14 @@ void BaseInstanceSegmentation::ensureMethodLoaded(
296302

297303
std::vector<types::Instance> BaseInstanceSegmentation::finalizeInstances(
298304
std::vector<types::Instance> instances, double iouThreshold,
299-
int32_t maxInstances) const {
305+
int32_t maxInstances, bool useWeightedNms) const {
300306

301307
if (applyNMS_) {
302-
instances =
303-
utils::computer_vision::nonMaxSuppression(instances, iouThreshold);
308+
instances = useWeightedNms
309+
? utils::computer_vision::weightedNonMaxSuppression(
310+
instances, iouThreshold)
311+
: utils::computer_vision::nonMaxSuppression(instances,
312+
iouThreshold);
304313
}
305314

306315
if (std::cmp_greater(instances.size(), maxInstances)) {

‎packages/react-native-executorch/common/rnexecutorch/models/instance_segmentation/BaseInstanceSegmentation.h‎

Lines changed: 11 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -28,30 +28,32 @@ class BaseInstanceSegmentation : public VisionModel {
2828
double iouThreshold, int32_t maxInstances,
2929
std::vector<int32_t> classIndices,
3030
bool returnMaskAtOriginalResolution,
31-
std::string methodName);
31+
std::string methodName, bool useWeightedNms);
3232

3333
[[nodiscard("Registered non-void function")]] std::vector<types::Instance>
3434
generateFromFrame(jsi::Runtime &runtime, const jsi::Value &frameData,
3535
double confidenceThreshold, double iouThreshold,
3636
int32_t maxInstances, std::vector<int32_t> classIndices,
37-
bool returnMaskAtOriginalResolution,
38-
std::string methodName);
37+
bool returnMaskAtOriginalResolution, std::string methodName,
38+
bool useWeightedNms);
3939

4040
[[nodiscard("Registered non-void function")]] std::vector<types::Instance>
4141
generateFromPixels(JSTensorViewIn tensorView, double confidenceThreshold,
4242
double iouThreshold, int32_t maxInstances,
4343
std::vector<int32_t> classIndices,
4444
bool returnMaskAtOriginalResolution,
45-
std::string methodName);
45+
std::string methodName, bool useWeightedNms);
4646

4747
protected:
4848
cv::Size modelInputSize() const override;
4949

5050
private:
51-
std::vector<types::Instance> runInference(
52-
const cv::Mat &image, double confidenceThreshold, double iouThreshold,
53-
int32_t maxInstances, const std::vector<int32_t> &classIndices,
54-
bool returnMaskAtOriginalResolution, const std::string &methodName);
51+
std::vector<types::Instance>
52+
runInference(const cv::Mat &image, double confidenceThreshold,
53+
double iouThreshold, int32_t maxInstances,
54+
const std::vector<int32_t> &classIndices,
55+
bool returnMaskAtOriginalResolution,
56+
const std::string &methodName, bool useWeightedNms);
5557

5658
TensorPtr buildInputTensor(const cv::Mat &image);
5759

@@ -89,7 +91,7 @@ class BaseInstanceSegmentation : public VisionModel {
8991

9092
std::vector<types::Instance>
9193
finalizeInstances(std::vector<types::Instance> instances, double iouThreshold,
92-
int32_t maxInstances) const;
94+
int32_t maxInstances, bool useWeightedNms) const;
9395

9496
cv::Mat processMaskFromLogits(
9597
const cv::Mat &logitsMat, const utils::computer_vision::BBox &bboxModel,

0 commit comments

Comments
 (0)