Skip to content

Commit 0bdc7c7

Browse files
[executorch][native-vk] Add a torchvision classifier runner (pytorch#23460)
This PR was created by the merge bot to help merge the original PR into the main branch. ghstack PR number: pytorch#23425 by @digantdesai ^ Please use this as the source of truth for the PR details, comments, and reviews ghstack PR base: https://github.com/pytorch/executorch/tree/gh/digantdesai/120/base ghstack PR head: https://github.com/pytorch/executorch/tree/gh/digantdesai/120/head Merge bot PR base: https://github.com/pytorch/executorch/tree/gh/digantdesai/119/orig Merge bot PR head: https://github.com/pytorch/executorch/tree/gh/digantdesai/120/orig Differential Revision: [D119601867](https://our.internmc.facebook.com/intern/diff/D119601867/) @diff-train-skip-merge Co-authored-by: Digant Desai <digantdesai@meta.com>
1 parent 81beb7e commit 0bdc7c7

7 files changed

Lines changed: 451 additions & 0 deletions

File tree

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
load("@fbcode_macros//build_defs:build_file_migration.bzl", "fbcode_target", "non_fbcode_target")
2+
load(":targets.bzl", "define_common_targets")
3+
4+
oncall("executorch")
5+
6+
non_fbcode_target(_kind = define_common_targets)
7+
8+
fbcode_target(_kind = define_common_targets)
Lines changed: 143 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,143 @@
1+
// Copyright (c) Meta Platforms, Inc. and affiliates.
2+
// All rights reserved.
3+
//
4+
// This source code is licensed under the BSD-style license found in the
5+
// LICENSE file in the root directory of this source tree.
6+
7+
#include <executorch/backends/native/examples/runner_utils/runner_utils.h>
8+
9+
#include <algorithm>
10+
#include <cmath>
11+
#include <cstdio>
12+
#include <fstream>
13+
#include <functional>
14+
#include <limits>
15+
#include <numeric>
16+
#include <stdexcept>
17+
18+
namespace ptn::runner_utils {
19+
20+
std::vector<float> read_floats(const std::string& path) {
21+
std::ifstream file(path, std::ios::binary | std::ios::ate);
22+
if (!file) {
23+
throw std::runtime_error("cannot open " + path);
24+
}
25+
const std::streamsize size = file.tellg();
26+
if (size < 0 || size % static_cast<std::streamsize>(sizeof(float)) != 0) {
27+
throw std::runtime_error("not a whole number of floats: " + path);
28+
}
29+
file.seekg(0, std::ios::beg);
30+
std::vector<float> out(static_cast<size_t>(size) / sizeof(float));
31+
// cppcheck-suppress invalidPointerCast
32+
if (size > 0 && !file.read(reinterpret_cast<char*>(out.data()), size)) {
33+
throw std::runtime_error("cannot read " + path);
34+
}
35+
return out;
36+
}
37+
38+
std::vector<std::string> read_lines(const std::string& path) {
39+
std::vector<std::string> lines;
40+
std::ifstream file(path);
41+
if (!file) {
42+
throw std::runtime_error("cannot open " + path);
43+
}
44+
std::string line;
45+
while (std::getline(file, line)) {
46+
lines.push_back(line);
47+
}
48+
return lines;
49+
}
50+
51+
int64_t numel_of(const std::vector<int64_t>& sizes) {
52+
return std::accumulate(
53+
sizes.begin(), sizes.end(), int64_t{1}, std::multiplies<int64_t>());
54+
}
55+
56+
std::string shape_str(const std::vector<int64_t>& sizes) {
57+
std::string out = "[";
58+
for (size_t i = 0; i < sizes.size(); ++i) {
59+
out += (i == 0 ? "" : ", ") + std::to_string(sizes[i]);
60+
}
61+
return out + "]";
62+
}
63+
64+
std::vector<int> top_k(const std::vector<float>& values, int k) {
65+
std::vector<int> order(values.size());
66+
std::iota(order.begin(), order.end(), 0);
67+
const int count = std::clamp<int>(k, 0, static_cast<int>(values.size()));
68+
std::partial_sort(
69+
order.begin(), order.begin() + count, order.end(), [&](int a, int b) {
70+
return values[a] > values[b];
71+
});
72+
order.resize(count);
73+
return order;
74+
}
75+
76+
bool compare_floats(
77+
const std::vector<float>& actual,
78+
const std::vector<float>& expected,
79+
double rtol,
80+
double atol) {
81+
if (!std::isfinite(rtol) || !std::isfinite(atol) || rtol < 0 || atol < 0) {
82+
std::fprintf(
83+
stderr,
84+
"error: tolerances must be finite and non-negative (rtol=%g, "
85+
"atol=%g)\n",
86+
rtol,
87+
atol);
88+
return false;
89+
}
90+
if (actual.size() != expected.size()) {
91+
std::fprintf(
92+
stderr,
93+
"error: reference has %zu values, model produced %zu\n",
94+
expected.size(),
95+
actual.size());
96+
return false;
97+
}
98+
size_t failures = 0;
99+
size_t worst_at = 0;
100+
double worst_excess = 0.0;
101+
double max_abs_err = 0.0;
102+
for (size_t i = 0; i < actual.size(); ++i) {
103+
if (!std::isfinite(actual[i]) || !std::isfinite(expected[i])) {
104+
++failures;
105+
if (!std::isinf(worst_excess)) {
106+
worst_at = i;
107+
worst_excess = std::numeric_limits<double>::infinity();
108+
}
109+
max_abs_err = std::numeric_limits<double>::infinity();
110+
continue;
111+
}
112+
const double abs_err = std::fabs(
113+
static_cast<double>(actual[i]) - static_cast<double>(expected[i]));
114+
const double budget =
115+
atol + rtol * std::fabs(static_cast<double>(expected[i]));
116+
max_abs_err = std::max(max_abs_err, abs_err);
117+
if (abs_err > budget) {
118+
++failures;
119+
if (abs_err - budget > worst_excess) {
120+
worst_excess = abs_err - budget;
121+
worst_at = i;
122+
}
123+
}
124+
}
125+
std::printf(
126+
"compare: %zu values, max abs err %.6g, rtol %g atol %g\n",
127+
actual.size(),
128+
max_abs_err,
129+
rtol,
130+
atol);
131+
if (failures == 0) {
132+
return true;
133+
}
134+
std::printf(
135+
" %zu value(s) outside tolerance; worst at %zu: got %.6g, want %.6g\n",
136+
failures,
137+
worst_at,
138+
static_cast<double>(actual[worst_at]),
139+
static_cast<double>(expected[worst_at]));
140+
return false;
141+
}
142+
143+
} // namespace ptn::runner_utils
Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
// Copyright (c) Meta Platforms, Inc. and affiliates.
2+
// All rights reserved.
3+
//
4+
// This source code is licensed under the BSD-style license found in the
5+
// LICENSE file in the root directory of this source tree.
6+
7+
#pragma once
8+
9+
#include <cstdint>
10+
#include <string>
11+
#include <vector>
12+
13+
namespace ptn::runner_utils {
14+
15+
std::vector<float> read_floats(const std::string& path);
16+
std::vector<std::string> read_lines(const std::string& path);
17+
18+
int64_t numel_of(const std::vector<int64_t>& sizes);
19+
std::string shape_str(const std::vector<int64_t>& sizes);
20+
std::vector<int> top_k(const std::vector<float>& values, int k);
21+
22+
bool compare_floats(
23+
const std::vector<float>& actual,
24+
const std::vector<float>& expected,
25+
double rtol,
26+
double atol);
27+
28+
} // namespace ptn::runner_utils
Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
load("@fbsource//xplat/executorch/build:runtime_wrapper.bzl", "runtime")
2+
3+
def define_common_targets():
4+
runtime.cxx_library(
5+
name = "runner_utils",
6+
srcs = ["runner_utils.cpp"],
7+
exported_headers = ["runner_utils.h"],
8+
visibility = ["//executorch/backends/native/examples/..."],
9+
)
10+
11+
runtime.cxx_test(
12+
name = "runner_utils_test",
13+
srcs = ["test_runner_utils.cpp"],
14+
deps = [":runner_utils"],
15+
)
Lines changed: 54 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,54 @@
1+
// Copyright (c) Meta Platforms, Inc. and affiliates.
2+
// All rights reserved.
3+
//
4+
// This source code is licensed under the BSD-style license found in the
5+
// LICENSE file in the root directory of this source tree.
6+
7+
#include <executorch/backends/native/examples/runner_utils/runner_utils.h>
8+
9+
#include <limits>
10+
#include <vector>
11+
12+
#include <gtest/gtest.h>
13+
14+
namespace ptn::runner_utils {
15+
namespace {
16+
17+
// cppcheck-suppress-begin syntaxError
18+
TEST(RunnerUtilsTest, CompareFloatsUsesAbsoluteAndRelativeTolerance) {
19+
EXPECT_TRUE(compare_floats({1.0f, 100.0f}, {1.0005f, 100.05f}, 1e-3, 1e-3));
20+
EXPECT_FALSE(compare_floats({1.0f, 100.0f}, {1.1f, 100.0f}, 1e-3, 1e-3));
21+
EXPECT_FALSE(compare_floats({1.0f}, {1.0f, 2.0f}, 1e-3, 1e-3));
22+
}
23+
24+
TEST(RunnerUtilsTest, CompareFloatsRejectsNonFiniteValues) {
25+
const float nan = std::numeric_limits<float>::quiet_NaN();
26+
const float infinity = std::numeric_limits<float>::infinity();
27+
28+
EXPECT_FALSE(compare_floats({nan}, {1.0f}, 1e-3, 1e-3));
29+
EXPECT_FALSE(compare_floats({1.0f}, {nan}, 1e-3, 1e-3));
30+
EXPECT_FALSE(compare_floats({infinity}, {infinity}, 1e-3, 1e-3));
31+
}
32+
33+
TEST(RunnerUtilsTest, CompareFloatsRejectsInvalidTolerances) {
34+
const double nan = std::numeric_limits<double>::quiet_NaN();
35+
const double infinity = std::numeric_limits<double>::infinity();
36+
37+
EXPECT_FALSE(compare_floats({1.0f}, {1.0f}, nan, 1e-3));
38+
EXPECT_FALSE(compare_floats({1.0f}, {1.0f}, 1e-3, infinity));
39+
EXPECT_FALSE(compare_floats({1.0f}, {1.0f}, -1e-3, 1e-3));
40+
}
41+
42+
TEST(RunnerUtilsTest, TopKReturnsEmptyForEmptyInput) {
43+
EXPECT_TRUE(top_k({}, 5).empty());
44+
EXPECT_TRUE(top_k({1.0f, 2.0f}, -1).empty());
45+
}
46+
47+
TEST(RunnerUtilsTest, TopKReturnsIndicesByDescendingValue) {
48+
EXPECT_EQ(top_k({0.1f, 0.9f, 0.5f, 0.7f}, 3), (std::vector<int>{1, 3, 2}));
49+
EXPECT_EQ(top_k({2.0f, 3.0f}, 5), (std::vector<int>{1, 0}));
50+
}
51+
// cppcheck-suppress-end syntaxError
52+
53+
} // namespace
54+
} // namespace ptn::runner_utils
Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
load("@fbcode_macros//build_defs:build_file_migration.bzl", "fbcode_target")
2+
load("@fbsource//xplat/executorch/build:runtime_wrapper.bzl", "runtime")
3+
4+
oncall("executorch")
5+
6+
# Run an exported .ptn on the Vulkan engine and check the logits against the
7+
# eager reference. Host-runnable on SwiftShader through the engine's transitive
8+
# ET-VK dependency.
9+
fbcode_target(
10+
_kind = runtime.cxx_binary,
11+
name = "run_classification",
12+
srcs = ["run_classification.cpp"],
13+
deps = [
14+
"//executorch/backends/native/examples/runner_utils:runner_utils",
15+
"//executorch/backends/native/runtime:runtime",
16+
"//executorch/backends/native/runtime/deserialize:package",
17+
"//executorch/backends/native/runtime/engine:engine",
18+
"//executorch/backends/native/runtime/graph:scalar_type",
19+
"//executorch/backends/native/runtime/vulkan:vulkan_engine",
20+
],
21+
external_deps = ["gflags"],
22+
visibility = ["PUBLIC"],
23+
)

0 commit comments

Comments
 (0)