Skip to content

Commit dc8c578

Browse files
committed
[ET-VK][testing] Reuse allocations in chained benchmarks
Pull Request resolved: pytorch#22625 Build each benchmark operator once and replay its `execute_nodes()` in a benchmark-owned reusable command buffer. This prevents chained timing from multiplying persistent prepack and intermediate GPU allocations while leaving production `ComputeGraph::execute()` behavior unchanged. Differential Revision: [D119237811](https://our.internmc.facebook.com/intern/diff/D119237811/) ghstack-source-id: 426683626
1 parent 9007da4 commit dc8c578

3 files changed

Lines changed: 208 additions & 45 deletions

File tree

‎backends/vulkan/test/custom_ops/utils.cpp‎

Lines changed: 89 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,56 @@ constexpr float kMinProbeTimeUs = 1.0f;
3131

3232
} // namespace
3333

34+
// Benchmark graphs are immutable after input upload, so the operator nodes can
35+
// be recorded repeatedly. Staging uploads/downloads are encoded once:
36+
// repeating them would redo host-visible copies on every repetition and bias
37+
// per-invocation timings.
38+
RepeatedGraphExecutor::RepeatedGraphExecutor(
39+
ComputeGraph& graph,
40+
int repetitions,
41+
OpNodeRange op_nodes)
42+
: graph_(graph) {
43+
VK_CHECK_COND(repetitions > 0);
44+
VK_CHECK_COND(op_nodes.begin <= op_nodes.end);
45+
VK_CHECK_COND(op_nodes.end <= graph_.execute_nodes().size());
46+
47+
api::Context* const context = graph_.context();
48+
context->flush();
49+
context->set_cmd(/*reusable=*/true);
50+
context->cmd_reset_querypool();
51+
52+
for (size_t i = 0; i < op_nodes.begin; ++i) {
53+
graph_.execute_nodes()[i]->encode(&graph_);
54+
}
55+
for (int i = 0; i < repetitions; ++i) {
56+
for (size_t j = op_nodes.begin; j < op_nodes.end; ++j) {
57+
graph_.execute_nodes()[j]->encode(&graph_);
58+
}
59+
}
60+
for (size_t i = op_nodes.end; i < graph_.execute_nodes().size(); ++i) {
61+
graph_.execute_nodes()[i]->encode(&graph_);
62+
}
63+
64+
command_ =
65+
std::make_unique<vkapi::CommandBuffer>(std::move(context->extract_cmd()));
66+
}
67+
68+
void RepeatedGraphExecutor::execute() {
69+
api::Context* const context = graph_.context();
70+
command_->end();
71+
72+
// Intentionally bypasses Context::submit_cmd_to_gpu(): its submit-count
73+
// bookkeeping and threshold-split path only serve submit_compute_job
74+
// recording, which benchmarks don't use.
75+
vkapi::VulkanFence fence = context->fences().get_fence();
76+
context->adapter_ptr()->submit_cmd(
77+
context->queue(),
78+
command_->get_submit_handle(/*final_use=*/false),
79+
fence.get_submit_handle());
80+
fence.wait();
81+
context->fences().return_fence(fence);
82+
}
83+
3484
int get_seed() {
3585
static int seed = 42;
3686
return seed++;
@@ -1433,17 +1483,23 @@ int64_t default_flop_calculator(const TestCase& test_case) {
14331483
return total_elements;
14341484
}
14351485

1436-
ComputeGraph setup_compute_graph(
1486+
BenchmarkGraph setup_compute_graph(
14371487
TestCase& test_case,
14381488
std::string op_name,
14391489
int op_invocations_per_execute) {
14401490
GraphConfig config;
14411491
config.enable_querypool = true;
1492+
// Pool sizing takes max(execute, prepack) * factor, so scaling the factor
1493+
// also over-reserves the prepack side (encoded once). Accepted: precise
1494+
// execute-only sizing would need runtime changes.
1495+
config.descriptor_pool_safety_factor *=
1496+
std::max(1, op_invocations_per_execute);
14421497
// Default-on (opt-out via TestCase::set_force_resize(false)): force every
1443-
// DynamicDispatchNode to run its resize function on each execute(),
1444-
// exercising the op's resize formula even when input shapes are unchanged.
1498+
// DynamicDispatchNode to run its resize function when execute_test_case
1499+
// runs propagate_resize() after prepack, exercising the op's resize formula
1500+
// even when input shapes are unchanged.
14451501
config.force_resize = test_case.get_force_resize();
1446-
ComputeGraph graph(config);
1502+
auto graph = std::make_unique<ComputeGraph>(config);
14471503

14481504
std::vector<ValueRef> input_values;
14491505

@@ -1452,17 +1508,17 @@ ComputeGraph setup_compute_graph(
14521508
const ValueSpec& input_spec = test_case.inputs()[i];
14531509

14541510
if (input_spec.is_none()) {
1455-
input_values.push_back(graph.add_none());
1511+
input_values.push_back(graph->add_none());
14561512
} else if (input_spec.is_float()) {
14571513
ValueRef input_value =
1458-
graph.add_scalar(static_cast<double>(input_spec.get_float_value()));
1514+
graph->add_scalar(static_cast<double>(input_spec.get_float_value()));
14591515
input_values.push_back(input_value);
14601516
} else if (input_spec.is_int()) {
14611517
ValueRef input_value =
1462-
graph.add_scalar(static_cast<int64_t>(input_spec.get_int_value()));
1518+
graph->add_scalar(static_cast<int64_t>(input_spec.get_int_value()));
14631519
input_values.push_back(input_value);
14641520
} else if (input_spec.is_bool()) {
1465-
ValueRef input_value = graph.add_scalar(input_spec.get_bool_value());
1521+
ValueRef input_value = graph->add_scalar(input_spec.get_bool_value());
14661522
input_values.push_back(input_value);
14671523
} else if (input_spec.is_int_list()) {
14681524
// Convert int32_t list to int64_t list for ComputeGraph
@@ -1472,20 +1528,20 @@ ComputeGraph setup_compute_graph(
14721528
for (int32_t val : int32_list) {
14731529
int64_list.push_back(static_cast<int64_t>(val));
14741530
}
1475-
ValueRef input_value = graph.add_scalar_list(std::move(int64_list));
1531+
ValueRef input_value = graph->add_scalar_list(std::move(int64_list));
14761532
input_values.push_back(input_value);
14771533
} else if (input_spec.is_string()) {
14781534
std::string str_copy = input_spec.get_string_value();
1479-
ValueRef input_value = graph.add_string(std::move(str_copy));
1535+
ValueRef input_value = graph->add_string(std::move(str_copy));
14801536
input_values.push_back(input_value);
14811537
} else if (input_spec.is_constant()) {
1482-
ValueRef input_value = graph.add_tensorref(
1538+
ValueRef input_value = graph->add_tensorref(
14831539
input_spec.get_tensor_sizes(),
14841540
input_spec.dtype,
14851541
input_spec.get_data_ptr());
14861542
input_values.push_back(input_value);
14871543
} else {
1488-
IOValueRef input_io = graph.add_input_tensor(
1544+
IOValueRef input_io = graph->add_input_tensor(
14891545
input_spec.get_tensor_sizes(),
14901546
input_spec.dtype,
14911547
input_spec.storage_type,
@@ -1505,7 +1561,7 @@ ComputeGraph setup_compute_graph(
15051561
}
15061562

15071563
// Create output tensor
1508-
ValueRef output_value = graph.add_tensor(
1564+
ValueRef output_value = graph->add_tensor(
15091565
output_spec.get_tensor_sizes(),
15101566
output_spec.dtype,
15111567
output_spec.storage_type,
@@ -1521,17 +1577,16 @@ ComputeGraph setup_compute_graph(
15211577
std::vector<ValueRef> op_args = input_values;
15221578
op_args.insert(op_args.end(), output_values.begin(), output_values.end());
15231579

1524-
// Invoke the op op_invocations_per_execute times to stack dispatches per
1525-
// graph.execute(). The output set_output_value() calls below still happen
1526-
// exactly once.
1527-
for (int i = 0; i < op_invocations_per_execute; ++i) {
1528-
opFn(graph, op_args);
1529-
}
1580+
// Nodes added before the op are staging uploads; nodes added after are
1581+
// staging downloads. Only the op's own nodes are repeated by benchmarks.
1582+
const size_t op_begin = graph->execute_nodes().size();
1583+
opFn(*graph, op_args);
1584+
const size_t op_end = graph->execute_nodes().size();
15301585

15311586
for (size_t i = 0; i < output_values.size(); ++i) {
1532-
graph.set_output_value(output_values[i]);
1587+
graph->set_output_value(output_values[i]);
15331588
}
1534-
return graph;
1589+
return {std::move(graph), {op_begin, op_end}};
15351590
}
15361591

15371592
// Test execution utilities
@@ -1549,16 +1604,20 @@ BenchmarkResult execute_test_case(
15491604
api::context()->initialize_querypool();
15501605
}
15511606

1552-
// Build the measurement graph with the requested chained_dispatches factor.
1553-
// The caller (typically execute_test_cases) decides what it should be —
1554-
// this function is a pure "run at the given chained_dispatches" primitive.
1555-
ComputeGraph graph = setup_compute_graph(
1607+
// Build the operator once. Benchmark repetition is encoded separately so
1608+
// persistent graph allocations are not duplicated.
1609+
BenchmarkGraph benchmark = setup_compute_graph(
15561610
test_case, test_case.operator_name(), chained_dispatches);
1611+
ComputeGraph& graph = *benchmark.graph;
15571612

15581613
// Prepare the graph
15591614
graph.prepare();
15601615
graph.prepack();
15611616

1617+
// Run resize functions once so force_resize exercises resize formulas even
1618+
// though the record/replay path below never calls propagate_resize().
1619+
graph.propagate_resize();
1620+
15621621
// Copy input data into the graph's staging buffers
15631622
size_t graph_input_idx = 0;
15641623
for (size_t i = 0; i < test_case.num_inputs(); ++i) {
@@ -1605,9 +1664,12 @@ BenchmarkResult execute_test_case(
16051664
++graph_input_idx;
16061665
}
16071666

1667+
RepeatedGraphExecutor graph_executor(
1668+
graph, chained_dispatches, benchmark.op_nodes);
1669+
16081670
// Warmup runs
16091671
for (int run = 0; run < warmup_runs; ++run) {
1610-
graph.execute();
1672+
graph_executor.execute();
16111673
}
16121674

16131675
// Benchmark runs - collect individual iteration timings
@@ -1619,7 +1681,7 @@ BenchmarkResult execute_test_case(
16191681
for (int run = 0; run < benchmark_runs; ++run) {
16201682
// Measure CPU time for each execute() call
16211683
auto cpu_start = std::chrono::high_resolution_clock::now();
1622-
graph.execute();
1684+
graph_executor.execute();
16231685
auto cpu_end = std::chrono::high_resolution_clock::now();
16241686

16251687
auto cpu_duration = std::chrono::duration_cast<std::chrono::microseconds>(

‎backends/vulkan/test/custom_ops/utils.h‎

Lines changed: 66 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -621,9 +621,9 @@ class TestCase {
621621
return shader_filter_;
622622
}
623623

624-
// Manual override for the number of times the op is dispatched per
625-
// graph.execute() (a.k.a. chained_dispatches). If > 0, the framework uses
626-
// this directly and skips the probe phase. 0 (the default) means adaptive
624+
// Manual override for the number of chained dispatches per measurement
625+
// iteration (a.k.a. chained_dispatches). If > 0, the framework uses this
626+
// directly and skips the probe phase. 0 (the default) means adaptive
627627
// (probe-then-scale).
628628
void set_op_invocations_per_execute(int n) {
629629
op_invocations_per_execute_ = n;
@@ -645,13 +645,15 @@ class TestCase {
645645

646646
// When true, the ComputeGraph built for this test case sets
647647
// GraphConfig::force_resize, so every DynamicDispatchNode runs its resize
648-
// function on each execute() even when no input shape changed. Because the
649-
// output is already allocated at the swept shape, the resize must recompute
650-
// the same shape from the current input — a wrong resize formula resizes the
651-
// output to a mismatched shape and surfaces as a test failure. Default true
652-
// (opt-out): every custom_ops test exercises its resize formulas across the
653-
// swept shapes. Call set_force_resize(false) for the rare op whose resize fn
654-
// is intentionally not shape-preserving under a fixed output allocation.
648+
// function once during measurement setup (execute_test_case runs
649+
// propagate_resize() after prepack) even when no input shape changed.
650+
// Because the output is already allocated at the swept shape, the resize
651+
// must recompute the same shape from the current input — a wrong resize
652+
// formula resizes the output to a mismatched shape and surfaces as a test
653+
// failure. Default true (opt-out): every custom_ops test exercises its
654+
// resize formulas across the swept shapes. Call set_force_resize(false) for
655+
// the rare op whose resize fn is intentionally not shape-preserving under a
656+
// fixed output allocation.
655657
void set_force_resize(bool force_resize) {
656658
force_resize_ = force_resize;
657659
}
@@ -940,9 +942,55 @@ int64_t default_flop_calculator(const TestCase& test_case);
940942

941943
using ReferenceComputeFunc = std::function<void(TestCase&)>;
942944

943-
// Runs a measurement at the given chained_dispatches factor (how many times
944-
// the op is stacked inside one graph.execute()). This is a primitive; the
945-
// probe-then-scale orchestration lives in execute_test_cases().
945+
// Half-open index range of the operator's own dispatch nodes within a
946+
// benchmark graph's execute_nodes(). Staging upload nodes precede it, staging
947+
// download nodes follow it.
948+
struct OpNodeRange {
949+
size_t begin = 0;
950+
size_t end = 0;
951+
};
952+
953+
// A benchmark graph plus the location of its repeatable operator nodes. The
954+
// graph is heap-held: ComputeGraph owns its Context and must never be moved
955+
// (a moved-from graph's destructor dereferences a null context).
956+
struct BenchmarkGraph {
957+
std::unique_ptr<ComputeGraph> graph;
958+
OpNodeRange op_nodes;
959+
};
960+
961+
// Benchmark-only executor that records a graph's execute nodes into a single
962+
// reusable command buffer, then replays it on every execute(). Staging uploads
963+
// are encoded once, operator nodes N times, staging downloads once, so each
964+
// iteration performs the same work as the old stacked-nodes layout (1 upload +
965+
// N ops + 1 download) and the per-invocation divisor is unchanged. Production
966+
// ComputeGraph::execute() behavior is unchanged.
967+
//
968+
// Notes for interpreting benchmark numbers:
969+
// - Submit granularity differs from stacking N distinct nodes: all encodings
970+
// live in one command buffer with one submit per iteration (the old path
971+
// could split across command buffers at the node-count threshold), so
972+
// per-dispatch times may shift systematically against older data.
973+
// - Resize functions run once via propagate_resize() in execute_test_case
974+
// before recording; replay itself never re-triggers resize.
975+
// - Repeated encodings share one node/dispatch id, so per-repetition
976+
// querypool attribution is unavailable (aggregation keys on kernel name).
977+
class RepeatedGraphExecutor final {
978+
public:
979+
RepeatedGraphExecutor(
980+
ComputeGraph& graph,
981+
int repetitions,
982+
OpNodeRange op_nodes);
983+
void execute();
984+
985+
private:
986+
ComputeGraph& graph_;
987+
std::unique_ptr<vkapi::CommandBuffer> command_;
988+
};
989+
990+
// Runs a measurement at the given chained_dispatches factor. The operator is
991+
// built once, then its execute nodes are encoded that many times into a
992+
// benchmark-only reusable command buffer. The probe-then-scale orchestration
993+
// lives in execute_test_cases().
946994
//
947995
// write_outputs controls whether the graph's staging output buffers are copied
948996
// back into test_case.outputs() at the end of the run. The probe path needs
@@ -1020,11 +1068,11 @@ void compute_weight_sums_4bit_grouped(
10201068
uint16_t float_to_half(float value);
10211069
float half_to_float(uint16_t half_val);
10221070

1023-
// Setup compute graph based on TestCase and operation name. The op function
1024-
// is invoked op_invocations_per_execute times so that one graph.execute()
1025-
// dispatches the op that many times (Google Benchmark-style stacking). The
1026-
// output set_output_value() calls still happen once at the end.
1027-
ComputeGraph setup_compute_graph(
1071+
// Setup compute graph based on TestCase and operation name. The op function is
1072+
// invoked once. op_invocations_per_execute is used only to reserve enough
1073+
// descriptor capacity for benchmark-only repeated command encoding. Returns
1074+
// the graph plus the range of the operator's own nodes for repeated encoding.
1075+
BenchmarkGraph setup_compute_graph(
10281076
TestCase& test_case,
10291077
std::string op_name,
10301078
int op_invocations_per_execute = 1);

‎backends/vulkan/test/custom_ops/utils_test.cpp‎

Lines changed: 53 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,32 @@
1111
namespace executorch::vulkan::prototyping {
1212
namespace {
1313

14+
int operator_build_count = 0;
15+
int encode_count = 0;
16+
17+
class CountingEncodeNode final : public ExecuteNode {
18+
public:
19+
explicit CountingEncodeNode(int& count) : count_(count) {}
20+
void encode(ComputeGraph* graph) override {
21+
(void)graph;
22+
++count_;
23+
}
24+
25+
private:
26+
int& count_;
27+
};
28+
29+
void counting_operator(ComputeGraph& graph, const std::vector<ValueRef>& args) {
30+
(void)args;
31+
++operator_build_count;
32+
graph.execute_nodes().emplace_back(
33+
std::make_unique<CountingEncodeNode>(encode_count));
34+
}
35+
36+
REGISTER_OPERATORS {
37+
VK_REGISTER_OP(test_etvk.counting_operator.default, counting_operator);
38+
}
39+
1440
TEST(ValueSpecTest, SetConstant_DoesNotMaterializeTensorData) {
1541
ValueSpec value(
1642
{16},
@@ -170,5 +196,32 @@ TEST(ValueSpecTest, ShareReferenceFrom_IgnoresNonTensorSpecs) {
170196
EXPECT_EQ(tensor.get_ref_float_data(), expected);
171197
}
172198

199+
TEST(BenchmarkGraphTest, ChainedDispatches_BuildOperatorOnce) {
200+
if (!vkcompute::api::available()) {
201+
return;
202+
}
203+
204+
TestCase test_case;
205+
operator_build_count = 0;
206+
encode_count = 0;
207+
208+
BenchmarkGraph benchmark = setup_compute_graph(
209+
test_case,
210+
"test_etvk.counting_operator.default",
211+
/*op_invocations_per_execute=*/8);
212+
ComputeGraph& graph = *benchmark.graph;
213+
214+
EXPECT_EQ(operator_build_count, 1);
215+
ASSERT_EQ(graph.execute_nodes().size(), 1u);
216+
EXPECT_EQ(benchmark.op_nodes.begin, 0u);
217+
EXPECT_EQ(benchmark.op_nodes.end, 1u);
218+
219+
// The replay path must encode the operator node once per repetition.
220+
RepeatedGraphExecutor graph_executor(
221+
graph, /*repetitions=*/8, benchmark.op_nodes);
222+
EXPECT_EQ(encode_count, 8);
223+
graph_executor.execute();
224+
}
225+
173226
} // namespace
174227
} // namespace executorch::vulkan::prototyping

0 commit comments

Comments
 (0)