Skip to content

Commit ad33804

Browse files
authored
Add prefill and decode token/s rates to LLM runner stats JSON (pytorch#18501)
Add prefill and decode token/s rates to LLM runner stats JSON
1 parent 3a64775 commit ad33804

1 file changed

Lines changed: 19 additions & 1 deletion

File tree

‎extension/llm/runner/stats.h‎

Lines changed: 19 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -97,7 +97,25 @@ struct ET_EXPERIMENTAL Stats {
9797

9898
inline std::string stats_to_json_string(const Stats& stats) {
9999
std::stringstream ss;
100-
ss << "{\"prompt_tokens\":" << stats.num_prompt_tokens << ","
100+
ss << "{";
101+
// Prefill and decode rates as first two metrics
102+
double prefill_ms =
103+
(double)(stats.prompt_eval_end_ms - stats.inference_start_ms);
104+
if (prefill_ms > 0) {
105+
ss << "\"prefill_token_per_sec\":"
106+
<< (stats.num_prompt_tokens / prefill_ms *
107+
stats.SCALING_FACTOR_UNITS_PER_SECOND)
108+
<< ",";
109+
}
110+
double decode_ms =
111+
(double)(stats.inference_end_ms - stats.prompt_eval_end_ms);
112+
if (decode_ms > 0) {
113+
ss << "\"decode_token_per_sec\":"
114+
<< (stats.num_generated_tokens / decode_ms *
115+
stats.SCALING_FACTOR_UNITS_PER_SECOND)
116+
<< ",";
117+
}
118+
ss << "\"prompt_tokens\":" << stats.num_prompt_tokens << ","
101119
<< "\"generated_tokens\":" << stats.num_generated_tokens << ","
102120
<< "\"model_load_start_ms\":" << stats.model_load_start_ms << ","
103121
<< "\"model_load_end_ms\":" << stats.model_load_end_ms << ","

0 commit comments

Comments
 (0)