Skip to content

Commit 92a8ba9

Browse files
authored
Qualcomm AI Engine Direct - Update value cache for all heads in lookahead mode (pytorch#16362)
### Summary - As mentioned in the title, it currently only updates once, but it should update for each num_heads. - Fixed soc_model of SM8350 <img width="730" height="241" alt="image" src="https://github.com/user-attachments/assets/372a6ff1-4f6b-44f8-91a3-0852758a6e36" /> ### Test Plan ``` python examples/qualcomm/oss_scripts/llama/llama.py -b build-android -H <host> -s <device id> -m SM8750 --temperature 0 --checkpoint consolidated.00.pth --params params.json --tokenizer_model tokenizer.model --decoder_model llama3_2-1b_instruct --model_mode lookahead --prefill_ar_len 64 --ngram 9 --window 4 --gcap 4 --max_seq_len 1024 --prompt "I would like to learn python, could you teach me with a simple example?" --artifact test_llama321b_lookahead944/ ```
1 parent 6e8b206 commit 92a8ba9

3 files changed

Lines changed: 15 additions & 11 deletions

File tree

‎backends/qualcomm/serialization/qc_compiler_spec.fbs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -34,7 +34,7 @@ table HtpInfo {
3434
enum QcomChipset: int {
3535
UNKNOWN_SM = 0,
3636
SA8295 = 39,
37-
SM8350 = 35,
37+
SM8350 = 30,
3838
SM8450 = 36,
3939
SM8475 = 42,
4040
SM8550 = 43,

‎backends/qualcomm/serialization/qc_schema.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -40,7 +40,7 @@ class HtpInfo:
4040
class QcomChipset(IntEnum):
4141
UNKNOWN_SM = 0
4242
SA8295 = 39 # v68
43-
SM8350 = 35 # v68
43+
SM8350 = 30 # v68
4444
SM8450 = 36 # v69
4545
SM8475 = 42 # v69
4646
SM8550 = 43 # v73

‎examples/qualcomm/oss_scripts/llama/runner/kv_manager.cpp‎

Lines changed: 13 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -348,16 +348,20 @@ void KVManager<T>::update_value(
348348
}
349349
} else {
350350
int32_t update_times = n_update;
351-
auto wp = write_ptr, rp = read_ptr;
352-
for (auto sel : selected) {
353-
if (sel) {
354-
std::memcpy(wp, rp, metadata_.head_dim * sizeof(T));
355-
wp += metadata_.head_dim;
356-
update_times--;
357-
if (update_times == 0)
358-
break;
351+
for (int i = 0; i < n_iter; ++i) {
352+
auto wp = write_ptr, rp = read_ptr;
353+
for (auto sel : selected) {
354+
if (sel) {
355+
std::memcpy(wp, rp, metadata_.head_dim * sizeof(T));
356+
wp += metadata_.head_dim;
357+
update_times--;
358+
if (update_times == 0)
359+
break;
360+
}
361+
rp += metadata_.head_dim;
359362
}
360-
rp += metadata_.head_dim;
363+
write_ptr += iter_size;
364+
read_ptr += out_size;
361365
}
362366
}
363367
}

0 commit comments

Comments
 (0)