Skip to content

Commit b4d3cac

Browse files
committed
Add GGUF model memory estimator
1 parent 34cec35 commit b4d3cac

4 files changed

Lines changed: 427 additions & 2 deletions

File tree

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
package tridefender.llama.snapdragon.model
2+
3+
data class ModelMemoryEstimate(
4+
val modelBytes: Long,
5+
val kvCacheBytes: Long,
6+
val overheadBytes: Long,
7+
val totalBytes: Long,
8+
val freeBytes: Long,
9+
val remainingBytes: Long,
10+
val fitState: MemoryFitState,
11+
val architecture: String? = null,
12+
val contextLength: Long? = null,
13+
val blockCount: Long? = null,
14+
val embeddingLength: Long? = null,
15+
val headCount: Long? = null,
16+
val headCountKv: Long? = null,
17+
val keyLength: Long? = null,
18+
val valueLength: Long? = null,
19+
val fileType: Long? = null,
20+
val quantizationVersion: Long? = null
21+
)
22+
23+
enum class MemoryFitState {
24+
WONT_FIT,
25+
TIGHT_FIT,
26+
GOOD_FIT
27+
}
Lines changed: 199 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,199 @@
1+
package tridefender.llama.snapdragon.util
2+
3+
import android.util.Log
4+
import java.io.File
5+
import java.io.RandomAccessFile
6+
import java.nio.ByteBuffer
7+
import java.nio.ByteOrder
8+
9+
data class GgufMetadata(
10+
val architecture: String? = null,
11+
val contextLength: Long? = null,
12+
val blockCount: Long? = null,
13+
val embeddingLength: Long? = null,
14+
val headCount: Long? = null,
15+
val headCountKv: Long? = null,
16+
val keyLength: Long? = null,
17+
val valueLength: Long? = null,
18+
val fileType: Long? = null,
19+
val quantizationVersion: Long? = null
20+
)
21+
22+
object GgufMetadataReader {
23+
private const val TAG = "GgufMetadataReader"
24+
private const val MAGIC = 0x46554747
25+
private const val MAX_STRING_BYTES = 1 shl 20
26+
27+
fun read(file: File): GgufMetadata? {
28+
if (!file.isFile || !file.canRead()) return null
29+
return runCatching {
30+
RandomAccessFile(file, "r").use { raf ->
31+
if (raf.readIntLe() != MAGIC) return null
32+
val version = raf.readIntLe()
33+
if (version !in 1..3) return null
34+
35+
raf.readLongLe() // tensor count
36+
val kvCount = raf.readLongLe()
37+
38+
var architecture: String? = null
39+
var contextLength: Long? = null
40+
var blockCount: Long? = null
41+
var embeddingLength: Long? = null
42+
var headCount: Long? = null
43+
var headCountKv: Long? = null
44+
var keyLength: Long? = null
45+
var valueLength: Long? = null
46+
var fileType: Long? = null
47+
var quantizationVersion: Long? = null
48+
49+
repeat(kvCount.toBoundedInt()) {
50+
val key = readString(raf) ?: return null
51+
val type = raf.readIntLe()
52+
when (key) {
53+
"general.architecture" -> architecture = readStringValue(raf, type)
54+
"general.quantization_version" -> quantizationVersion = readUIntValue(raf, type)
55+
"general.file_type" -> fileType = readUIntValue(raf, type)
56+
"llama.context_length",
57+
"mpt.context_length",
58+
"gptneox.context_length",
59+
"gptj.context_length",
60+
"gpt2.context_length",
61+
"bloom.context_length",
62+
"falcon.context_length",
63+
"mamba.context_length",
64+
"rwkv.context_length",
65+
"whisper.encoder.context_length",
66+
"whisper.decoder.context_length",
67+
"llm.context_length" -> contextLength = readUIntValue(raf, type)
68+
"llama.block_count",
69+
"mpt.block_count",
70+
"gptneox.block_count",
71+
"gptj.block_count",
72+
"gpt2.block_count",
73+
"bloom.block_count",
74+
"falcon.block_count",
75+
"mamba.block_count",
76+
"rwkv.block_count",
77+
"whisper.encoder.block_count",
78+
"whisper.decoder.block_count",
79+
"llm.block_count" -> blockCount = readUIntValue(raf, type)
80+
"llama.embedding_length",
81+
"mpt.embedding_length",
82+
"gptneox.embedding_length",
83+
"gptj.embedding_length",
84+
"gpt2.embedding_length",
85+
"bloom.embedding_length",
86+
"falcon.embedding_length",
87+
"mamba.embedding_length",
88+
"rwkv.embedding_length",
89+
"whisper.encoder.embedding_length",
90+
"whisper.decoder.embedding_length",
91+
"llm.embedding_length" -> embeddingLength = readUIntValue(raf, type)
92+
"llama.attention.head_count",
93+
"mpt.attention.head_count",
94+
"gptneox.attention.head_count",
95+
"gptj.attention.head_count",
96+
"gpt2.attention.head_count",
97+
"bloom.attention.head_count",
98+
"falcon.attention.head_count",
99+
"whisper.encoder.attention.head_count",
100+
"whisper.decoder.attention.head_count",
101+
"llm.attention.head_count" -> headCount = readUIntValue(raf, type)
102+
"llama.attention.head_count_kv",
103+
"falcon.attention.head_count_kv",
104+
"llm.attention.head_count_kv" -> headCountKv = readUIntValue(raf, type)
105+
"llm.attention.key_length" -> keyLength = readUIntValue(raf, type)
106+
"llm.attention.value_length" -> valueLength = readUIntValue(raf, type)
107+
else -> skipValue(raf, type)
108+
}
109+
}
110+
111+
GgufMetadata(
112+
architecture = architecture,
113+
contextLength = contextLength,
114+
blockCount = blockCount,
115+
embeddingLength = embeddingLength,
116+
headCount = headCount,
117+
headCountKv = headCountKv,
118+
keyLength = keyLength,
119+
valueLength = valueLength,
120+
fileType = fileType,
121+
quantizationVersion = quantizationVersion
122+
)
123+
}
124+
}.getOrElse { error ->
125+
Log.w(TAG, "Failed to read GGUF metadata from ${file.absolutePath}", error)
126+
null
127+
}
128+
}
129+
130+
private fun readUIntValue(raf: RandomAccessFile, type: Int): Long? {
131+
return when (type) {
132+
0 -> raf.readByteUnsigned().toLong()
133+
2 -> raf.readShortLe().toLong()
134+
4 -> raf.readIntLeUnsigned()
135+
10 -> raf.readLongLeUnsigned()
136+
else -> {
137+
skipValue(raf, type)
138+
null
139+
}
140+
}
141+
}
142+
143+
private fun readStringValue(raf: RandomAccessFile, type: Int): String? {
144+
return when (type) {
145+
8 -> readString(raf)
146+
else -> {
147+
skipValue(raf, type)
148+
null
149+
}
150+
}
151+
}
152+
153+
private fun skipValue(raf: RandomAccessFile, type: Int) {
154+
when (type) {
155+
0, 1, 7 -> raf.skipBytesCompat(1)
156+
2, 3 -> raf.skipBytesCompat(2)
157+
4, 5, 6 -> raf.skipBytesCompat(4)
158+
8 -> readString(raf)
159+
9 -> {
160+
val arrayType = raf.readIntLe()
161+
val length = raf.readLongLe().toBoundedInt()
162+
repeat(length) { skipValue(raf, arrayType) }
163+
}
164+
10, 11, 12 -> raf.skipBytesCompat(8)
165+
}
166+
}
167+
168+
private fun readString(raf: RandomAccessFile): String? {
169+
val rawLength = raf.readLongLe()
170+
if (rawLength < 0 || rawLength > MAX_STRING_BYTES) return null
171+
val length = rawLength.toInt()
172+
val bytes = ByteArray(length)
173+
raf.readFully(bytes)
174+
return bytes.toString(Charsets.UTF_8)
175+
}
176+
177+
private fun RandomAccessFile.skipBytesCompat(length: Int) {
178+
if (length <= 0) return
179+
seek(filePointer + length)
180+
}
181+
182+
private fun RandomAccessFile.readByteUnsigned(): Int = readUnsignedByte()
183+
private fun RandomAccessFile.readShortLe(): Int = ByteBuffer.wrap(readBytes(2)).order(ByteOrder.LITTLE_ENDIAN).short.toInt()
184+
private fun RandomAccessFile.readIntLe(): Int = ByteBuffer.wrap(readBytes(4)).order(ByteOrder.LITTLE_ENDIAN).int
185+
private fun RandomAccessFile.readIntLeUnsigned(): Long = readIntLe().toLong() and 0xffffffffL
186+
private fun RandomAccessFile.readLongLe(): Long = ByteBuffer.wrap(readBytes(8)).order(ByteOrder.LITTLE_ENDIAN).long
187+
private fun RandomAccessFile.readLongLeUnsigned(): Long = readLongLe()
188+
private fun RandomAccessFile.readBytes(count: Int): ByteArray {
189+
val bytes = ByteArray(count)
190+
readFully(bytes)
191+
return bytes
192+
}
193+
194+
private fun Long.toBoundedInt(): Int = when {
195+
this <= 0L -> 0
196+
this >= Int.MAX_VALUE.toLong() -> Int.MAX_VALUE
197+
else -> this.toInt()
198+
}
199+
}
Lines changed: 83 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,83 @@
1+
package tridefender.llama.snapdragon.util
2+
3+
import tridefender.llama.snapdragon.model.CacheType
4+
import tridefender.llama.snapdragon.model.MemoryFitState
5+
import tridefender.llama.snapdragon.model.ModelMemoryEstimate
6+
import kotlin.math.max
7+
8+
object ModelMemoryCalculator {
9+
private const val ONE_GIB = 1L shl 30
10+
private const val DEFAULT_OVERHEAD_BYTES = 256L shl 20
11+
12+
fun estimate(
13+
fileSizeBytes: Long,
14+
metadata: GgufMetadata?,
15+
contextSize: Int,
16+
cacheTypeK: CacheType,
17+
cacheTypeV: CacheType,
18+
freeBytes: Long
19+
): ModelMemoryEstimate {
20+
val architecture = metadata?.architecture?.lowercase()
21+
val contextLength = metadata?.contextLength
22+
val blockCount = metadata?.blockCount
23+
val embeddingLength = metadata?.embeddingLength
24+
val headCount = metadata?.headCount
25+
val headCountKv = metadata?.headCountKv?.takeIf { it > 0 } ?: headCount
26+
val keyLength = metadata?.keyLength ?: if (embeddingLength != null && headCount != null && headCount > 0) {
27+
max(1L, embeddingLength / headCount)
28+
} else {
29+
null
30+
}
31+
val valueLength = metadata?.valueLength ?: keyLength
32+
33+
val layers = blockCount ?: 0L
34+
val kvTokens = maxOf(contextSize, 1)
35+
val kvCacheBytes = if (layers > 0 && headCountKv != null && keyLength != null && valueLength != null) {
36+
val perToken = layers * headCountKv * (keyLength * bytesPerElement(cacheTypeK) + valueLength * bytesPerElement(cacheTypeV))
37+
perToken * kvTokens
38+
} else {
39+
0L
40+
}
41+
42+
val overheadBytes = DEFAULT_OVERHEAD_BYTES + if (architecture == "llama") 64L shl 20 else 0L
43+
val totalBytes = fileSizeBytes + kvCacheBytes + overheadBytes
44+
val remainingBytes = freeBytes - totalBytes
45+
val fitState = when {
46+
remainingBytes < 0 -> MemoryFitState.WONT_FIT
47+
remainingBytes < ONE_GIB -> MemoryFitState.TIGHT_FIT
48+
else -> MemoryFitState.GOOD_FIT
49+
}
50+
51+
return ModelMemoryEstimate(
52+
modelBytes = fileSizeBytes,
53+
kvCacheBytes = kvCacheBytes,
54+
overheadBytes = overheadBytes,
55+
totalBytes = totalBytes,
56+
freeBytes = freeBytes,
57+
remainingBytes = remainingBytes,
58+
fitState = fitState,
59+
architecture = metadata?.architecture,
60+
contextLength = contextLength,
61+
blockCount = blockCount,
62+
embeddingLength = embeddingLength,
63+
headCount = headCount,
64+
headCountKv = headCountKv,
65+
keyLength = keyLength,
66+
valueLength = valueLength,
67+
fileType = metadata?.fileType,
68+
quantizationVersion = metadata?.quantizationVersion
69+
)
70+
}
71+
72+
private fun bytesPerElement(type: CacheType): Long = when (type) {
73+
CacheType.F32 -> 4L
74+
CacheType.F16 -> 2L
75+
CacheType.BF16 -> 2L
76+
CacheType.Q8_0 -> 1L
77+
CacheType.Q4_0 -> 1L
78+
CacheType.Q4_1 -> 1L
79+
CacheType.Q5_0 -> 1L
80+
CacheType.Q5_1 -> 1L
81+
CacheType.IQ4_NL -> 1L
82+
}
83+
}

0 commit comments

Comments
 (0)