Skip to content

Commit 71b4c02

Browse files
HylfrdGlavo
andauthored
解决导出 Modrinth 整合包因大文件导致的 Out Of Memory (#6631)
Co-authored-by: Glavo <zjx001202@gmail.com>
1 parent 45f49d0 commit 71b4c02

5 files changed

Lines changed: 579 additions & 67 deletions

File tree

HMCLCore/src/main/java/org/jackhuang/hmcl/addon/repository/CurseForgeRemoteAddonRepository.java

Lines changed: 58 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -33,17 +33,19 @@
3333
import org.jetbrains.annotations.NotNull;
3434
import org.jetbrains.annotations.Nullable;
3535

36-
import java.io.ByteArrayOutputStream;
3736
import java.io.IOException;
38-
import java.io.InputStream;
3937
import java.net.URI;
38+
import java.nio.ByteBuffer;
39+
import java.nio.channels.SeekableByteChannel;
4040
import java.nio.file.Files;
4141
import java.nio.file.Path;
42+
import java.nio.file.StandardOpenOption;
4243
import java.time.Instant;
4344
import java.util.*;
4445
import java.util.concurrent.Semaphore;
4546
import java.util.stream.Collectors;
4647
import java.util.stream.Stream;
48+
import java.util.zip.Checksum;
4749

4850
import static org.jackhuang.hmcl.util.Lang.mapOf;
4951
import static org.jackhuang.hmcl.util.Pair.pair;
@@ -205,23 +207,65 @@ public SearchResult search(DownloadProvider downloadProvider, String gameVersion
205207
}
206208
}
207209

208-
@Override
209-
public Optional<RemoteAddon.Version> getRemoteVersionByLocalFile(Path file) throws IOException {
210-
ByteArrayOutputStream baos = new ByteArrayOutputStream();
211-
try (InputStream stream = Files.newInputStream(file)) {
212-
byte[] buf = new byte[1024];
213-
int len;
214-
while ((len = stream.read(buf, 0, buf.length)) != -1) {
210+
/// Calculates the CurseForge fingerprint without retaining the filtered file in memory.
211+
static long calculateFingerprint(Path file) throws IOException {
212+
try (SeekableByteChannel channel = Files.newByteChannel(file, StandardOpenOption.READ)) {
213+
long startPosition = channel.position();
214+
215+
byte[] bufferArray = new byte[1024 * 1024];
216+
ByteBuffer buffer = ByteBuffer.wrap(bufferArray);
217+
218+
long filteredLength = 0;
219+
while (channel.read(buffer) > 0) {
220+
int len = buffer.position();
215221
for (int i = 0; i < len; i++) {
216-
byte b = buf[i];
222+
byte b = bufferArray[i];
217223
if (b != 0x9 && b != 0xa && b != 0xd && b != 0x20) {
218-
baos.write(b);
224+
filteredLength++;
225+
}
226+
}
227+
buffer.clear();
228+
}
229+
230+
channel.position(startPosition);
231+
232+
Checksum hasher = MurmurHash2.hash32(filteredLength, 1);
233+
while (channel.read(buffer) > 0) {
234+
int len = buffer.position();
235+
236+
int pos = 0;
237+
while (pos < len) {
238+
byte b = bufferArray[pos];
239+
if (b == 0x9 || b == 0xa || b == 0xd || b == 0x20) {
240+
break;
241+
}
242+
pos++;
243+
}
244+
245+
if (pos < len) {
246+
int pos2 = pos + 1;
247+
while (pos2 < len) {
248+
byte b = bufferArray[pos2];
249+
if (b != 0x9 && b != 0xa && b != 0xd && b != 0x20) {
250+
bufferArray[pos++] = b;
251+
}
252+
pos2++;
219253
}
220254
}
255+
256+
hasher.update(bufferArray, 0, pos);
257+
buffer.clear();
221258
}
259+
return hasher.getValue();
260+
} catch (IllegalArgumentException | IllegalStateException e) {
261+
throw new IOException(e);
222262
}
263+
}
223264

224-
long hash = Integer.toUnsignedLong(MurmurHash2.hash32(baos.toByteArray(), baos.size(), 1));
265+
/// Finds the remote CurseForge version matching a local file.
266+
@Override
267+
public Optional<RemoteAddon.Version> getRemoteVersionByLocalFile(Path file) throws IOException {
268+
long hash = calculateFingerprint(file);
225269
if (hash == 811513880) { // Workaround for https://github.com/HMCL-dev/HMCL/issues/4597
226270
return Optional.empty();
227271
}
@@ -309,7 +353,8 @@ public String getAddonChangelog(DownloadProvider downloadProvider, String addonI
309353
case SECTION_ADDONS -> "mc-addons";
310354
case SECTION_CUSTOMIZATION -> "customization";
311355
case SECTION_SHADER -> "shaders";
312-
default -> throw new IllegalArgumentException("Unsupported CurseForge class id [%d]".formatted(classId));
356+
default ->
357+
throw new IllegalArgumentException("Unsupported CurseForge class id [%d]".formatted(classId));
313358
};
314359
return "%s/minecraft/%s/%s/files/%s".formatted(BASE, clazz, addon.slug(), version.versionId());
315360
} finally {

HMCLCore/src/main/java/org/jackhuang/hmcl/util/MurmurHash2.java

Lines changed: 176 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,11 @@
1717
*/
1818
package org.jackhuang.hmcl.util;
1919

20+
import org.jetbrains.annotations.NotNullByDefault;
21+
2022
import java.nio.charset.StandardCharsets;
23+
import java.util.Objects;
24+
import java.util.zip.Checksum;
2125

2226
/**
2327
* Implementation of the MurmurHash2 32-bit and 64-bit hash functions.
@@ -48,6 +52,7 @@
4852
* Original MurmurHash2 c++ code</a>
4953
* @since 1.13
5054
*/
55+
@NotNullByDefault
5156
public final class MurmurHash2 {
5257

5358
// Constants for 32-bit variant
@@ -64,6 +69,177 @@ public final class MurmurHash2 {
6469
private MurmurHash2() {
6570
}
6671

72+
/// Creates a streaming MurmurHash2 32-bit checksum for exactly `length` bytes.
73+
///
74+
/// The length is incorporated into the initial hash state using its low 32 bits. Calls to
75+
/// [Checksum#update(int)] and [Checksum#update(byte[], int, int)] may divide the input at
76+
/// arbitrary byte boundaries. [Checksum#getValue()] returns the hash as an unsigned 32-bit
77+
/// value represented by a `long`, and does not change the checksum state.
78+
///
79+
/// The returned checksum verifies the number of supplied bytes when `getValue()` is called.
80+
/// If too few bytes have been supplied, the caller may continue updating the checksum and call
81+
/// `getValue()` again. Once too many bytes have been supplied, [Checksum#reset()] must be called
82+
/// before a value can be obtained. Resetting retains the configured length and seed. The
83+
/// returned checksum is mutable and is not safe for concurrent use.
84+
///
85+
/// @param length the exact number of input bytes
86+
/// @param seed the initial seed value
87+
/// @return a new checksum initialized with the specified length and seed
88+
/// @throws IllegalArgumentException if `length` is negative
89+
public static Checksum hash32(final long length, final int seed) {
90+
if (length < 0) {
91+
throw new IllegalArgumentException("length must not be negative: " + length);
92+
}
93+
return new Hash32Checksum(length, seed);
94+
}
95+
96+
/// Computes a MurmurHash2 32-bit value from updates whose total length is known in advance.
97+
private static final class Hash32Checksum implements Checksum {
98+
/// The exact number of bytes required before the hash can be obtained.
99+
private final long expectedLength;
100+
101+
/// The hash state restored by [#reset()].
102+
private final int initialHash;
103+
104+
/// The hash state after all complete four-byte blocks received so far.
105+
private int hash;
106+
107+
/// The number of bytes counted before [#inputLengthExceeded] becomes `true`.
108+
private long inputLength;
109+
110+
/// Whether more than [#expectedLength] bytes have been received.
111+
private boolean inputLengthExceeded;
112+
113+
/// Up to three unprocessed bytes packed in little-endian order.
114+
private int tail;
115+
116+
/// The number of bytes currently stored in [#tail].
117+
private int tailLength;
118+
119+
/// Creates a checksum with a precomputed initial hash state.
120+
///
121+
/// @param expectedLength the exact number of input bytes
122+
/// @param seed the initial seed value
123+
private Hash32Checksum(long expectedLength, int seed) {
124+
this.expectedLength = expectedLength;
125+
this.initialHash = seed ^ (int) expectedLength;
126+
this.hash = initialHash;
127+
}
128+
129+
/// Incorporates the low eight bits of `value` into this checksum.
130+
///
131+
/// @param value the value whose low eight bits are incorporated
132+
@Override
133+
public void update(int value) {
134+
addInputLength(1);
135+
appendByte(value);
136+
}
137+
138+
/// Incorporates `length` bytes beginning at `offset` into this checksum.
139+
///
140+
/// @param data the array containing the input bytes
141+
/// @param offset the offset of the first input byte
142+
/// @param length the number of bytes to incorporate
143+
@Override
144+
public void update(byte[] data, int offset, int length) {
145+
Objects.checkFromIndexSize(offset, length, data.length);
146+
addInputLength(length);
147+
148+
int index = offset;
149+
final int end = offset + length;
150+
151+
while (tailLength != 0 && index < end) {
152+
appendByte(data[index++]);
153+
}
154+
155+
while (index <= end - Integer.BYTES) {
156+
mixBlock(ByteArray.getIntLE(data, index));
157+
index += Integer.BYTES;
158+
}
159+
160+
while (index < end) {
161+
appendByte(data[index++]);
162+
}
163+
}
164+
165+
/// Returns the MurmurHash2 value after verifying the exact input length.
166+
///
167+
/// @return the unsigned 32-bit hash value represented by a `long`
168+
/// @throws IllegalStateException if the number of supplied bytes differs from the expected
169+
/// length
170+
@Override
171+
public long getValue() {
172+
if (inputLengthExceeded) {
173+
throw new IllegalStateException(
174+
"Expected " + expectedLength + " bytes, but received more than expected");
175+
}
176+
if (inputLength != expectedLength) {
177+
throw new IllegalStateException(
178+
"Expected " + expectedLength + " bytes, but received " + inputLength);
179+
}
180+
181+
int result = hash;
182+
if (tailLength != 0) {
183+
result ^= tail;
184+
result *= M32;
185+
}
186+
187+
result ^= result >>> 13;
188+
result *= M32;
189+
result ^= result >>> 15;
190+
return Integer.toUnsignedLong(result);
191+
}
192+
193+
/// Restores this checksum to its initial state while retaining its expected length and seed.
194+
@Override
195+
public void reset() {
196+
hash = initialHash;
197+
inputLength = 0;
198+
inputLengthExceeded = false;
199+
tail = 0;
200+
tailLength = 0;
201+
}
202+
203+
/// Records that `length` more input bytes have been supplied.
204+
///
205+
/// @param length the non-negative number of additional bytes
206+
private void addInputLength(int length) {
207+
if (inputLengthExceeded) {
208+
return;
209+
}
210+
if (length > expectedLength - inputLength) {
211+
inputLengthExceeded = true;
212+
} else {
213+
inputLength += length;
214+
}
215+
}
216+
217+
/// Buffers one byte and mixes the resulting block when four bytes are available.
218+
///
219+
/// @param value the value whose low eight bits are appended
220+
private void appendByte(int value) {
221+
tail |= (value & 0xff) << (tailLength * Byte.SIZE);
222+
tailLength++;
223+
if (tailLength == Integer.BYTES) {
224+
mixBlock(tail);
225+
tail = 0;
226+
tailLength = 0;
227+
}
228+
}
229+
230+
/// Mixes one little-endian four-byte block into the current hash state.
231+
///
232+
/// @param block the block to mix
233+
private void mixBlock(int block) {
234+
int mixedBlock = block;
235+
mixedBlock *= M32;
236+
mixedBlock ^= mixedBlock >>> R32;
237+
mixedBlock *= M32;
238+
hash *= M32;
239+
hash ^= mixedBlock;
240+
}
241+
}
242+
67243
/**
68244
* Generates a 32-bit hash from byte array with the given length and seed.
69245
*

HMCLCore/src/test/java/org/jackhuang/hmcl/addon/curse/CurseForgeRemoteAddonRepositoryTest.java

Lines changed: 0 additions & 54 deletions
This file was deleted.

0 commit comments

Comments
 (0)