Skip to content

Commit e7c65dd

Browse files
authored
change, tests, and changelog (#51105)
1 parent 9cb7ad2 commit e7c65dd

3 files changed

Lines changed: 110 additions & 27 deletions

File tree

‎sdk/core/azure-core/CHANGELOG.md‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,10 @@
1414

1515
- Fixed `RetryPolicy` synchronous retry backoff not restoring the thread's interrupt status after catching
1616
`InterruptedException`, which caused callers to silently lose the interruption signal.
17+
- Fixed `FluxUtil.toFluxByteBuffer` emitting each partial read backed by an array of the full chunk size, which caused
18+
consumers that retain the emitted buffers, such as uploads from an `InputStream` returning small reads, to use many
19+
times more memory than the data read and fail with `OutOfMemoryError`. Emitted buffers are now sized to the data
20+
read.
1721

1822
### Other Changes
1923

‎sdk/core/azure-core/src/main/java/com/azure/core/util/FluxUtil.java‎

Lines changed: 32 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,7 @@
4040
import java.nio.channels.FileChannel;
4141
import java.nio.channels.WritableByteChannel;
4242
import java.nio.file.StandardOpenOption;
43+
import java.util.Arrays;
4344
import java.util.Collections;
4445
import java.util.Map;
4546
import java.util.Objects;
@@ -320,42 +321,46 @@ public static Flux<ByteBuffer> toFluxByteBuffer(InputStream inputStream, int chu
320321
});
321322
}
322323

323-
return Flux.<ByteBuffer, InputStream>generate(() -> inputStream, (stream, sink) -> {
324+
return Flux.defer(() -> {
325+
// Reads go into one array reused for the subscription and each emission copies out only the bytes read,
326+
// so consumers retaining the emitted buffers don't retain a chunkSize array for every partial read.
324327
byte[] buffer = new byte[chunkSize];
325328

326-
try {
327-
int readCount;
328-
do {
329-
readCount = stream.read(buffer, 0, chunkSize);
330-
} while (readCount == 0);
331-
332-
if (readCount == -1) {
333-
sink.complete();
334-
return stream;
335-
}
329+
return Flux.<ByteBuffer, InputStream>generate(() -> inputStream, (stream, sink) -> {
330+
try {
331+
int readCount;
332+
do {
333+
readCount = stream.read(buffer, 0, chunkSize);
334+
} while (readCount == 0);
336335

337-
int offset = readCount;
338-
while (offset < chunkSize) {
339-
// Availability is only a batching hint; zero ends coalescing, not the stream.
340-
int available = stream.available();
341-
if (available <= 0) {
342-
break;
336+
if (readCount == -1) {
337+
sink.complete();
338+
return stream;
343339
}
344340

345-
readCount = stream.read(buffer, offset, Math.min(available, chunkSize - offset));
346-
if (readCount <= 0) {
347-
break;
341+
int offset = readCount;
342+
while (offset < chunkSize) {
343+
// Availability is only a batching hint; zero ends coalescing, not the stream.
344+
int available = stream.available();
345+
if (available <= 0) {
346+
break;
347+
}
348+
349+
readCount = stream.read(buffer, offset, Math.min(available, chunkSize - offset));
350+
if (readCount <= 0) {
351+
break;
352+
}
353+
354+
offset += readCount;
348355
}
349356

350-
offset += readCount;
357+
sink.next(ByteBuffer.wrap(Arrays.copyOf(buffer, offset)));
358+
} catch (IOException ex) {
359+
sink.error(ex);
351360
}
352361

353-
sink.next(ByteBuffer.wrap(buffer, 0, offset));
354-
} catch (IOException ex) {
355-
sink.error(ex);
356-
}
357-
358-
return stream;
362+
return stream;
363+
});
359364
}).filter(ByteBuffer::hasRemaining);
360365
}
361366

‎sdk/core/azure-core/src/test/java/com/azure/core/util/FluxUtilTest.java‎

Lines changed: 74 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -53,6 +53,7 @@
5353
import java.nio.file.StandardOpenOption;
5454
import java.time.Duration;
5555
import java.util.Collections;
56+
import java.util.List;
5657
import java.util.UUID;
5758
import java.util.concurrent.CompletableFuture;
5859
import java.util.concurrent.ExecutionException;
@@ -65,6 +66,7 @@
6566
import static com.azure.core.CoreTestUtils.fillArray;
6667
import static org.junit.jupiter.api.Assertions.assertEquals;
6768
import static org.junit.jupiter.api.Assertions.assertFalse;
69+
import static org.junit.jupiter.api.Assertions.assertNotSame;
6870
import static org.junit.jupiter.api.Assertions.assertTrue;
6971

7072
public class FluxUtilTest {
@@ -524,6 +526,15 @@ public void toFluxByteBufferEmitsPartialReadBeforeEndOfStream(boolean defaultAva
524526
}
525527

526528
private static InputStream withDefaultAvailable(InputStream inputStream, int maxReadSize) {
529+
return withDefaultAvailable(inputStream, new int[] { maxReadSize });
530+
}
531+
532+
/*
533+
* Wraps the stream so that available() reports 0 and each read returns at most the next of readSizes, repeating
534+
* the last once exhausted.
535+
*/
536+
private static InputStream withDefaultAvailable(InputStream inputStream, int[] readSizes) {
537+
AtomicInteger readIndex = new AtomicInteger();
527538
return new InputStream() {
528539
@Override
529540
public int read() throws IOException {
@@ -532,6 +543,7 @@ public int read() throws IOException {
532543

533544
@Override
534545
public int read(byte[] buffer, int offset, int length) throws IOException {
546+
int maxReadSize = readSizes[Math.min(readIndex.getAndIncrement(), readSizes.length - 1)];
535547
return inputStream.read(buffer, offset, Math.min(length, maxReadSize));
536548
}
537549

@@ -542,6 +554,68 @@ public void close() throws IOException {
542554
};
543555
}
544556

557+
@Test
558+
public void toFluxByteBufferPartialReadsAreRightSized() throws IOException {
559+
byte[] expected = new byte[100];
560+
fillArray(expected);
561+
562+
try (InputStream inputStream = withDefaultAvailable(new ByteArrayInputStream(expected), 7)) {
563+
List<ByteBuffer> buffers = FluxUtil.toFluxByteBuffer(inputStream, 64).collectList().block();
564+
565+
assertEquals(15, buffers.size());
566+
for (ByteBuffer buffer : buffers) {
567+
// A consumer retaining partial reads must not retain a chunkSize array for each one.
568+
assertEquals(buffer.remaining(), buffer.array().length);
569+
}
570+
assertArraysEqual(expected, collectBytes(buffers));
571+
}
572+
}
573+
574+
@Test
575+
public void toFluxByteBufferMixedPartialAndFullReads() throws IOException {
576+
byte[] expected = new byte[20];
577+
fillArray(expected);
578+
579+
// Partial, full, partial, full, then the 2 remaining bytes, all read into the same reused array.
580+
try (InputStream inputStream
581+
= withDefaultAvailable(new ByteArrayInputStream(expected), new int[] { 3, 8, 1, 8, 8 })) {
582+
List<ByteBuffer> buffers = FluxUtil.toFluxByteBuffer(inputStream, 8).collectList().block();
583+
584+
int[] expectedSizes = { 3, 8, 1, 8 };
585+
assertEquals(expectedSizes.length, buffers.size());
586+
for (int i = 0; i < expectedSizes.length; i++) {
587+
assertEquals(expectedSizes[i], buffers.get(i).remaining());
588+
assertEquals(expectedSizes[i], buffers.get(i).array().length);
589+
}
590+
// Retained buffers must not share an array that a later read overwrites.
591+
assertArraysEqual(expected, collectBytes(buffers));
592+
}
593+
}
594+
595+
@Test
596+
public void toFluxByteBufferFullChunksAreIndependent() {
597+
byte[] expected = new byte[10];
598+
fillArray(expected);
599+
600+
List<ByteBuffer> buffers
601+
= FluxUtil.toFluxByteBuffer(new ByteArrayInputStream(expected), 4).collectList().block();
602+
603+
assertEquals(3, buffers.size());
604+
assertEquals(4, buffers.get(0).array().length);
605+
assertEquals(4, buffers.get(1).array().length);
606+
assertEquals(2, buffers.get(2).array().length);
607+
assertNotSame(buffers.get(0).array(), buffers.get(1).array());
608+
assertArraysEqual(expected, collectBytes(buffers));
609+
}
610+
611+
private static byte[] collectBytes(List<ByteBuffer> buffers) {
612+
ByteArrayOutputStream collected = new ByteArrayOutputStream();
613+
for (ByteBuffer buffer : buffers) {
614+
collected.write(buffer.array(), buffer.arrayOffset() + buffer.position(), buffer.remaining());
615+
}
616+
return collected.toByteArray();
617+
}
618+
545619
@Test
546620
public void toFluxByteBufferMultipleSubscriptions() {
547621
byte[] singleRead = new byte[4096];

0 commit comments

Comments
 (0)