Skip to content

Commit 21b2615

Browse files
author
Dhriti Chopra
committed
test(storage): fast open failure and retry behaviour
Fourth step of fast open BiDi reads (go/fast-open-bidi-read, b/552263062). Pins how fast open behaves when things go wrong, using the existing read stream: - when the server fails the stream with OUT_OF_RANGE, the open fails with that error and is not retried; - when the server rejects only the fast open range (OUT_OF_RANGE naming its read id), the session re-opens without it and readAs fails with the error. This differs from the "fail open" choice in the design doc; it is kept as is until the error shape zonal buckets return is confirmed; - when the connection drops before any reply, the retried open sends the fast open range again; - when the connection drops part way through the fast open range, the re-open asks only for the remaining bytes and readAs returns the full range.
1 parent 32cfd9a commit 21b2615

1 file changed

Lines changed: 175 additions & 0 deletions

File tree

‎java-storage/google-cloud-storage/src/test/java/com/google/cloud/storage/ITFastOpenFakeTest.java‎

Lines changed: 175 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,22 +20,32 @@
2020
import static com.google.cloud.storage.ITObjectReadSessionFakeTest.getReadRange;
2121
import static com.google.cloud.storage.ITObjectReadSessionFakeTest.read;
2222
import static com.google.cloud.storage.PackagePrivateMethodWorkarounds.maybeGetStorageDataClient;
23+
import static com.google.cloud.storage.TestUtils.GRPC_STATUS_DETAILS_KEY;
2324
import static com.google.cloud.storage.TestUtils.xxd;
2425
import static com.google.common.truth.Truth.assertThat;
2526
import static java.util.concurrent.TimeUnit.SECONDS;
27+
import static java.util.stream.Collectors.toList;
2628
import static org.junit.Assert.assertThrows;
2729

30+
import com.google.api.core.ApiFuture;
2831
import com.google.api.gax.grpc.GrpcCallContext;
32+
import com.google.api.gax.rpc.OutOfRangeException;
2933
import com.google.cloud.storage.ITObjectReadSessionFakeTest.FakeStorage;
3034
import com.google.cloud.storage.it.ChecksummedTestContent;
35+
import com.google.common.base.Throwables;
3136
import com.google.common.collect.ImmutableList;
3237
import com.google.common.collect.ImmutableMap;
38+
import com.google.protobuf.Any;
39+
import com.google.storage.v2.BidiReadObjectError;
3340
import com.google.storage.v2.BidiReadObjectRequest;
3441
import com.google.storage.v2.BidiReadObjectResponse;
3542
import com.google.storage.v2.BidiReadObjectSpec;
3643
import com.google.storage.v2.BucketName;
3744
import com.google.storage.v2.Object;
3845
import com.google.storage.v2.ObjectRangeData;
46+
import com.google.storage.v2.ReadRangeError;
47+
import io.grpc.Metadata;
48+
import io.grpc.Status;
3949
import io.grpc.stub.StreamObserver;
4050
import java.io.IOException;
4151
import java.io.UncheckedIOException;
@@ -45,6 +55,8 @@
4555
import java.util.List;
4656
import java.util.Map;
4757
import java.util.concurrent.CompletableFuture;
58+
import java.util.concurrent.ExecutionException;
59+
import java.util.concurrent.atomic.AtomicInteger;
4860
import java.util.function.Consumer;
4961
import org.junit.Test;
5062

@@ -286,6 +298,169 @@ public void readSession_withASeekableChannelConfig_isRejected() throws Exception
286298
assertThat(requestsReceived).isEmpty();
287299
}
288300

301+
@Test
302+
public void open_whenTheServerFailsTheStreamWithOutOfRange_failsWithoutRetrying()
303+
throws Exception {
304+
FakeStorage server =
305+
FakeStorage.of(
306+
ImmutableMap.of(
307+
OPEN_WITH_READ_OF_10_TO_20,
308+
client -> {
309+
requestsReceived.add(OPEN_WITH_READ_OF_10_TO_20);
310+
client.onError(Status.OUT_OF_RANGE.asRuntimeException());
311+
}));
312+
313+
try (FakeServer fakeServer = FakeServer.of(server);
314+
Storage storage = fakeServer.getGrpcStorageOptions().getService()) {
315+
StorageDataClient dataClient = maybeGetStorageDataClient(storage);
316+
317+
ApiFuture<ObjectReadSession> open =
318+
dataClient.readSession(OPEN, CTX, ImmutableList.of(READ_10_TO_20));
319+
320+
ExecutionException e = assertThrows(ExecutionException.class, () -> open.get(5, SECONDS));
321+
// the error the server returned is somewhere in the chain of causes
322+
assertThat(
323+
Throwables.getCausalChain(e).stream()
324+
.<Class<?>>map(Throwable::getClass)
325+
.collect(toList()))
326+
.contains(OutOfRangeException.class);
327+
}
328+
329+
assertThat(requestsReceived).containsExactly(OPEN_WITH_READ_OF_10_TO_20);
330+
}
331+
332+
@Test
333+
public void open_whenTheServerRejectsOnlyTheFastOpenRange_reopensWithoutItAndReadAsFails()
334+
throws Exception {
335+
// OUT_OF_RANGE that names the failing read, as the server reports a bad range on a bidi read
336+
BidiReadObjectError rangeError =
337+
BidiReadObjectError.newBuilder()
338+
.addReadRangeErrors(
339+
ReadRangeError.newBuilder()
340+
.setReadId(1)
341+
.setStatus(
342+
com.google.rpc.Status.newBuilder()
343+
.setCode(com.google.rpc.Code.OUT_OF_RANGE_VALUE)))
344+
.build();
345+
Metadata trailers = new Metadata();
346+
trailers.put(
347+
GRPC_STATUS_DETAILS_KEY,
348+
com.google.rpc.Status.newBuilder()
349+
.setCode(com.google.rpc.Code.OUT_OF_RANGE_VALUE)
350+
.addDetails(Any.pack(rangeError))
351+
.build());
352+
FakeStorage server =
353+
FakeStorage.of(
354+
ImmutableMap.of(
355+
OPEN_WITH_READ_OF_10_TO_20,
356+
client -> {
357+
requestsReceived.add(OPEN_WITH_READ_OF_10_TO_20);
358+
client.onError(Status.OUT_OF_RANGE.asRuntimeException(trailers));
359+
},
360+
OPEN,
361+
client -> {
362+
requestsReceived.add(OPEN);
363+
client.onNext(metadataOnly());
364+
}));
365+
366+
try (FakeServer fakeServer = FakeServer.of(server);
367+
Storage storage = fakeServer.getGrpcStorageOptions().getService()) {
368+
StorageDataClient dataClient = maybeGetStorageDataClient(storage);
369+
370+
try (ObjectReadSession session =
371+
dataClient.readSession(OPEN, CTX, ImmutableList.of(READ_10_TO_20)).get(10, SECONDS)) {
372+
ApiFuture<byte[]> read = session.readAs(READ_10_TO_20);
373+
374+
ExecutionException e = assertThrows(ExecutionException.class, () -> read.get(5, SECONDS));
375+
assertThat(
376+
Throwables.getCausalChain(e).stream()
377+
.<Class<?>>map(Throwable::getClass)
378+
.collect(toList()))
379+
.contains(OutOfRangeException.class);
380+
}
381+
}
382+
383+
assertThat(requestsReceived).containsExactly(OPEN_WITH_READ_OF_10_TO_20, OPEN).inOrder();
384+
}
385+
386+
@Test
387+
public void open_whenTheConnectionDropsBeforeAnyReply_retriesWithTheSameRange() throws Exception {
388+
AtomicInteger attempts = new AtomicInteger();
389+
FakeStorage server =
390+
FakeStorage.of(
391+
ImmutableMap.of(
392+
OPEN_WITH_READ_OF_10_TO_20,
393+
client -> {
394+
requestsReceived.add(OPEN_WITH_READ_OF_10_TO_20);
395+
if (attempts.getAndIncrement() == 0) {
396+
client.onError(Status.UNAVAILABLE.asRuntimeException());
397+
} else {
398+
client.onNext(metadataAndBytes(1, 10, 10));
399+
}
400+
}));
401+
402+
try (FakeServer fakeServer = FakeServer.of(server);
403+
Storage storage = fakeServer.getGrpcStorageOptions().getService()) {
404+
StorageDataClient dataClient = maybeGetStorageDataClient(storage);
405+
406+
try (ObjectReadSession session =
407+
dataClient.readSession(OPEN, CTX, ImmutableList.of(READ_10_TO_20)).get(10, SECONDS)) {
408+
byte[] actual = session.readAs(READ_10_TO_20).get(5, SECONDS);
409+
assertThat(xxd(actual)).isEqualTo(xxd(slice(10, 10)));
410+
}
411+
}
412+
413+
assertThat(requestsReceived)
414+
.containsExactly(OPEN_WITH_READ_OF_10_TO_20, OPEN_WITH_READ_OF_10_TO_20)
415+
.inOrder();
416+
}
417+
418+
@Test
419+
public void open_whenTheConnectionDropsHalfway_fetchesOnlyTheRestOfTheRange() throws Exception {
420+
// the first reply has bytes 10-15 only, it is not the end of the range
421+
BidiReadObjectResponse bytes10To15 = metadataAndBytes(1, 10, 5);
422+
BidiReadObjectResponse firstReply =
423+
bytes10To15.toBuilder()
424+
.setObjectDataRanges(
425+
0, bytes10To15.getObjectDataRanges(0).toBuilder().setRangeEnd(false))
426+
.build();
427+
// after the first reply the session knows the object's generation, so a re-open pins it
428+
BidiReadObjectRequest reopenForBytes15To20 =
429+
OPEN.toBuilder()
430+
.setReadObjectSpec(OPEN.getReadObjectSpec().toBuilder().setGeneration(1))
431+
.addReadRanges(getReadRange(1, 15, 5))
432+
.build();
433+
FakeStorage server =
434+
FakeStorage.of(
435+
ImmutableMap.of(
436+
OPEN_WITH_READ_OF_10_TO_20,
437+
client -> {
438+
requestsReceived.add(OPEN_WITH_READ_OF_10_TO_20);
439+
client.onNext(firstReply);
440+
client.onError(Status.UNAVAILABLE.asRuntimeException());
441+
},
442+
reopenForBytes15To20,
443+
client -> {
444+
requestsReceived.add(reopenForBytes15To20);
445+
client.onNext(metadataAndBytes(1, 15, 5));
446+
}));
447+
448+
try (FakeServer fakeServer = FakeServer.of(server);
449+
Storage storage = fakeServer.getGrpcStorageOptions().getService()) {
450+
StorageDataClient dataClient = maybeGetStorageDataClient(storage);
451+
452+
try (ObjectReadSession session =
453+
dataClient.readSession(OPEN, CTX, ImmutableList.of(READ_10_TO_20)).get(10, SECONDS)) {
454+
byte[] actual = session.readAs(READ_10_TO_20).get(10, SECONDS);
455+
assertThat(xxd(actual)).isEqualTo(xxd(slice(10, 10)));
456+
}
457+
}
458+
459+
assertThat(requestsReceived)
460+
.containsExactly(OPEN_WITH_READ_OF_10_TO_20, reopenForBytes15To20)
461+
.inOrder();
462+
}
463+
289464
/**
290465
* A fake server that answers each request in {@code responses} and adds it to {@link
291466
* #requestsReceived}. Any other request is answered with an error.

0 commit comments

Comments
 (0)