|
20 | 20 | import static com.google.cloud.storage.ITObjectReadSessionFakeTest.getReadRange; |
21 | 21 | import static com.google.cloud.storage.ITObjectReadSessionFakeTest.read; |
22 | 22 | import static com.google.cloud.storage.PackagePrivateMethodWorkarounds.maybeGetStorageDataClient; |
| 23 | +import static com.google.cloud.storage.TestUtils.GRPC_STATUS_DETAILS_KEY; |
23 | 24 | import static com.google.cloud.storage.TestUtils.xxd; |
24 | 25 | import static com.google.common.truth.Truth.assertThat; |
25 | 26 | import static java.util.concurrent.TimeUnit.SECONDS; |
| 27 | +import static java.util.stream.Collectors.toList; |
26 | 28 | import static org.junit.Assert.assertThrows; |
27 | 29 |
|
| 30 | +import com.google.api.core.ApiFuture; |
28 | 31 | import com.google.api.gax.grpc.GrpcCallContext; |
| 32 | +import com.google.api.gax.rpc.OutOfRangeException; |
29 | 33 | import com.google.cloud.storage.ITObjectReadSessionFakeTest.FakeStorage; |
30 | 34 | import com.google.cloud.storage.it.ChecksummedTestContent; |
| 35 | +import com.google.common.base.Throwables; |
31 | 36 | import com.google.common.collect.ImmutableList; |
32 | 37 | import com.google.common.collect.ImmutableMap; |
| 38 | +import com.google.protobuf.Any; |
| 39 | +import com.google.storage.v2.BidiReadObjectError; |
33 | 40 | import com.google.storage.v2.BidiReadObjectRequest; |
34 | 41 | import com.google.storage.v2.BidiReadObjectResponse; |
35 | 42 | import com.google.storage.v2.BidiReadObjectSpec; |
36 | 43 | import com.google.storage.v2.BucketName; |
37 | 44 | import com.google.storage.v2.Object; |
38 | 45 | import com.google.storage.v2.ObjectRangeData; |
| 46 | +import com.google.storage.v2.ReadRangeError; |
| 47 | +import io.grpc.Metadata; |
| 48 | +import io.grpc.Status; |
39 | 49 | import io.grpc.stub.StreamObserver; |
40 | 50 | import java.io.IOException; |
41 | 51 | import java.io.UncheckedIOException; |
|
45 | 55 | import java.util.List; |
46 | 56 | import java.util.Map; |
47 | 57 | import java.util.concurrent.CompletableFuture; |
| 58 | +import java.util.concurrent.ExecutionException; |
| 59 | +import java.util.concurrent.atomic.AtomicInteger; |
48 | 60 | import java.util.function.Consumer; |
49 | 61 | import org.junit.Test; |
50 | 62 |
|
@@ -286,6 +298,169 @@ public void readSession_withASeekableChannelConfig_isRejected() throws Exception |
286 | 298 | assertThat(requestsReceived).isEmpty(); |
287 | 299 | } |
288 | 300 |
|
| 301 | + @Test |
| 302 | + public void open_whenTheServerFailsTheStreamWithOutOfRange_failsWithoutRetrying() |
| 303 | + throws Exception { |
| 304 | + FakeStorage server = |
| 305 | + FakeStorage.of( |
| 306 | + ImmutableMap.of( |
| 307 | + OPEN_WITH_READ_OF_10_TO_20, |
| 308 | + client -> { |
| 309 | + requestsReceived.add(OPEN_WITH_READ_OF_10_TO_20); |
| 310 | + client.onError(Status.OUT_OF_RANGE.asRuntimeException()); |
| 311 | + })); |
| 312 | + |
| 313 | + try (FakeServer fakeServer = FakeServer.of(server); |
| 314 | + Storage storage = fakeServer.getGrpcStorageOptions().getService()) { |
| 315 | + StorageDataClient dataClient = maybeGetStorageDataClient(storage); |
| 316 | + |
| 317 | + ApiFuture<ObjectReadSession> open = |
| 318 | + dataClient.readSession(OPEN, CTX, ImmutableList.of(READ_10_TO_20)); |
| 319 | + |
| 320 | + ExecutionException e = assertThrows(ExecutionException.class, () -> open.get(5, SECONDS)); |
| 321 | + // the error the server returned is somewhere in the chain of causes |
| 322 | + assertThat( |
| 323 | + Throwables.getCausalChain(e).stream() |
| 324 | + .<Class<?>>map(Throwable::getClass) |
| 325 | + .collect(toList())) |
| 326 | + .contains(OutOfRangeException.class); |
| 327 | + } |
| 328 | + |
| 329 | + assertThat(requestsReceived).containsExactly(OPEN_WITH_READ_OF_10_TO_20); |
| 330 | + } |
| 331 | + |
| 332 | + @Test |
| 333 | + public void open_whenTheServerRejectsOnlyTheFastOpenRange_reopensWithoutItAndReadAsFails() |
| 334 | + throws Exception { |
| 335 | + // OUT_OF_RANGE that names the failing read, as the server reports a bad range on a bidi read |
| 336 | + BidiReadObjectError rangeError = |
| 337 | + BidiReadObjectError.newBuilder() |
| 338 | + .addReadRangeErrors( |
| 339 | + ReadRangeError.newBuilder() |
| 340 | + .setReadId(1) |
| 341 | + .setStatus( |
| 342 | + com.google.rpc.Status.newBuilder() |
| 343 | + .setCode(com.google.rpc.Code.OUT_OF_RANGE_VALUE))) |
| 344 | + .build(); |
| 345 | + Metadata trailers = new Metadata(); |
| 346 | + trailers.put( |
| 347 | + GRPC_STATUS_DETAILS_KEY, |
| 348 | + com.google.rpc.Status.newBuilder() |
| 349 | + .setCode(com.google.rpc.Code.OUT_OF_RANGE_VALUE) |
| 350 | + .addDetails(Any.pack(rangeError)) |
| 351 | + .build()); |
| 352 | + FakeStorage server = |
| 353 | + FakeStorage.of( |
| 354 | + ImmutableMap.of( |
| 355 | + OPEN_WITH_READ_OF_10_TO_20, |
| 356 | + client -> { |
| 357 | + requestsReceived.add(OPEN_WITH_READ_OF_10_TO_20); |
| 358 | + client.onError(Status.OUT_OF_RANGE.asRuntimeException(trailers)); |
| 359 | + }, |
| 360 | + OPEN, |
| 361 | + client -> { |
| 362 | + requestsReceived.add(OPEN); |
| 363 | + client.onNext(metadataOnly()); |
| 364 | + })); |
| 365 | + |
| 366 | + try (FakeServer fakeServer = FakeServer.of(server); |
| 367 | + Storage storage = fakeServer.getGrpcStorageOptions().getService()) { |
| 368 | + StorageDataClient dataClient = maybeGetStorageDataClient(storage); |
| 369 | + |
| 370 | + try (ObjectReadSession session = |
| 371 | + dataClient.readSession(OPEN, CTX, ImmutableList.of(READ_10_TO_20)).get(10, SECONDS)) { |
| 372 | + ApiFuture<byte[]> read = session.readAs(READ_10_TO_20); |
| 373 | + |
| 374 | + ExecutionException e = assertThrows(ExecutionException.class, () -> read.get(5, SECONDS)); |
| 375 | + assertThat( |
| 376 | + Throwables.getCausalChain(e).stream() |
| 377 | + .<Class<?>>map(Throwable::getClass) |
| 378 | + .collect(toList())) |
| 379 | + .contains(OutOfRangeException.class); |
| 380 | + } |
| 381 | + } |
| 382 | + |
| 383 | + assertThat(requestsReceived).containsExactly(OPEN_WITH_READ_OF_10_TO_20, OPEN).inOrder(); |
| 384 | + } |
| 385 | + |
| 386 | + @Test |
| 387 | + public void open_whenTheConnectionDropsBeforeAnyReply_retriesWithTheSameRange() throws Exception { |
| 388 | + AtomicInteger attempts = new AtomicInteger(); |
| 389 | + FakeStorage server = |
| 390 | + FakeStorage.of( |
| 391 | + ImmutableMap.of( |
| 392 | + OPEN_WITH_READ_OF_10_TO_20, |
| 393 | + client -> { |
| 394 | + requestsReceived.add(OPEN_WITH_READ_OF_10_TO_20); |
| 395 | + if (attempts.getAndIncrement() == 0) { |
| 396 | + client.onError(Status.UNAVAILABLE.asRuntimeException()); |
| 397 | + } else { |
| 398 | + client.onNext(metadataAndBytes(1, 10, 10)); |
| 399 | + } |
| 400 | + })); |
| 401 | + |
| 402 | + try (FakeServer fakeServer = FakeServer.of(server); |
| 403 | + Storage storage = fakeServer.getGrpcStorageOptions().getService()) { |
| 404 | + StorageDataClient dataClient = maybeGetStorageDataClient(storage); |
| 405 | + |
| 406 | + try (ObjectReadSession session = |
| 407 | + dataClient.readSession(OPEN, CTX, ImmutableList.of(READ_10_TO_20)).get(10, SECONDS)) { |
| 408 | + byte[] actual = session.readAs(READ_10_TO_20).get(5, SECONDS); |
| 409 | + assertThat(xxd(actual)).isEqualTo(xxd(slice(10, 10))); |
| 410 | + } |
| 411 | + } |
| 412 | + |
| 413 | + assertThat(requestsReceived) |
| 414 | + .containsExactly(OPEN_WITH_READ_OF_10_TO_20, OPEN_WITH_READ_OF_10_TO_20) |
| 415 | + .inOrder(); |
| 416 | + } |
| 417 | + |
| 418 | + @Test |
| 419 | + public void open_whenTheConnectionDropsHalfway_fetchesOnlyTheRestOfTheRange() throws Exception { |
| 420 | + // the first reply has bytes 10-15 only, it is not the end of the range |
| 421 | + BidiReadObjectResponse bytes10To15 = metadataAndBytes(1, 10, 5); |
| 422 | + BidiReadObjectResponse firstReply = |
| 423 | + bytes10To15.toBuilder() |
| 424 | + .setObjectDataRanges( |
| 425 | + 0, bytes10To15.getObjectDataRanges(0).toBuilder().setRangeEnd(false)) |
| 426 | + .build(); |
| 427 | + // after the first reply the session knows the object's generation, so a re-open pins it |
| 428 | + BidiReadObjectRequest reopenForBytes15To20 = |
| 429 | + OPEN.toBuilder() |
| 430 | + .setReadObjectSpec(OPEN.getReadObjectSpec().toBuilder().setGeneration(1)) |
| 431 | + .addReadRanges(getReadRange(1, 15, 5)) |
| 432 | + .build(); |
| 433 | + FakeStorage server = |
| 434 | + FakeStorage.of( |
| 435 | + ImmutableMap.of( |
| 436 | + OPEN_WITH_READ_OF_10_TO_20, |
| 437 | + client -> { |
| 438 | + requestsReceived.add(OPEN_WITH_READ_OF_10_TO_20); |
| 439 | + client.onNext(firstReply); |
| 440 | + client.onError(Status.UNAVAILABLE.asRuntimeException()); |
| 441 | + }, |
| 442 | + reopenForBytes15To20, |
| 443 | + client -> { |
| 444 | + requestsReceived.add(reopenForBytes15To20); |
| 445 | + client.onNext(metadataAndBytes(1, 15, 5)); |
| 446 | + })); |
| 447 | + |
| 448 | + try (FakeServer fakeServer = FakeServer.of(server); |
| 449 | + Storage storage = fakeServer.getGrpcStorageOptions().getService()) { |
| 450 | + StorageDataClient dataClient = maybeGetStorageDataClient(storage); |
| 451 | + |
| 452 | + try (ObjectReadSession session = |
| 453 | + dataClient.readSession(OPEN, CTX, ImmutableList.of(READ_10_TO_20)).get(10, SECONDS)) { |
| 454 | + byte[] actual = session.readAs(READ_10_TO_20).get(10, SECONDS); |
| 455 | + assertThat(xxd(actual)).isEqualTo(xxd(slice(10, 10))); |
| 456 | + } |
| 457 | + } |
| 458 | + |
| 459 | + assertThat(requestsReceived) |
| 460 | + .containsExactly(OPEN_WITH_READ_OF_10_TO_20, reopenForBytes15To20) |
| 461 | + .inOrder(); |
| 462 | + } |
| 463 | + |
289 | 464 | /** |
290 | 465 | * A fake server that answers each request in {@code responses} and adds it to {@link |
291 | 466 | * #requestsReceived}. Any other request is answered with an error. |
|
0 commit comments