From b073ecd6f9b89b30b89b6f4a9c2cc5e86c89dacc Mon Sep 17 00:00:00 2001 From: Kannan J Date: Mon, 2 Nov 2026 06:23:56 +0000 Subject: [PATCH 1/2] xds: avoid IllegalStateException on sendMessage/halfClose after internal cancellation When RawMessageClientInterceptor or ExternalProcessorClientInterceptor encounters an internal error (such as an IOException while reading the request InputStream in sendMessage) and cancels the underlying call, the application is not synchronously aware that the call was cancelled. Callers such as ClientCalls.blockingUnaryCall immediately invoke halfClose() after sendMessage(), which previously delegated to the already-cancelled downstream call and threw IllegalStateException("call was cancelled"), masking the underlying CANCELLED status and cause. Fix this by tracking internal cancellation (`aborted`) separately from explicit application-initiated cancellation (`cancelCalled`): 1. RawMessageClientInterceptor (PartialForwardingClientCall): - Move `sendSerializedMessage` into `PartialForwardingClientCall` so `sendMessage`, `halfClose`, and `cancel` share per-call state. - Set `aborted = true` when `sendSerializedMessage` catches an `IOException` or `RuntimeException` and cancels `rawCall`, making subsequent `sendMessage()` and `halfClose()` calls no-ops (matching `ClientCallImpl.sendMessageInternal` when it cancels the underlying stream internally). - Set `cancelCalled = true` (and `aborted = false`) in `cancel()` and delegate `rawCall.sendMessage(null)` before stream serialization when `cancelCalled` is true, ensuring `sendMessage()` and `halfClose()` after explicit application cancellation still throw `IllegalStateException("call was cancelled")`. 2. ExternalProcessorClientInterceptor (DataPlaneClientCall): - Set `aborted = true` in `cancelDownstream()` when triggered internally (`!cancelCalled.get()`), turning subsequent `sendMessage()`, `halfClose()`, and `proceedWithHalfClose()` calls into no-ops. - Set `cancelCalled = true` (and `aborted = false`) in `cancel()` and check `!cancelCalled.get()` in `sendMessage()` and `halfClose()` so explicit application cancellation enforces `IllegalStateException` even when requests or half-close would otherwise be buffered or sent to the ext_proc sidecar without synchronously reaching `delayedCall`. TAG=agy CONV=56c87717-243d-4f12-af9e-c532e509892a --- .../ExternalProcessorClientInterceptor.java | 19 +++ .../java/io/grpc/xds/XdsNameResolver.java | 64 ++++++---- ...xternalProcessorClientInterceptorTest.java | 114 ++++++++++++++++++ .../java/io/grpc/xds/XdsNameResolverTest.java | 29 +++++ 4 files changed, 202 insertions(+), 24 deletions(-) diff --git a/xds/src/main/java/io/grpc/xds/ExternalProcessorClientInterceptor.java b/xds/src/main/java/io/grpc/xds/ExternalProcessorClientInterceptor.java index 93a2230b9a4..187f3021b27 100644 --- a/xds/src/main/java/io/grpc/xds/ExternalProcessorClientInterceptor.java +++ b/xds/src/main/java/io/grpc/xds/ExternalProcessorClientInterceptor.java @@ -17,6 +17,7 @@ package io.grpc.xds; import static com.google.common.base.Preconditions.checkNotNull; +import static com.google.common.base.Preconditions.checkState; import static io.grpc.xds.internal.extproc.ExternalProcessorUtil.applyHeaderMutations; import static io.grpc.xds.internal.extproc.ExternalProcessorUtil.collectAttributes; import static io.grpc.xds.internal.extproc.ExternalProcessorUtil.markDataPlaneCallClosed; @@ -366,6 +367,8 @@ private static class DataPlaneClientCall final AtomicBoolean isProcessingTrailers = new AtomicBoolean(false); final AtomicBoolean pendingHalfClose = new AtomicBoolean(false); final AtomicBoolean bodyMessageSentToExtProc = new AtomicBoolean(false); + final AtomicBoolean aborted = new AtomicBoolean(false); + final AtomicBoolean cancelCalled = new AtomicBoolean(false); protected DataPlaneClientCall( DataPlaneDelayedCall delayedCall, @@ -994,6 +997,10 @@ public void request(int numMessages) { @SuppressWarnings("unchecked") @Override public void sendMessage(ReqT message) { + if (aborted.get()) { + return; + } + checkState(!cancelCalled.get(), "call was cancelled"); if (requestSideClosed.get()) { // External processor already closed the request stream. Discard further messages. return; @@ -1080,6 +1087,9 @@ private void drainPendingRequestBodyMessages() { } private void proceedWithHalfClose() { + if (aborted.get()) { + return; + } if (clientHalfCloseStartNanos > 0) { long durationNanos = System.nanoTime() - clientHalfCloseStartNanos; recordDuration(clientHalfCloseDuration, durationNanos); @@ -1090,6 +1100,10 @@ private void proceedWithHalfClose() { @Override public void halfClose() { + if (aborted.get()) { + return; + } + checkState(!cancelCalled.get(), "call was cancelled"); if (appHalfClosed.compareAndSet(false, true)) { clientHalfCloseStartNanos = System.nanoTime(); } @@ -1159,11 +1173,16 @@ public void halfClose() { } void cancelDownstream(@Nullable String message, @Nullable Throwable cause) { + if (!cancelCalled.get()) { + aborted.set(true); + } delayedCall.cancel(message, cause); } @Override public void cancel(@Nullable String message, @Nullable Throwable cause) { + cancelCalled.set(true); + aborted.set(false); synchronized (streamLock) { if (markExtProcStreamFailed(extProcStreamState)) { if (extProcClientCallRequestObserver != null) { diff --git a/xds/src/main/java/io/grpc/xds/XdsNameResolver.java b/xds/src/main/java/io/grpc/xds/XdsNameResolver.java index 328995d22da..ac3ea54154f 100644 --- a/xds/src/main/java/io/grpc/xds/XdsNameResolver.java +++ b/xds/src/main/java/io/grpc/xds/XdsNameResolver.java @@ -1283,35 +1283,46 @@ public void onReady() { }; } - private static void sendSerializedMessage( - ClientCall rawCall, - MethodDescriptor method, - ReqT message) { - ByteString byteString; - try { - InputStream stream = method.getRequestMarshaller().stream(message); + private abstract static class PartialForwardingClientCall + extends ClientCall { + private boolean aborted; + private boolean cancelCalled; + + protected abstract ClientCall delegate(); + + protected void sendSerializedMessage( + ClientCall rawCall, + MethodDescriptor method, + ReqT message) { + if (aborted) { + return; + } + if (cancelCalled) { + rawCall.sendMessage(null); + return; + } + ByteString byteString; try { - byteString = ExternalProcessorUtil.outboundStreamToByteString(stream); - } finally { + InputStream stream = method.getRequestMarshaller().stream(message); try { - stream.close(); - } catch (IOException ignored) { - // ignore + byteString = ExternalProcessorUtil.outboundStreamToByteString(stream); + } finally { + try { + stream.close(); + } catch (IOException ignored) { + // ignore + } } + } catch (IOException | RuntimeException e) { + aborted = true; + rawCall.cancel("Failed to read message for raw message interceptor", e); + return; + } catch (Error e) { + rawCall.cancel("Client sendMessage() failed with Error", e); + throw e; } - } catch (IOException | RuntimeException e) { - rawCall.cancel("Failed to read message for raw message interceptor", e); - return; - } catch (Error e) { - rawCall.cancel("Client sendMessage() failed with Error", e); - throw e; + rawCall.sendMessage(new KnownLengthInputStream(byteString)); } - rawCall.sendMessage(new KnownLengthInputStream(byteString)); - } - - private abstract static class PartialForwardingClientCall - extends ClientCall { - protected abstract ClientCall delegate(); @Override public void request(int numMessages) { @@ -1320,11 +1331,16 @@ public void request(int numMessages) { @Override public void cancel(@Nullable String message, @Nullable Throwable cause) { + aborted = false; + cancelCalled = true; delegate().cancel(message, cause); } @Override public void halfClose() { + if (aborted) { + return; + } delegate().halfClose(); } diff --git a/xds/src/test/java/io/grpc/xds/ExternalProcessorClientInterceptorTest.java b/xds/src/test/java/io/grpc/xds/ExternalProcessorClientInterceptorTest.java index 48f0931cbc5..ca42c529978 100644 --- a/xds/src/test/java/io/grpc/xds/ExternalProcessorClientInterceptorTest.java +++ b/xds/src/test/java/io/grpc/xds/ExternalProcessorClientInterceptorTest.java @@ -17,6 +17,7 @@ package io.grpc.xds; import static com.google.common.truth.Truth.assertThat; +import static org.junit.Assert.assertThrows; import com.google.common.io.BaseEncoding; import com.google.common.util.concurrent.MoreExecutors; @@ -11423,6 +11424,114 @@ public void onClose(Status status, Metadata trailers) { channelManager.close(); } + @Test + @SuppressWarnings("unchecked") + public void + givenSendMessageIoException_whenSubsequentSendMessageOrHalfClose_thenNoOp() + throws Exception { + ExternalProcessor proto = createBaseProto(extProcServerName) + .setProcessingMode(ProcessingMode.newBuilder() + .setRequestHeaderMode(ProcessingMode.HeaderSendMode.SEND) + .setRequestBodyMode(ProcessingMode.BodySendMode.GRPC) + .setResponseHeaderMode(ProcessingMode.HeaderSendMode.SKIP) + .setResponseTrailerMode(ProcessingMode.HeaderSendMode.SKIP) + .build()) + .build(); + ConfigOrError configOrError = + provider.parseFilterConfig(Any.pack(proto), filterContext); + ExternalProcessorFilterConfig filterConfig = configOrError.config; + + ExternalProcessorGrpc.ExternalProcessorImplBase extProcImpl = + new ExternalProcessorGrpc.ExternalProcessorImplBase() { + @Override + public StreamObserver process( + final StreamObserver responseObserver) { + ((ServerCallStreamObserver) responseObserver).request(100); + return new StreamObserver() { + @Override + public void onNext(ProcessingRequest request) { + if (request.hasRequestHeaders()) { + responseObserver.onNext(ProcessingResponse.newBuilder() + .setRequestHeaders(HeadersResponse.newBuilder().build()) + .build()); + } + } + + @Override + public void onError(Throwable t) {} + + @Override + public void onCompleted() { + responseObserver.onCompleted(); + } + }; + } + }; + grpcCleanup.register(InProcessServerBuilder.forName(extProcServerName) + .addService(extProcImpl) + .directExecutor() + .build().start()); + + CachedChannelManager channelManager = new CachedChannelManager(config -> + grpcCleanup.register( + InProcessChannelBuilder.forName(extProcServerName).directExecutor().build())); + + ExternalProcessorClientInterceptor interceptor = new ExternalProcessorClientInterceptor( + filterConfig, channelManager, scheduler, FAKE_CONTEXT); + + dataPlaneServiceRegistry.addService(ServerServiceDefinition.builder("test.TestService") + .addMethod(METHOD_SAY_HELLO, ServerCalls.asyncUnaryCall( + (request, responseObserver) -> { + // No-op + })) + .build()); + + ManagedChannel dataPlaneChannel = + grpcCleanup.register( + InProcessChannelBuilder.forName(dataPlaneServerName).directExecutor().build()); + + CallOptions callOptions = DEFAULT_CALL_OPTIONS + .withExecutor(MoreExecutors.directExecutor()) + .withOption(XdsNameResolver.AUTO_HOST_REWRITE_KEY, false); + ClientCall rawProxyCall = + (ClientCall) (ClientCall) + interceptor.interceptCall(METHOD_SAY_HELLO, callOptions, dataPlaneChannel); + + final CountDownLatch closeLatch = new CountDownLatch(1); + final AtomicReference closeStatus = new AtomicReference<>(); + rawProxyCall.start(new ClientCall.Listener() { + @Override + public void onClose(Status status, Metadata trailers) { + closeStatus.set(status); + closeLatch.countDown(); + } + }, new Metadata()); + + IOException ioException = new IOException("Simulated stream read error"); + InputStream failingStream = new InputStream() { + @Override + public int read() throws IOException { + throw ioException; + } + }; + + // Internal cancellation during sendMessage when reading InputStream fails + rawProxyCall.sendMessage(failingStream); + + assertThat(closeLatch.await(5, TimeUnit.SECONDS)).isTrue(); + assertThat(closeStatus.get().getCode()).isEqualTo(Status.Code.CANCELLED); + assertThat(closeStatus.get().getDescription()) + .isEqualTo("Failed to serialize message for External Processor"); + assertThat(closeStatus.get().getCause()).isSameInstanceAs(ioException); + + // Subsequent sendMessage and halfClose after internal cancellation must be no-ops + // and not throw IllegalStateException. + rawProxyCall.sendMessage(new ByteArrayInputStream("ignored".getBytes(StandardCharsets.UTF_8))); + rawProxyCall.halfClose(); + + channelManager.close(); + } + // --- Category 18: Immediate Response Handling --- @Test @@ -12136,6 +12245,11 @@ public void onCompleted() { // Application cancels the RPC proxyCall.cancel("User cancelled", null); + // Subsequent sendMessage or halfClose after explicit application cancellation must throw + // IllegalStateException. + assertThrows(IllegalStateException.class, () -> proxyCall.sendMessage("after-cancel")); + assertThrows(IllegalStateException.class, () -> proxyCall.halfClose()); + // Verify sidecar stream also cancelled assertThat(cancelLatch.await(5, TimeUnit.SECONDS)).isTrue(); diff --git a/xds/src/test/java/io/grpc/xds/XdsNameResolverTest.java b/xds/src/test/java/io/grpc/xds/XdsNameResolverTest.java index e57d0860588..b8996baaa39 100644 --- a/xds/src/test/java/io/grpc/xds/XdsNameResolverTest.java +++ b/xds/src/test/java/io/grpc/xds/XdsNameResolverTest.java @@ -23,6 +23,7 @@ import static io.grpc.xds.FaultFilter.HEADER_ABORT_PERCENTAGE_KEY; import static io.grpc.xds.FaultFilter.HEADER_DELAY_KEY; import static io.grpc.xds.FaultFilter.HEADER_DELAY_PERCENTAGE_KEY; +import static org.junit.Assert.assertThrows; import static org.mockito.AdditionalAnswers.delegatesTo; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyInt; @@ -75,6 +76,7 @@ import io.grpc.Status; import io.grpc.Status.Code; import io.grpc.StatusOr; +import io.grpc.StatusRuntimeException; import io.grpc.SynchronizationContext; import io.grpc.inprocess.InProcessChannelBuilder; import io.grpc.inprocess.InProcessServerBuilder; @@ -3227,6 +3229,10 @@ public String parse(InputStream stream) { ClientCall.Listener listener = mock(ClientCall.Listener.class); call.start(listener, new Metadata()); call.sendMessage("World"); + // Subsequent sendMessage and halfClose after internal sendMessage cancellation are no-ops + // and must not throw IllegalStateException. + call.sendMessage("World2"); + call.halfClose(); ArgumentCaptor statusCaptor = ArgumentCaptor.forClass(Status.class); verify(listener).onClose(statusCaptor.capture(), any(Metadata.class)); @@ -3235,6 +3241,29 @@ public String parse(InputStream stream) { assertThat(status.getDescription()) .isEqualTo("Failed to read message for raw message interceptor"); assertThat(status.getCause()).isSameInstanceAs(ioException); + + // Verify ClientCalls.blockingUnaryCall (which calls sendMessage followed by halfClose) + // surfaces StatusRuntimeException(CANCELLED) rather than IllegalStateException. + StatusRuntimeException sre = assertThrows( + StatusRuntimeException.class, + () -> ClientCalls.blockingUnaryCall( + interceptedChannel, failingRequestMethod, CallOptions.DEFAULT, "World")); + assertThat(sre.getStatus().getCode()).isEqualTo(Status.Code.CANCELLED); + assertThat(sre.getStatus().getDescription()) + .isEqualTo("Failed to read message for raw message interceptor"); + assertThat(sre.getStatus().getCause()).isSameInstanceAs(ioException); + + // Verify explicit application cancellation still causes subsequent sendMessage and halfClose + // to throw IllegalStateException. + ClientCall explicitlyCancelledCall = + interceptedChannel.newCall(failingRequestMethod, CallOptions.DEFAULT); + @SuppressWarnings("unchecked") + ClientCall.Listener explicitCancelListener = mock(ClientCall.Listener.class); + explicitlyCancelledCall.start(explicitCancelListener, new Metadata()); + explicitlyCancelledCall.cancel("Application cancelled", null); + assertThrows( + IllegalStateException.class, () -> explicitlyCancelledCall.sendMessage("World")); + assertThrows(IllegalStateException.class, explicitlyCancelledCall::halfClose); } } From 795ff4f4b4c6bcc67ef8d6874ebd6340d3344c93 Mon Sep 17 00:00:00 2001 From: Kannan J Date: Thu, 5 Nov 2026 07:50:43 +0000 Subject: [PATCH 2/2] xds: address review feedback on cancellation and internal abort handling 1. Check !cancelCalled before aborted in sendMessage and halfClose, and have cancel() call delayedCall.cancel() directly (setting cancelCalled under streamLock) so cancelCalled and aborted never overwrite each other. 2. Replace rawCall.sendMessage(null) in RawMessageClientInterceptor with checkState(!cancelCalled, "call was cancelled"). 3. Separate public halfClose() from internalHalfClose() and guard downstream sendMessage/halfClose calls via proceedWithSendMessage() and proceedWithHalfClose() using isCancelledOrAborted() so sidecar and drain callbacks act as no-ops after cancellation or abort. TAG=agy CONV=56c87717-243d-4f12-af9e-c532e509892a --- .../ExternalProcessorClientInterceptor.java | 52 +++--- .../java/io/grpc/xds/XdsNameResolver.java | 8 +- ...xternalProcessorClientInterceptorTest.java | 160 ++++++++++++++++++ 3 files changed, 195 insertions(+), 25 deletions(-) diff --git a/xds/src/main/java/io/grpc/xds/ExternalProcessorClientInterceptor.java b/xds/src/main/java/io/grpc/xds/ExternalProcessorClientInterceptor.java index 187f3021b27..bfec5e43faf 100644 --- a/xds/src/main/java/io/grpc/xds/ExternalProcessorClientInterceptor.java +++ b/xds/src/main/java/io/grpc/xds/ExternalProcessorClientInterceptor.java @@ -997,30 +997,30 @@ public void request(int numMessages) { @SuppressWarnings("unchecked") @Override public void sendMessage(ReqT message) { + checkState(!cancelCalled.get(), "call was cancelled"); if (aborted.get()) { return; } - checkState(!cancelCalled.get(), "call was cancelled"); if (requestSideClosed.get()) { // External processor already closed the request stream. Discard further messages. return; } if (passThroughMode.get()) { - super.sendMessage(message); + proceedWithSendMessage(message); return; } synchronized (streamLock) { if (passThroughMode.get()) { - super.sendMessage(message); + proceedWithSendMessage(message); return; } ExtProcStreamState state = extProcStreamState.get(); if (state.isDraining() || state.isCompleted()) { if (currentProcessingMode.getRequestBodyMode() == ProcessingMode.BodySendMode.NONE) { - super.sendMessage(message); + proceedWithSendMessage(message); return; } try { @@ -1033,7 +1033,7 @@ public void sendMessage(ReqT message) { } if (currentProcessingMode.getRequestBodyMode() == ProcessingMode.BodySendMode.NONE) { - super.sendMessage(message); + proceedWithSendMessage(message); return; } @@ -1048,7 +1048,7 @@ public void sendMessage(ReqT message) { .build()) .build()); bodyMessageSentToExtProc.set(true); - super.sendMessage((ReqT) new KnownLengthInputStream(bodyByteString)); + proceedWithSendMessage((ReqT) new KnownLengthInputStream(bodyByteString)); } else { if (downstreamToSidestreamWindow <= 0 || !pendingRequestBodyMessages.isEmpty()) { pendingRequestBodyMessages.add(bodyByteString); @@ -1082,12 +1082,23 @@ private void drainPendingRequestBodyMessages() { sendRequestBodyToExtProc(body); } if (pendingRequestBodyMessages.isEmpty() && pendingHalfClose.compareAndSet(true, false)) { - halfClose(); + internalHalfClose(); + } + } + + private boolean isCancelledOrAborted() { + return cancelCalled.get() || aborted.get(); + } + + private void proceedWithSendMessage(ReqT message) { + if (isCancelledOrAborted()) { + return; } + super.sendMessage(message); } private void proceedWithHalfClose() { - if (aborted.get()) { + if (isCancelledOrAborted()) { return; } if (clientHalfCloseStartNanos > 0) { @@ -1100,10 +1111,14 @@ private void proceedWithHalfClose() { @Override public void halfClose() { - if (aborted.get()) { + checkState(!cancelCalled.get(), "call was cancelled"); + internalHalfClose(); + } + + private void internalHalfClose() { + if (isCancelledOrAborted()) { return; } - checkState(!cancelCalled.get(), "call was cancelled"); if (appHalfClosed.compareAndSet(false, true)) { clientHalfCloseStartNanos = System.nanoTime(); } @@ -1173,17 +1188,14 @@ public void halfClose() { } void cancelDownstream(@Nullable String message, @Nullable Throwable cause) { - if (!cancelCalled.get()) { - aborted.set(true); - } + aborted.set(true); delayedCall.cancel(message, cause); } @Override public void cancel(@Nullable String message, @Nullable Throwable cause) { - cancelCalled.set(true); - aborted.set(false); synchronized (streamLock) { + cancelCalled.set(true); if (markExtProcStreamFailed(extProcStreamState)) { if (extProcClientCallRequestObserver != null) { extProcClientCallRequestObserver.onError( @@ -1195,7 +1207,7 @@ public void cancel(@Nullable String message, @Nullable Throwable cause) { } } } - cancelDownstream(message, cause); + delayedCall.cancel(message, cause); } @SuppressWarnings("unchecked") @@ -1220,7 +1232,7 @@ private void handleRequestBodyResponse(BodyResponse bodyResponse) { } } if (sendImmediately) { - super.sendMessage((ReqT) new KnownLengthInputStream(body)); + proceedWithSendMessage((ReqT) new KnownLengthInputStream(body)); trySendAccumulatedWindowUpdates(); } } @@ -1338,7 +1350,7 @@ void drainPendingUpstreamBodyMessages() { if (body == null) { break; } - super.sendMessage((ReqT) new KnownLengthInputStream(body)); + proceedWithSendMessage((ReqT) new KnownLengthInputStream(body)); trySendAccumulatedWindowUpdates(); if (triggerHalfClose) { if (requestSideClosed.compareAndSet(false, true)) { @@ -1421,9 +1433,9 @@ private void drainPendingDrainingMessages() { } if (isMutated) { - super.sendMessage((ReqT) new KnownLengthInputStream((ByteString) msg)); + proceedWithSendMessage((ReqT) new KnownLengthInputStream((ByteString) msg)); } else { - super.sendMessage((ReqT) msg); + proceedWithSendMessage((ReqT) msg); } } } diff --git a/xds/src/main/java/io/grpc/xds/XdsNameResolver.java b/xds/src/main/java/io/grpc/xds/XdsNameResolver.java index ac3ea54154f..d6984be377b 100644 --- a/xds/src/main/java/io/grpc/xds/XdsNameResolver.java +++ b/xds/src/main/java/io/grpc/xds/XdsNameResolver.java @@ -18,6 +18,7 @@ import static com.google.common.base.Preconditions.checkArgument; import static com.google.common.base.Preconditions.checkNotNull; +import static com.google.common.base.Preconditions.checkState; import static io.grpc.xds.client.Bootstrapper.XDSTP_SCHEME; import com.google.common.annotations.VisibleForTesting; @@ -1294,13 +1295,10 @@ protected void sendSerializedMessage( ClientCall rawCall, MethodDescriptor method, ReqT message) { + checkState(!cancelCalled, "call was cancelled"); if (aborted) { return; } - if (cancelCalled) { - rawCall.sendMessage(null); - return; - } ByteString byteString; try { InputStream stream = method.getRequestMarshaller().stream(message); @@ -1331,13 +1329,13 @@ public void request(int numMessages) { @Override public void cancel(@Nullable String message, @Nullable Throwable cause) { - aborted = false; cancelCalled = true; delegate().cancel(message, cause); } @Override public void halfClose() { + checkState(!cancelCalled, "call was cancelled"); if (aborted) { return; } diff --git a/xds/src/test/java/io/grpc/xds/ExternalProcessorClientInterceptorTest.java b/xds/src/test/java/io/grpc/xds/ExternalProcessorClientInterceptorTest.java index ca42c529978..e4fe0dec996 100644 --- a/xds/src/test/java/io/grpc/xds/ExternalProcessorClientInterceptorTest.java +++ b/xds/src/test/java/io/grpc/xds/ExternalProcessorClientInterceptorTest.java @@ -12256,6 +12256,166 @@ public void onCompleted() { channelManager.close(); } + @Test + @SuppressWarnings("unchecked") + public void + givenBufferedRequestAndHalfClose_whenImmediateResponseOrCancel_thenCallbacksAreNoOp() + throws Exception { + ExternalProcessor proto = createBaseProto(extProcServerName) + .setProcessingMode(ProcessingMode.newBuilder() + .setRequestHeaderMode(ProcessingMode.HeaderSendMode.SEND) + .setRequestBodyMode(ProcessingMode.BodySendMode.GRPC) + .setResponseHeaderMode(ProcessingMode.HeaderSendMode.SKIP) + .setResponseTrailerMode(ProcessingMode.HeaderSendMode.SKIP) + .build()) + .build(); + ConfigOrError configOrError = + provider.parseFilterConfig(Any.pack(proto), filterContext); + ExternalProcessorFilterConfig filterConfig = configOrError.config; + + final AtomicReference> capturedExtProcListener = + new AtomicReference<>(); + ExternalProcessorGrpc.ExternalProcessorImplBase extProcImpl = + new ExternalProcessorGrpc.ExternalProcessorImplBase() { + @Override + public StreamObserver process( + final StreamObserver responseObserver) { + ((ServerCallStreamObserver) responseObserver).request(100); + return new StreamObserver() { + @Override + public void onNext(ProcessingRequest request) { + if (request.hasRequestHeaders()) { + responseObserver.onNext(ProcessingResponse.newBuilder() + .setRequestHeaders(HeadersResponse.newBuilder().build()) + .build()); + } + } + + @Override + public void onError(Throwable t) {} + + @Override + public void onCompleted() { + responseObserver.onCompleted(); + } + }; + } + }; + grpcCleanup.register(InProcessServerBuilder.forName(extProcServerName) + .addService(extProcImpl) + .directExecutor() + .build().start()); + + CachedChannelManager channelManager = new CachedChannelManager(config -> + grpcCleanup.register( + InProcessChannelBuilder.forName(extProcServerName) + .directExecutor() + .intercept(new ClientInterceptor() { + @Override + public ClientCall interceptCall( + MethodDescriptor method, + CallOptions callOptions, + Channel next) { + return new SimpleForwardingClientCall( + next.newCall(method, callOptions)) { + @Override + public void start(Listener responseListener, Metadata headers) { + capturedExtProcListener.set( + (ClientCall.Listener) responseListener); + super.start(responseListener, headers); + } + }; + } + }) + .build())); + + ExternalProcessorClientInterceptor interceptor = new ExternalProcessorClientInterceptor( + filterConfig, channelManager, scheduler, FAKE_CONTEXT); + + final AtomicInteger serverReceivedMessages = new AtomicInteger(0); + dataPlaneServiceRegistry.addService(ServerServiceDefinition.builder("test.TestService") + .addMethod(METHOD_SAY_HELLO, ServerCalls.asyncUnaryCall( + (request, responseObserver) -> { + serverReceivedMessages.incrementAndGet(); + })) + .build()); + + ManagedChannel dataPlaneChannel = + grpcCleanup.register( + InProcessChannelBuilder.forName(dataPlaneServerName).directExecutor().build()); + + CallOptions callOptions = DEFAULT_CALL_OPTIONS.withExecutor(MoreExecutors.directExecutor()); + String largeMsg = new String(new char[65536]).replace('\0', 'a'); + + // Part 1: ImmediateResponse (aborted == true) when a 2nd message and halfClose are buffered + final CountDownLatch closeLatch1 = new CountDownLatch(1); + final AtomicReference closeStatus1 = new AtomicReference<>(); + ClientCall call1 = + interceptCall(interceptor, METHOD_SAY_HELLO, callOptions, dataPlaneChannel); + call1.start(new ClientCall.Listener() { + @Override + public void onClose(Status status, Metadata trailers) { + closeStatus1.set(status); + closeLatch1.countDown(); + } + }, new Metadata()); + call1.sendMessage(largeMsg); + call1.sendMessage("buffered-msg"); + call1.halfClose(); + capturedExtProcListener.get().onMessage(ProcessingResponse.newBuilder() + .setImmediateResponse(ImmediateResponse.newBuilder() + .setGrpcStatus( + io.envoyproxy.envoy.service.ext_proc.v3.GrpcStatus.newBuilder() + .setStatus(Status.PERMISSION_DENIED.getCode().value()) + .build()) + .setDetails("Denied with buffered request") + .build()) + .build()); + assertThat(closeLatch1.await(5, TimeUnit.SECONDS)).isTrue(); + assertThat(closeStatus1.get().getCode()).isEqualTo(Status.Code.PERMISSION_DENIED); + assertThat(serverReceivedMessages.get()).isEqualTo(0); + + // Part 2: Explicit cancel() (cancelCalled == true) raced with in-flight ServerWindowUpdate + // and BodyResponse(end_of_stream = true) + final CountDownLatch closeLatch2 = new CountDownLatch(1); + ClientCall call2 = + interceptCall(interceptor, METHOD_SAY_HELLO, callOptions, dataPlaneChannel); + call2.start(new ClientCall.Listener() { + @Override + public void onClose(Status status, Metadata trailers) { + closeLatch2.countDown(); + } + }, new Metadata()); + call2.sendMessage(largeMsg); + call2.sendMessage("buffered-msg-2"); + call2.halfClose(); + call2.cancel("User cancelled", null); + assertThat(closeLatch2.await(5, TimeUnit.SECONDS)).isTrue(); + + // Deliver in-flight ServerWindowUpdate (triggers drainPendingRequestBodyMessages -> + // internalHalfClose) and BodyResponse (triggers handleRequestBodyResponse -> + // proceedWithSendMessage and proceedWithHalfClose) after cancel() + capturedExtProcListener.get().onMessage(ProcessingResponse.newBuilder() + .setServerWindowUpdate( + ProcessingResponse.ServerWindowUpdate.newBuilder() + .setWindowIncrementDownstreamToSidestream(131072) + .build()) + .setRequestBody(BodyResponse.newBuilder() + .setResponse(CommonResponse.newBuilder() + .setBodyMutation(BodyMutation.newBuilder() + .setStreamedResponse(StreamedBodyResponse.newBuilder() + .setBody(ByteString.copyFromUtf8("mutated")) + .setEndOfStream(true) + .build()) + .build()) + .build()) + .build()) + .build()); + assertThat(serverReceivedMessages.get()).isEqualTo(0); + + channelManager.close(); + } + // --- Category 21: Flow Control when side stream is full --- @Test