diff --git a/xds/src/main/java/io/grpc/xds/ExternalProcessorClientInterceptor.java b/xds/src/main/java/io/grpc/xds/ExternalProcessorClientInterceptor.java index 93a2230b9a4..bfec5e43faf 100644 --- a/xds/src/main/java/io/grpc/xds/ExternalProcessorClientInterceptor.java +++ b/xds/src/main/java/io/grpc/xds/ExternalProcessorClientInterceptor.java @@ -17,6 +17,7 @@ package io.grpc.xds; import static com.google.common.base.Preconditions.checkNotNull; +import static com.google.common.base.Preconditions.checkState; import static io.grpc.xds.internal.extproc.ExternalProcessorUtil.applyHeaderMutations; import static io.grpc.xds.internal.extproc.ExternalProcessorUtil.collectAttributes; import static io.grpc.xds.internal.extproc.ExternalProcessorUtil.markDataPlaneCallClosed; @@ -366,6 +367,8 @@ private static class DataPlaneClientCall final AtomicBoolean isProcessingTrailers = new AtomicBoolean(false); final AtomicBoolean pendingHalfClose = new AtomicBoolean(false); final AtomicBoolean bodyMessageSentToExtProc = new AtomicBoolean(false); + final AtomicBoolean aborted = new AtomicBoolean(false); + final AtomicBoolean cancelCalled = new AtomicBoolean(false); protected DataPlaneClientCall( DataPlaneDelayedCall delayedCall, @@ -994,26 +997,30 @@ public void request(int numMessages) { @SuppressWarnings("unchecked") @Override public void sendMessage(ReqT message) { + checkState(!cancelCalled.get(), "call was cancelled"); + if (aborted.get()) { + return; + } if (requestSideClosed.get()) { // External processor already closed the request stream. Discard further messages. return; } if (passThroughMode.get()) { - super.sendMessage(message); + proceedWithSendMessage(message); return; } synchronized (streamLock) { if (passThroughMode.get()) { - super.sendMessage(message); + proceedWithSendMessage(message); return; } ExtProcStreamState state = extProcStreamState.get(); if (state.isDraining() || state.isCompleted()) { if (currentProcessingMode.getRequestBodyMode() == ProcessingMode.BodySendMode.NONE) { - super.sendMessage(message); + proceedWithSendMessage(message); return; } try { @@ -1026,7 +1033,7 @@ public void sendMessage(ReqT message) { } if (currentProcessingMode.getRequestBodyMode() == ProcessingMode.BodySendMode.NONE) { - super.sendMessage(message); + proceedWithSendMessage(message); return; } @@ -1041,7 +1048,7 @@ public void sendMessage(ReqT message) { .build()) .build()); bodyMessageSentToExtProc.set(true); - super.sendMessage((ReqT) new KnownLengthInputStream(bodyByteString)); + proceedWithSendMessage((ReqT) new KnownLengthInputStream(bodyByteString)); } else { if (downstreamToSidestreamWindow <= 0 || !pendingRequestBodyMessages.isEmpty()) { pendingRequestBodyMessages.add(bodyByteString); @@ -1075,11 +1082,25 @@ private void drainPendingRequestBodyMessages() { sendRequestBodyToExtProc(body); } if (pendingRequestBodyMessages.isEmpty() && pendingHalfClose.compareAndSet(true, false)) { - halfClose(); + internalHalfClose(); + } + } + + private boolean isCancelledOrAborted() { + return cancelCalled.get() || aborted.get(); + } + + private void proceedWithSendMessage(ReqT message) { + if (isCancelledOrAborted()) { + return; } + super.sendMessage(message); } private void proceedWithHalfClose() { + if (isCancelledOrAborted()) { + return; + } if (clientHalfCloseStartNanos > 0) { long durationNanos = System.nanoTime() - clientHalfCloseStartNanos; recordDuration(clientHalfCloseDuration, durationNanos); @@ -1090,6 +1111,14 @@ private void proceedWithHalfClose() { @Override public void halfClose() { + checkState(!cancelCalled.get(), "call was cancelled"); + internalHalfClose(); + } + + private void internalHalfClose() { + if (isCancelledOrAborted()) { + return; + } if (appHalfClosed.compareAndSet(false, true)) { clientHalfCloseStartNanos = System.nanoTime(); } @@ -1159,12 +1188,14 @@ public void halfClose() { } void cancelDownstream(@Nullable String message, @Nullable Throwable cause) { + aborted.set(true); delayedCall.cancel(message, cause); } @Override public void cancel(@Nullable String message, @Nullable Throwable cause) { synchronized (streamLock) { + cancelCalled.set(true); if (markExtProcStreamFailed(extProcStreamState)) { if (extProcClientCallRequestObserver != null) { extProcClientCallRequestObserver.onError( @@ -1176,7 +1207,7 @@ public void cancel(@Nullable String message, @Nullable Throwable cause) { } } } - cancelDownstream(message, cause); + delayedCall.cancel(message, cause); } @SuppressWarnings("unchecked") @@ -1201,7 +1232,7 @@ private void handleRequestBodyResponse(BodyResponse bodyResponse) { } } if (sendImmediately) { - super.sendMessage((ReqT) new KnownLengthInputStream(body)); + proceedWithSendMessage((ReqT) new KnownLengthInputStream(body)); trySendAccumulatedWindowUpdates(); } } @@ -1319,7 +1350,7 @@ void drainPendingUpstreamBodyMessages() { if (body == null) { break; } - super.sendMessage((ReqT) new KnownLengthInputStream(body)); + proceedWithSendMessage((ReqT) new KnownLengthInputStream(body)); trySendAccumulatedWindowUpdates(); if (triggerHalfClose) { if (requestSideClosed.compareAndSet(false, true)) { @@ -1402,9 +1433,9 @@ private void drainPendingDrainingMessages() { } if (isMutated) { - super.sendMessage((ReqT) new KnownLengthInputStream((ByteString) msg)); + proceedWithSendMessage((ReqT) new KnownLengthInputStream((ByteString) msg)); } else { - super.sendMessage((ReqT) msg); + proceedWithSendMessage((ReqT) msg); } } } diff --git a/xds/src/main/java/io/grpc/xds/XdsNameResolver.java b/xds/src/main/java/io/grpc/xds/XdsNameResolver.java index 328995d22da..d6984be377b 100644 --- a/xds/src/main/java/io/grpc/xds/XdsNameResolver.java +++ b/xds/src/main/java/io/grpc/xds/XdsNameResolver.java @@ -18,6 +18,7 @@ import static com.google.common.base.Preconditions.checkArgument; import static com.google.common.base.Preconditions.checkNotNull; +import static com.google.common.base.Preconditions.checkState; import static io.grpc.xds.client.Bootstrapper.XDSTP_SCHEME; import com.google.common.annotations.VisibleForTesting; @@ -1283,35 +1284,43 @@ public void onReady() { }; } - private static void sendSerializedMessage( - ClientCall rawCall, - MethodDescriptor method, - ReqT message) { - ByteString byteString; - try { - InputStream stream = method.getRequestMarshaller().stream(message); + private abstract static class PartialForwardingClientCall + extends ClientCall { + private boolean aborted; + private boolean cancelCalled; + + protected abstract ClientCall delegate(); + + protected void sendSerializedMessage( + ClientCall rawCall, + MethodDescriptor method, + ReqT message) { + checkState(!cancelCalled, "call was cancelled"); + if (aborted) { + return; + } + ByteString byteString; try { - byteString = ExternalProcessorUtil.outboundStreamToByteString(stream); - } finally { + InputStream stream = method.getRequestMarshaller().stream(message); try { - stream.close(); - } catch (IOException ignored) { - // ignore + byteString = ExternalProcessorUtil.outboundStreamToByteString(stream); + } finally { + try { + stream.close(); + } catch (IOException ignored) { + // ignore + } } + } catch (IOException | RuntimeException e) { + aborted = true; + rawCall.cancel("Failed to read message for raw message interceptor", e); + return; + } catch (Error e) { + rawCall.cancel("Client sendMessage() failed with Error", e); + throw e; } - } catch (IOException | RuntimeException e) { - rawCall.cancel("Failed to read message for raw message interceptor", e); - return; - } catch (Error e) { - rawCall.cancel("Client sendMessage() failed with Error", e); - throw e; + rawCall.sendMessage(new KnownLengthInputStream(byteString)); } - rawCall.sendMessage(new KnownLengthInputStream(byteString)); - } - - private abstract static class PartialForwardingClientCall - extends ClientCall { - protected abstract ClientCall delegate(); @Override public void request(int numMessages) { @@ -1320,11 +1329,16 @@ public void request(int numMessages) { @Override public void cancel(@Nullable String message, @Nullable Throwable cause) { + cancelCalled = true; delegate().cancel(message, cause); } @Override public void halfClose() { + checkState(!cancelCalled, "call was cancelled"); + if (aborted) { + return; + } delegate().halfClose(); } diff --git a/xds/src/test/java/io/grpc/xds/ExternalProcessorClientInterceptorTest.java b/xds/src/test/java/io/grpc/xds/ExternalProcessorClientInterceptorTest.java index 48f0931cbc5..e4fe0dec996 100644 --- a/xds/src/test/java/io/grpc/xds/ExternalProcessorClientInterceptorTest.java +++ b/xds/src/test/java/io/grpc/xds/ExternalProcessorClientInterceptorTest.java @@ -17,6 +17,7 @@ package io.grpc.xds; import static com.google.common.truth.Truth.assertThat; +import static org.junit.Assert.assertThrows; import com.google.common.io.BaseEncoding; import com.google.common.util.concurrent.MoreExecutors; @@ -11423,6 +11424,114 @@ public void onClose(Status status, Metadata trailers) { channelManager.close(); } + @Test + @SuppressWarnings("unchecked") + public void + givenSendMessageIoException_whenSubsequentSendMessageOrHalfClose_thenNoOp() + throws Exception { + ExternalProcessor proto = createBaseProto(extProcServerName) + .setProcessingMode(ProcessingMode.newBuilder() + .setRequestHeaderMode(ProcessingMode.HeaderSendMode.SEND) + .setRequestBodyMode(ProcessingMode.BodySendMode.GRPC) + .setResponseHeaderMode(ProcessingMode.HeaderSendMode.SKIP) + .setResponseTrailerMode(ProcessingMode.HeaderSendMode.SKIP) + .build()) + .build(); + ConfigOrError configOrError = + provider.parseFilterConfig(Any.pack(proto), filterContext); + ExternalProcessorFilterConfig filterConfig = configOrError.config; + + ExternalProcessorGrpc.ExternalProcessorImplBase extProcImpl = + new ExternalProcessorGrpc.ExternalProcessorImplBase() { + @Override + public StreamObserver process( + final StreamObserver responseObserver) { + ((ServerCallStreamObserver) responseObserver).request(100); + return new StreamObserver() { + @Override + public void onNext(ProcessingRequest request) { + if (request.hasRequestHeaders()) { + responseObserver.onNext(ProcessingResponse.newBuilder() + .setRequestHeaders(HeadersResponse.newBuilder().build()) + .build()); + } + } + + @Override + public void onError(Throwable t) {} + + @Override + public void onCompleted() { + responseObserver.onCompleted(); + } + }; + } + }; + grpcCleanup.register(InProcessServerBuilder.forName(extProcServerName) + .addService(extProcImpl) + .directExecutor() + .build().start()); + + CachedChannelManager channelManager = new CachedChannelManager(config -> + grpcCleanup.register( + InProcessChannelBuilder.forName(extProcServerName).directExecutor().build())); + + ExternalProcessorClientInterceptor interceptor = new ExternalProcessorClientInterceptor( + filterConfig, channelManager, scheduler, FAKE_CONTEXT); + + dataPlaneServiceRegistry.addService(ServerServiceDefinition.builder("test.TestService") + .addMethod(METHOD_SAY_HELLO, ServerCalls.asyncUnaryCall( + (request, responseObserver) -> { + // No-op + })) + .build()); + + ManagedChannel dataPlaneChannel = + grpcCleanup.register( + InProcessChannelBuilder.forName(dataPlaneServerName).directExecutor().build()); + + CallOptions callOptions = DEFAULT_CALL_OPTIONS + .withExecutor(MoreExecutors.directExecutor()) + .withOption(XdsNameResolver.AUTO_HOST_REWRITE_KEY, false); + ClientCall rawProxyCall = + (ClientCall) (ClientCall) + interceptor.interceptCall(METHOD_SAY_HELLO, callOptions, dataPlaneChannel); + + final CountDownLatch closeLatch = new CountDownLatch(1); + final AtomicReference closeStatus = new AtomicReference<>(); + rawProxyCall.start(new ClientCall.Listener() { + @Override + public void onClose(Status status, Metadata trailers) { + closeStatus.set(status); + closeLatch.countDown(); + } + }, new Metadata()); + + IOException ioException = new IOException("Simulated stream read error"); + InputStream failingStream = new InputStream() { + @Override + public int read() throws IOException { + throw ioException; + } + }; + + // Internal cancellation during sendMessage when reading InputStream fails + rawProxyCall.sendMessage(failingStream); + + assertThat(closeLatch.await(5, TimeUnit.SECONDS)).isTrue(); + assertThat(closeStatus.get().getCode()).isEqualTo(Status.Code.CANCELLED); + assertThat(closeStatus.get().getDescription()) + .isEqualTo("Failed to serialize message for External Processor"); + assertThat(closeStatus.get().getCause()).isSameInstanceAs(ioException); + + // Subsequent sendMessage and halfClose after internal cancellation must be no-ops + // and not throw IllegalStateException. + rawProxyCall.sendMessage(new ByteArrayInputStream("ignored".getBytes(StandardCharsets.UTF_8))); + rawProxyCall.halfClose(); + + channelManager.close(); + } + // --- Category 18: Immediate Response Handling --- @Test @@ -12136,12 +12245,177 @@ public void onCompleted() { // Application cancels the RPC proxyCall.cancel("User cancelled", null); + // Subsequent sendMessage or halfClose after explicit application cancellation must throw + // IllegalStateException. + assertThrows(IllegalStateException.class, () -> proxyCall.sendMessage("after-cancel")); + assertThrows(IllegalStateException.class, () -> proxyCall.halfClose()); + // Verify sidecar stream also cancelled assertThat(cancelLatch.await(5, TimeUnit.SECONDS)).isTrue(); channelManager.close(); } + @Test + @SuppressWarnings("unchecked") + public void + givenBufferedRequestAndHalfClose_whenImmediateResponseOrCancel_thenCallbacksAreNoOp() + throws Exception { + ExternalProcessor proto = createBaseProto(extProcServerName) + .setProcessingMode(ProcessingMode.newBuilder() + .setRequestHeaderMode(ProcessingMode.HeaderSendMode.SEND) + .setRequestBodyMode(ProcessingMode.BodySendMode.GRPC) + .setResponseHeaderMode(ProcessingMode.HeaderSendMode.SKIP) + .setResponseTrailerMode(ProcessingMode.HeaderSendMode.SKIP) + .build()) + .build(); + ConfigOrError configOrError = + provider.parseFilterConfig(Any.pack(proto), filterContext); + ExternalProcessorFilterConfig filterConfig = configOrError.config; + + final AtomicReference> capturedExtProcListener = + new AtomicReference<>(); + ExternalProcessorGrpc.ExternalProcessorImplBase extProcImpl = + new ExternalProcessorGrpc.ExternalProcessorImplBase() { + @Override + public StreamObserver process( + final StreamObserver responseObserver) { + ((ServerCallStreamObserver) responseObserver).request(100); + return new StreamObserver() { + @Override + public void onNext(ProcessingRequest request) { + if (request.hasRequestHeaders()) { + responseObserver.onNext(ProcessingResponse.newBuilder() + .setRequestHeaders(HeadersResponse.newBuilder().build()) + .build()); + } + } + + @Override + public void onError(Throwable t) {} + + @Override + public void onCompleted() { + responseObserver.onCompleted(); + } + }; + } + }; + grpcCleanup.register(InProcessServerBuilder.forName(extProcServerName) + .addService(extProcImpl) + .directExecutor() + .build().start()); + + CachedChannelManager channelManager = new CachedChannelManager(config -> + grpcCleanup.register( + InProcessChannelBuilder.forName(extProcServerName) + .directExecutor() + .intercept(new ClientInterceptor() { + @Override + public ClientCall interceptCall( + MethodDescriptor method, + CallOptions callOptions, + Channel next) { + return new SimpleForwardingClientCall( + next.newCall(method, callOptions)) { + @Override + public void start(Listener responseListener, Metadata headers) { + capturedExtProcListener.set( + (ClientCall.Listener) responseListener); + super.start(responseListener, headers); + } + }; + } + }) + .build())); + + ExternalProcessorClientInterceptor interceptor = new ExternalProcessorClientInterceptor( + filterConfig, channelManager, scheduler, FAKE_CONTEXT); + + final AtomicInteger serverReceivedMessages = new AtomicInteger(0); + dataPlaneServiceRegistry.addService(ServerServiceDefinition.builder("test.TestService") + .addMethod(METHOD_SAY_HELLO, ServerCalls.asyncUnaryCall( + (request, responseObserver) -> { + serverReceivedMessages.incrementAndGet(); + })) + .build()); + + ManagedChannel dataPlaneChannel = + grpcCleanup.register( + InProcessChannelBuilder.forName(dataPlaneServerName).directExecutor().build()); + + CallOptions callOptions = DEFAULT_CALL_OPTIONS.withExecutor(MoreExecutors.directExecutor()); + String largeMsg = new String(new char[65536]).replace('\0', 'a'); + + // Part 1: ImmediateResponse (aborted == true) when a 2nd message and halfClose are buffered + final CountDownLatch closeLatch1 = new CountDownLatch(1); + final AtomicReference closeStatus1 = new AtomicReference<>(); + ClientCall call1 = + interceptCall(interceptor, METHOD_SAY_HELLO, callOptions, dataPlaneChannel); + call1.start(new ClientCall.Listener() { + @Override + public void onClose(Status status, Metadata trailers) { + closeStatus1.set(status); + closeLatch1.countDown(); + } + }, new Metadata()); + call1.sendMessage(largeMsg); + call1.sendMessage("buffered-msg"); + call1.halfClose(); + capturedExtProcListener.get().onMessage(ProcessingResponse.newBuilder() + .setImmediateResponse(ImmediateResponse.newBuilder() + .setGrpcStatus( + io.envoyproxy.envoy.service.ext_proc.v3.GrpcStatus.newBuilder() + .setStatus(Status.PERMISSION_DENIED.getCode().value()) + .build()) + .setDetails("Denied with buffered request") + .build()) + .build()); + assertThat(closeLatch1.await(5, TimeUnit.SECONDS)).isTrue(); + assertThat(closeStatus1.get().getCode()).isEqualTo(Status.Code.PERMISSION_DENIED); + assertThat(serverReceivedMessages.get()).isEqualTo(0); + + // Part 2: Explicit cancel() (cancelCalled == true) raced with in-flight ServerWindowUpdate + // and BodyResponse(end_of_stream = true) + final CountDownLatch closeLatch2 = new CountDownLatch(1); + ClientCall call2 = + interceptCall(interceptor, METHOD_SAY_HELLO, callOptions, dataPlaneChannel); + call2.start(new ClientCall.Listener() { + @Override + public void onClose(Status status, Metadata trailers) { + closeLatch2.countDown(); + } + }, new Metadata()); + call2.sendMessage(largeMsg); + call2.sendMessage("buffered-msg-2"); + call2.halfClose(); + call2.cancel("User cancelled", null); + assertThat(closeLatch2.await(5, TimeUnit.SECONDS)).isTrue(); + + // Deliver in-flight ServerWindowUpdate (triggers drainPendingRequestBodyMessages -> + // internalHalfClose) and BodyResponse (triggers handleRequestBodyResponse -> + // proceedWithSendMessage and proceedWithHalfClose) after cancel() + capturedExtProcListener.get().onMessage(ProcessingResponse.newBuilder() + .setServerWindowUpdate( + ProcessingResponse.ServerWindowUpdate.newBuilder() + .setWindowIncrementDownstreamToSidestream(131072) + .build()) + .setRequestBody(BodyResponse.newBuilder() + .setResponse(CommonResponse.newBuilder() + .setBodyMutation(BodyMutation.newBuilder() + .setStreamedResponse(StreamedBodyResponse.newBuilder() + .setBody(ByteString.copyFromUtf8("mutated")) + .setEndOfStream(true) + .build()) + .build()) + .build()) + .build()) + .build()); + assertThat(serverReceivedMessages.get()).isEqualTo(0); + + channelManager.close(); + } + // --- Category 21: Flow Control when side stream is full --- @Test diff --git a/xds/src/test/java/io/grpc/xds/XdsNameResolverTest.java b/xds/src/test/java/io/grpc/xds/XdsNameResolverTest.java index e57d0860588..b8996baaa39 100644 --- a/xds/src/test/java/io/grpc/xds/XdsNameResolverTest.java +++ b/xds/src/test/java/io/grpc/xds/XdsNameResolverTest.java @@ -23,6 +23,7 @@ import static io.grpc.xds.FaultFilter.HEADER_ABORT_PERCENTAGE_KEY; import static io.grpc.xds.FaultFilter.HEADER_DELAY_KEY; import static io.grpc.xds.FaultFilter.HEADER_DELAY_PERCENTAGE_KEY; +import static org.junit.Assert.assertThrows; import static org.mockito.AdditionalAnswers.delegatesTo; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyInt; @@ -75,6 +76,7 @@ import io.grpc.Status; import io.grpc.Status.Code; import io.grpc.StatusOr; +import io.grpc.StatusRuntimeException; import io.grpc.SynchronizationContext; import io.grpc.inprocess.InProcessChannelBuilder; import io.grpc.inprocess.InProcessServerBuilder; @@ -3227,6 +3229,10 @@ public String parse(InputStream stream) { ClientCall.Listener listener = mock(ClientCall.Listener.class); call.start(listener, new Metadata()); call.sendMessage("World"); + // Subsequent sendMessage and halfClose after internal sendMessage cancellation are no-ops + // and must not throw IllegalStateException. + call.sendMessage("World2"); + call.halfClose(); ArgumentCaptor statusCaptor = ArgumentCaptor.forClass(Status.class); verify(listener).onClose(statusCaptor.capture(), any(Metadata.class)); @@ -3235,6 +3241,29 @@ public String parse(InputStream stream) { assertThat(status.getDescription()) .isEqualTo("Failed to read message for raw message interceptor"); assertThat(status.getCause()).isSameInstanceAs(ioException); + + // Verify ClientCalls.blockingUnaryCall (which calls sendMessage followed by halfClose) + // surfaces StatusRuntimeException(CANCELLED) rather than IllegalStateException. + StatusRuntimeException sre = assertThrows( + StatusRuntimeException.class, + () -> ClientCalls.blockingUnaryCall( + interceptedChannel, failingRequestMethod, CallOptions.DEFAULT, "World")); + assertThat(sre.getStatus().getCode()).isEqualTo(Status.Code.CANCELLED); + assertThat(sre.getStatus().getDescription()) + .isEqualTo("Failed to read message for raw message interceptor"); + assertThat(sre.getStatus().getCause()).isSameInstanceAs(ioException); + + // Verify explicit application cancellation still causes subsequent sendMessage and halfClose + // to throw IllegalStateException. + ClientCall explicitlyCancelledCall = + interceptedChannel.newCall(failingRequestMethod, CallOptions.DEFAULT); + @SuppressWarnings("unchecked") + ClientCall.Listener explicitCancelListener = mock(ClientCall.Listener.class); + explicitlyCancelledCall.start(explicitCancelListener, new Metadata()); + explicitlyCancelledCall.cancel("Application cancelled", null); + assertThrows( + IllegalStateException.class, () -> explicitlyCancelledCall.sendMessage("World")); + assertThrows(IllegalStateException.class, explicitlyCancelledCall::halfClose); } }