Skip to content

Commit dc0370c

Browse files
committed
Evaluate flow control violations based on the window size before applying the current message size, rather than after.
1 parent 5855bc8 commit dc0370c

2 files changed

Lines changed: 211 additions & 21 deletions

File tree

‎xds/src/main/java/io/grpc/xds/ExternalProcessorClientInterceptor.java‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -765,14 +765,14 @@ private void trySendAccumulatedWindowUpdates() {
765765
if (extProcStreamState.get().isCompleted()) {
766766
return;
767767
}
768-
long incrementUpstream = (super.isReady() || sidestreamToUpstreamWindow < 0)
768+
long incrementUpstream = (super.isReady() || sidestreamToUpstreamWindow <= 0)
769769
? accumulatedWindowUpdateSidestreamToUpstream : 0;
770770
long incrementDownstream = accumulatedWindowUpdateSidestreamToDownstream;
771771

772772
boolean shouldSend = (incrementUpstream >= WINDOW_UPDATE_THRESHOLD)
773773
|| (incrementDownstream >= WINDOW_UPDATE_THRESHOLD)
774-
|| (sidestreamToUpstreamWindow < 0 && accumulatedWindowUpdateSidestreamToUpstream > 0)
775-
|| (sidestreamToDownstreamWindow < 0 && accumulatedWindowUpdateSidestreamToDownstream > 0);
774+
|| (sidestreamToUpstreamWindow <= 0 && accumulatedWindowUpdateSidestreamToUpstream > 0)
775+
|| (sidestreamToDownstreamWindow <= 0 && accumulatedWindowUpdateSidestreamToDownstream > 0);
776776

777777
if (shouldSend) {
778778
accumulatedWindowUpdateSidestreamToUpstream -= incrementUpstream;
@@ -1086,7 +1086,7 @@ private void handleRequestBodyResponse(BodyResponse bodyResponse) {
10861086
com.google.protobuf.ByteString body = streamed.getBody();
10871087
if (!config.getObservabilityMode() && currentProcessingMode.getRequestBodyMode() == ProcessingMode.BodySendMode.GRPC) {
10881088
synchronized (streamLock) {
1089-
if (body.size() > sidestreamToUpstreamWindow) {
1089+
if (sidestreamToUpstreamWindow <= 0) {
10901090
internalOnError(Status.INTERNAL
10911091
.withDescription("Flow control violation: received client body from ext_proc when window is closed")
10921092
.asRuntimeException());
@@ -1120,7 +1120,7 @@ private void handleResponseBodyResponse(
11201120
final int bodySize = body.size();
11211121
if (!config.getObservabilityMode() && currentProcessingMode.getResponseBodyMode() == ProcessingMode.BodySendMode.GRPC) {
11221122
synchronized (streamLock) {
1123-
if (bodySize > sidestreamToDownstreamWindow) {
1123+
if (sidestreamToDownstreamWindow <= 0) {
11241124
internalOnError(Status.INTERNAL
11251125
.withDescription("Flow control violation: received server body from ext_proc when window is closed")
11261126
.asRuntimeException());

‎xds/src/test/java/io/grpc/xds/ExternalProcessorClientInterceptorTest.java‎

Lines changed: 206 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -10988,17 +10988,12 @@ public void onNext(ProcessingRequest request) {
1098810988
firstBodyLatch.countDown();
1098910989
} else if (request.hasRequestBody()) {
1099010990
if (firstBodyLatch.getCount() > 0) {
10991-
// This is the first body (70000 bytes). Truncate to 40000 to fit return window (65536) and trigger update.
10992-
ByteString bodyToSend = request.getRequestBody().getBody();
10993-
if (bodyToSend.size() == 70000) {
10994-
bodyToSend = bodyToSend.substring(0, 40000);
10995-
}
1099610991
responseObserver.onNext(ProcessingResponse.newBuilder()
1099710992
.setRequestBody(BodyResponse.newBuilder()
1099810993
.setResponse(CommonResponse.newBuilder()
1099910994
.setBodyMutation(BodyMutation.newBuilder()
1100010995
.setStreamedResponse(StreamedBodyResponse.newBuilder()
11001-
.setBody(bodyToSend)
10996+
.setBody(request.getRequestBody().getBody())
1100210997
.build())
1100310998
.build())
1100410999
.build())
@@ -11117,7 +11112,7 @@ public void onCompleted() {
1111711112
assertThat(proxyCall.isReady()).isTrue();
1111811113

1111911114
// Verify both messages reached the backend service
11120-
assertThat(dataPlaneReceivedMessages).containsExactly(largeMessage70k.substring(0, 40000), largeMessage30k).inOrder();
11115+
assertThat(dataPlaneReceivedMessages).containsExactly(largeMessage70k, largeMessage30k).inOrder();
1112111116

1112211117
proxyCall.cancel("Cleanup", null);
1112311118
channelManager.close();
@@ -11187,9 +11182,7 @@ public void onNext(ProcessingRequest request) {
1118711182
} else if (request.hasResponseBody()) {
1118811183
com.google.protobuf.ByteString originalBody = request.getResponseBody().getBody();
1118911184
com.google.protobuf.ByteString bodyToSend = originalBody;
11190-
if (originalBody.size() == 70000) {
11191-
bodyToSend = bodyToSend.substring(0, 40000);
11192-
}
11185+
// Return the original 70,000 bytes as-is
1119311186
responseObserver.onNext(ProcessingResponse.newBuilder()
1119411187
.setResponseBody(BodyResponse.newBuilder()
1119511188
.setResponse(CommonResponse.newBuilder()
@@ -11334,7 +11327,7 @@ public void testSidestreamToDownstreamFlowControl_Violations() throws Exception
1133411327
assertThat(configOrError.errorDetail).isNull();
1133511328
ExternalProcessorFilterConfig filterConfig = configOrError.config;
1133611329

11337-
final String mutatedMessageTooLarge = new String(new char[70000]).replace('\0', 'd');
11330+
final String mutatedMessage = new String(new char[10000]).replace('\0', 'd');
1133811331
final CountDownLatch callClosedLatch = new CountDownLatch(1);
1133911332
final AtomicReference<Status> capturedStatus = new AtomicReference<>();
1134011333

@@ -11368,13 +11361,12 @@ public void onNext(ProcessingRequest request) {
1136811361
.setResponseHeaders(HeadersResponse.newBuilder().build())
1136911362
.build());
1137011363
} else if (request.hasResponseBody()) {
11371-
// Respond with a mutated body that is larger than the initial window size of 65536
1137211364
responseObserver.onNext(ProcessingResponse.newBuilder()
1137311365
.setResponseBody(BodyResponse.newBuilder()
1137411366
.setResponse(CommonResponse.newBuilder()
1137511367
.setBodyMutation(BodyMutation.newBuilder()
1137611368
.setStreamedResponse(StreamedBodyResponse.newBuilder()
11377-
.setBody(ByteString.copyFromUtf8(mutatedMessageTooLarge))
11369+
.setBody(ByteString.copyFromUtf8(mutatedMessage))
1137811370
.build())
1137911371
.build())
1138011372
.build())
@@ -11444,11 +11436,18 @@ public void onClose(Status status, Metadata trailers) {
1144411436

1144511437
proxyCall.sendMessage("Client Msg");
1144611438

11439+
// Retrieve the DataPlaneClientCall and set the sidestreamToDownstreamWindow to 0 via reflection
11440+
Object dataPlaneCall = unwrapToDataPlaneClientCall(proxyCall);
11441+
java.lang.reflect.Field windowField = dataPlaneCall.getClass().getDeclaredField("sidestreamToDownstreamWindow");
11442+
windowField.setAccessible(true);
11443+
windowField.setLong(dataPlaneCall, 0);
11444+
1144711445
// Send a response from upstream to trigger headers and then the body response
1144811446
StreamObserver<String> upstreamResponseObserver = dataPlaneResponseObserverRef.get();
1144911447
upstreamResponseObserver.onNext("Response Msg");
1145011448

1145111449
assertThat(callClosedLatch.await(5, TimeUnit.SECONDS)).isTrue();
11450+
1145211451
// The call should fail immediately with INTERNAL error code due to flow control violation
1145311452
assertThat(capturedStatus.get().getCode()).isEqualTo(Status.Code.INTERNAL);
1145411453
assertThat(capturedStatus.get().getDescription()).isEqualTo("External processor stream failed");
@@ -11482,7 +11481,7 @@ public void testSidestreamToUpstreamFlowControl_Violations() throws Exception {
1148211481
assertThat(configOrError.errorDetail).isNull();
1148311482
ExternalProcessorFilterConfig filterConfig = configOrError.config;
1148411483

11485-
final String mutatedMessageTooLarge = new String(new char[70000]).replace('\0', 'c');
11484+
final String mutatedMessage = new String(new char[10000]).replace('\0', 'c');
1148611485
final CountDownLatch callClosedLatch = new CountDownLatch(1);
1148711486
final AtomicReference<Status> capturedStatus = new AtomicReference<>();
1148811487

@@ -11500,13 +11499,12 @@ public void onNext(ProcessingRequest request) {
1150011499
.setRequestHeaders(HeadersResponse.newBuilder().build())
1150111500
.build());
1150211501
} else if (request.hasRequestBody()) {
11503-
// Respond with a mutated body that is larger than the initial window size of 65536
1150411502
responseObserver.onNext(ProcessingResponse.newBuilder()
1150511503
.setRequestBody(BodyResponse.newBuilder()
1150611504
.setResponse(CommonResponse.newBuilder()
1150711505
.setBodyMutation(BodyMutation.newBuilder()
1150811506
.setStreamedResponse(StreamedBodyResponse.newBuilder()
11509-
.setBody(ByteString.copyFromUtf8(mutatedMessageTooLarge))
11507+
.setBody(ByteString.copyFromUtf8(mutatedMessage))
1151011508
.build())
1151111509
.build())
1151211510
.build())
@@ -11562,9 +11560,16 @@ public void onClose(Status status, Metadata trailers) {
1156211560
}
1156311561
}, new Metadata());
1156411562

11563+
// Retrieve the DataPlaneClientCall and set the sidestreamToUpstreamWindow to 0 via reflection
11564+
Object dataPlaneCall = unwrapToDataPlaneClientCall(proxyCall);
11565+
java.lang.reflect.Field windowField = dataPlaneCall.getClass().getDeclaredField("sidestreamToUpstreamWindow");
11566+
windowField.setAccessible(true);
11567+
windowField.setLong(dataPlaneCall, 0);
11568+
1156511569
proxyCall.sendMessage("Message 1");
1156611570

1156711571
assertThat(callClosedLatch.await(5, TimeUnit.SECONDS)).isTrue();
11572+
1156811573
// The call should fail immediately with INTERNAL error code due to flow control violation
1156911574
assertThat(capturedStatus.get().getCode()).isEqualTo(Status.Code.INTERNAL);
1157011575
assertThat(capturedStatus.get().getDescription()).isEqualTo("External processor stream failed");
@@ -12604,6 +12609,155 @@ public void onMessage(String message) {
1260412609
channelManager.close();
1260512610
}
1260612611

12612+
@Test
12613+
@SuppressWarnings("unchecked")
12614+
public void testClientWindowUpdateSentImmediatelyWhenSidestreamToUpstreamWindowExhaustedEvenIfClientCallNotReady() throws Exception {
12615+
ExternalProcessor proto = ExternalProcessor.newBuilder()
12616+
.setGrpcService(GrpcService.newBuilder()
12617+
.setGoogleGrpc(GrpcService.GoogleGrpc.newBuilder()
12618+
.setTargetUri("in-process:///" + extProcServerName)
12619+
.addChannelCredentialsPlugin(Any.newBuilder()
12620+
.setTypeUrl("type.googleapis.com/envoy.extensions.grpc_service.channel_credentials.insecure.v3.InsecureCredentials")
12621+
.build())
12622+
.build())
12623+
.build())
12624+
.setProcessingMode(ProcessingMode.newBuilder()
12625+
.setRequestBodyMode(ProcessingMode.BodySendMode.GRPC)
12626+
.setResponseBodyMode(ProcessingMode.BodySendMode.GRPC)
12627+
.setResponseHeaderMode(ProcessingMode.HeaderSendMode.SEND)
12628+
.setResponseTrailerMode(ProcessingMode.HeaderSendMode.SEND)
12629+
.build())
12630+
.build();
12631+
ConfigOrError<ExternalProcessorFilterConfig> configOrError =
12632+
provider.parseFilterConfig(Any.pack(proto), filterContext);
12633+
assertThat(configOrError.errorDetail).isNull();
12634+
ExternalProcessorFilterConfig filterConfig = configOrError.config;
12635+
12636+
final List<ProcessingRequest> receivedRequests = new CopyOnWriteArrayList<>();
12637+
final CountDownLatch extProcLatch = new CountDownLatch(2); // Headers + Request Body
12638+
final AtomicReference<StreamObserver<ProcessingResponse>> responseObserverRef = new AtomicReference<>();
12639+
12640+
ExternalProcessorGrpc.ExternalProcessorImplBase extProcImpl =
12641+
new ExternalProcessorGrpc.ExternalProcessorImplBase() {
12642+
@Override
12643+
public StreamObserver<ProcessingRequest> process(
12644+
final StreamObserver<ProcessingResponse> responseObserver) {
12645+
responseObserverRef.set(responseObserver);
12646+
((ServerCallStreamObserver<ProcessingResponse>) responseObserver).request(100);
12647+
return new StreamObserver<ProcessingRequest>() {
12648+
@Override
12649+
public void onNext(ProcessingRequest request) {
12650+
receivedRequests.add(request);
12651+
if (request.hasRequestHeaders()) {
12652+
extProcLatch.countDown();
12653+
responseObserver.onNext(ProcessingResponse.newBuilder()
12654+
.setRequestHeaders(HeadersResponse.newBuilder().build())
12655+
.build());
12656+
} else if (request.hasRequestBody()) {
12657+
extProcLatch.countDown();
12658+
// Mutate request body and send back 65536 bytes. This completely exhausts the return window (starts at 65536).
12659+
responseObserver.onNext(ProcessingResponse.newBuilder()
12660+
.setRequestBody(BodyResponse.newBuilder()
12661+
.setResponse(CommonResponse.newBuilder()
12662+
.setBodyMutation(BodyMutation.newBuilder()
12663+
.setStreamedResponse(StreamedBodyResponse.newBuilder()
12664+
.setBody(ByteString.copyFrom(new byte[65536]))
12665+
.build())
12666+
.build())
12667+
.build())
12668+
.build())
12669+
.build());
12670+
}
12671+
}
12672+
12673+
@Override
12674+
public void onError(Throwable t) {}
12675+
12676+
@Override
12677+
public void onCompleted() {
12678+
responseObserver.onCompleted();
12679+
}
12680+
};
12681+
}
12682+
};
12683+
12684+
String uniqueExtProcServerName = InProcessServerBuilder.generateName();
12685+
grpcCleanup.register(InProcessServerBuilder.forName(uniqueExtProcServerName)
12686+
.addService(extProcImpl)
12687+
.directExecutor()
12688+
.build().start());
12689+
12690+
CachedChannelManager channelManager = new CachedChannelManager(config -> {
12691+
return grpcCleanup.register(
12692+
InProcessChannelBuilder.forName(uniqueExtProcServerName).directExecutor().build());
12693+
});
12694+
12695+
ExternalProcessorClientInterceptor interceptor = new ExternalProcessorClientInterceptor(
12696+
filterConfig, channelManager, scheduler, FAKE_CONTEXT);
12697+
12698+
dataPlaneServiceRegistry.addService(ServerServiceDefinition.builder("test.TestService")
12699+
.addMethod(METHOD_CLIENT_STREAMING, ServerCalls.asyncClientStreamingCall(
12700+
new ServerCalls.ClientStreamingMethod<String, String>() {
12701+
@Override
12702+
public StreamObserver<String> invoke(StreamObserver<String> responseObserver) {
12703+
return new StreamObserver<String>() {
12704+
@Override
12705+
public void onNext(String value) {}
12706+
@Override
12707+
public void onError(Throwable t) {}
12708+
@Override
12709+
public void onCompleted() {
12710+
responseObserver.onNext("Response");
12711+
responseObserver.onCompleted();
12712+
}
12713+
};
12714+
}
12715+
}))
12716+
.build());
12717+
12718+
ManagedChannel dataPlaneChannel = grpcCleanup.register(
12719+
InProcessChannelBuilder.forName(dataPlaneServerName)
12720+
.intercept(new ClientInterceptor() {
12721+
@Override
12722+
public <ReqT, RespT> ClientCall<ReqT, RespT> interceptCall(
12723+
MethodDescriptor<ReqT, RespT> method, CallOptions callOptions, Channel next) {
12724+
return new io.grpc.ForwardingClientCall.SimpleForwardingClientCall<ReqT, RespT>(
12725+
next.newCall(method, callOptions)) {
12726+
@Override
12727+
public boolean isReady() {
12728+
return false; // Force client call to be not ready
12729+
}
12730+
};
12731+
}
12732+
})
12733+
.directExecutor()
12734+
.build());
12735+
12736+
ClientCall<String, String> proxyCall =
12737+
interceptCall(interceptor, METHOD_CLIENT_STREAMING, DEFAULT_CALL_OPTIONS.withExecutor(MoreExecutors.directExecutor()), dataPlaneChannel);
12738+
12739+
proxyCall.start(new ClientCall.Listener<String>() {}, new Metadata());
12740+
12741+
// Send 10k message to ext_proc.
12742+
String body10k = new String(new char[10000]).replace('\0', 'a');
12743+
proxyCall.sendMessage(body10k);
12744+
12745+
assertThat(extProcLatch.await(5, TimeUnit.SECONDS)).isTrue();
12746+
12747+
// Since the window is exhausted (65536 bytes returned from sidecar), the ClientWindowUpdate
12748+
// should be sent immediately even though the client call is not ready (isReady() == false).
12749+
long startTime = System.currentTimeMillis();
12750+
while (receivedRequests.size() < 3 && System.currentTimeMillis() - startTime < 5000) {
12751+
Thread.sleep(10);
12752+
}
12753+
assertThat(receivedRequests).hasSize(3);
12754+
assertThat(receivedRequests.get(2).hasClientWindowUpdate()).isTrue();
12755+
assertThat(receivedRequests.get(2).getClientWindowUpdate().getWindowIncrementSidestreamToUpstream()).isEqualTo(65536);
12756+
12757+
proxyCall.cancel("Cleanup", null);
12758+
channelManager.close();
12759+
}
12760+
1260712761
// --- Category 21: Streaming Completeness (Client & Bi-Di) ---
1260812762

1260912763
@Test
@@ -17182,4 +17336,40 @@ private void shutdownAndAwaitTermination(ExecutorService executor) {
1718217336
Thread.currentThread().interrupt();
1718317337
}
1718417338
}
17339+
17340+
private static Object unwrapToDataPlaneClientCall(ClientCall<?, ?> call) throws Exception {
17341+
ClientCall<?, ?> current = call;
17342+
while (current != null) {
17343+
if (current.getClass().getName().equals("io.grpc.xds.ExternalProcessorClientInterceptor$DataPlaneClientCall")) {
17344+
return current;
17345+
}
17346+
java.lang.reflect.Field delegateField = null;
17347+
Class<?> c = current.getClass();
17348+
while (c != null) {
17349+
try {
17350+
delegateField = c.getDeclaredField("delegate");
17351+
break;
17352+
} catch (NoSuchFieldException e) {
17353+
try {
17354+
delegateField = c.getDeclaredField("val$rawCall");
17355+
break;
17356+
} catch (NoSuchFieldException e2) {
17357+
c = c.getSuperclass();
17358+
}
17359+
}
17360+
}
17361+
if (delegateField == null) {
17362+
try {
17363+
java.lang.reflect.Method getDelegateMethod = current.getClass().getMethod("delegate");
17364+
current = (ClientCall<?, ?>) getDelegateMethod.invoke(current);
17365+
continue;
17366+
} catch (Throwable ignored) {}
17367+
throw new IllegalArgumentException("Could not find delegate or val$rawCall field in " + current.getClass().getName());
17368+
}
17369+
delegateField.setAccessible(true);
17370+
current = (ClientCall<?, ?>) delegateField.get(current);
17371+
}
17372+
throw new IllegalArgumentException("Could not unwrap to DataPlaneClientCall: " + call);
17373+
}
1718517374
}
17375+

0 commit comments

Comments
 (0)