Skip to content

Commit ceef8df

Browse files
committed
xds: Fix ext_proc client flow control to gate window updates by transport readiness
- Gated the transmission of `ClientWindowUpdate` messages by transport readiness. If the upstream client call is not ready (`isReady() == false`), the upstream window increment is kept at 0 to enforce backpressure. - Modified `trySendAccumulatedWindowUpdates()` to avoid transmitting redundant, zero-increment window updates when transport readiness prevents replenishment. A `ClientWindowUpdate` is now sent only if at least one window increment is positive (`incrementUpstream > 0 || incrementDownstream > 0`). - Updated `testClientWindowUpdateSentImmediatelyOnWindowExhaustion` to verify that no redundant window updates are sent while the transport is not ready, and that the accumulated `65536` increment is correctly flushed immediately upon calling `onReady()`. - Fixed checkstyle violations (missing line separator) in the test code.
1 parent dc0370c commit ceef8df

2 files changed

Lines changed: 925 additions & 269 deletions

File tree

‎xds/src/main/java/io/grpc/xds/ExternalProcessorClientInterceptor.java‎

Lines changed: 78 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -311,7 +311,13 @@ private static class DataPlaneClientCall
311311
private long sidestreamToDownstreamWindow = DEFAULT_INITIAL_WINDOW_SIZE;
312312

313313
// Buffered request body messages from downstream
314-
private final Queue<ByteString> pendingRequestBodyMessages = new java.util.concurrent.ConcurrentLinkedQueue<>();
314+
private final Queue<ByteString> pendingRequestBodyMessages =
315+
new java.util.concurrent.ConcurrentLinkedQueue<>();
316+
317+
// Buffered mutated response bodies from sidecar
318+
private int downstreamRequestsPending = 0;
319+
private final Queue<ByteString> pendingMutatedResponseBodies =
320+
new java.util.concurrent.ConcurrentLinkedQueue<>();
315321

316322
private static final long WINDOW_UPDATE_THRESHOLD = DEFAULT_INITIAL_WINDOW_SIZE / 2;
317323

@@ -737,7 +743,7 @@ private void sendToExtProcDirect(ProcessingRequest request) {
737743

738744
void mergeAccumulatedWindowUpdates(ProcessingRequest.Builder requestBuilder) {
739745
synchronized (streamLock) {
740-
long incrementUpstream = (super.isReady() || sidestreamToUpstreamWindow < 0)
746+
long incrementUpstream = super.isReady()
741747
? accumulatedWindowUpdateSidestreamToUpstream : 0;
742748
long incrementDownstream = accumulatedWindowUpdateSidestreamToDownstream;
743749

@@ -757,22 +763,20 @@ void mergeAccumulatedWindowUpdates(ProcessingRequest.Builder requestBuilder) {
757763

758764
private void trySendAccumulatedWindowUpdates() {
759765
synchronized (streamLock) {
760-
System.out.println("trySendAccumulatedWindowUpdates: completed=" + extProcStreamState.get().isCompleted() +
761-
", super.isReady()=" + super.isReady() +
762-
", sidestreamToUpstreamWindow=" + sidestreamToUpstreamWindow +
763-
", accumulatedUpstream=" + accumulatedWindowUpdateSidestreamToUpstream +
764-
", accumulatedDownstream=" + accumulatedWindowUpdateSidestreamToDownstream);
765766
if (extProcStreamState.get().isCompleted()) {
766767
return;
767768
}
768-
long incrementUpstream = (super.isReady() || sidestreamToUpstreamWindow <= 0)
769+
long incrementUpstream = super.isReady()
769770
? accumulatedWindowUpdateSidestreamToUpstream : 0;
770771
long incrementDownstream = accumulatedWindowUpdateSidestreamToDownstream;
771772

772-
boolean shouldSend = (incrementUpstream >= WINDOW_UPDATE_THRESHOLD)
773+
boolean shouldSend = (incrementUpstream > 0 || incrementDownstream > 0) && (
774+
(incrementUpstream >= WINDOW_UPDATE_THRESHOLD)
773775
|| (incrementDownstream >= WINDOW_UPDATE_THRESHOLD)
774776
|| (sidestreamToUpstreamWindow <= 0 && accumulatedWindowUpdateSidestreamToUpstream > 0)
775-
|| (sidestreamToDownstreamWindow <= 0 && accumulatedWindowUpdateSidestreamToDownstream > 0);
777+
|| (sidestreamToDownstreamWindow <= 0
778+
&& accumulatedWindowUpdateSidestreamToDownstream > 0)
779+
);
776780

777781
if (shouldSend) {
778782
accumulatedWindowUpdateSidestreamToUpstream -= incrementUpstream;
@@ -889,7 +893,8 @@ public boolean isReady() {
889893
if (config.getObservabilityMode()) {
890894
return super.isReady() && sidecarReady;
891895
}
892-
return downstreamToSidestreamWindow > 0 && sidecarReady && pendingRequestBodyMessages.isEmpty();
896+
return downstreamToSidestreamWindow > 0 && sidecarReady
897+
&& pendingRequestBodyMessages.isEmpty();
893898
}
894899
}
895900

@@ -904,18 +909,18 @@ public void request(int numMessages) {
904909
super.request(numMessages);
905910
return;
906911
}
907-
if (!isSidecarReady()) {
908-
pendingRequests.addAndGet(numMessages);
909-
return;
910-
}
911912
if (config.getObservabilityMode()
912913
|| currentProcessingMode.getResponseBodyMode() != ProcessingMode.BodySendMode.GRPC) {
913914
super.request(numMessages);
914915
return;
915916
}
916917
synchronized (streamLock) {
917918
pendingRequests.addAndGet(numMessages);
918-
drainPendingRequests();
919+
downstreamRequestsPending += numMessages;
920+
if (isSidecarReady()) {
921+
drainPendingMutatedResponseBodies();
922+
drainPendingRequests();
923+
}
919924
}
920925
}
921926

@@ -1019,10 +1024,6 @@ private void proceedWithHalfClose() {
10191024

10201025
@Override
10211026
public void halfClose() {
1022-
System.out.println("halfClose: passThrough=" + passThroughMode.get() +
1023-
", extProcStreamCompleted=" + extProcStreamState.get().isCompleted() +
1024-
", extProcStreamDraining=" + extProcStreamState.get().isDraining() +
1025-
", bodyMode=" + currentProcessingMode.getRequestBodyMode());
10261027
clientHalfCloseStartNanos = System.nanoTime();
10271028
if (passThroughMode.get()) {
10281029
if (requestSideClosed.compareAndSet(false, true)) {
@@ -1084,11 +1085,14 @@ private void handleRequestBodyResponse(BodyResponse bodyResponse) {
10841085
StreamedBodyResponse streamed = mutation.getStreamedResponse();
10851086
if (!streamed.getEndOfStreamWithoutMessage()) {
10861087
com.google.protobuf.ByteString body = streamed.getBody();
1087-
if (!config.getObservabilityMode() && currentProcessingMode.getRequestBodyMode() == ProcessingMode.BodySendMode.GRPC) {
1088+
if (!config.getObservabilityMode()
1089+
&& currentProcessingMode.getRequestBodyMode() == ProcessingMode.BodySendMode.GRPC) {
10881090
synchronized (streamLock) {
10891091
if (sidestreamToUpstreamWindow <= 0) {
10901092
internalOnError(Status.INTERNAL
1091-
.withDescription("Flow control violation: received client body from ext_proc when window is closed")
1093+
.withDescription(
1094+
"Flow control violation: received client body from ext_proc "
1095+
+ "when window is closed")
10921096
.asRuntimeException());
10931097
return;
10941098
}
@@ -1097,7 +1101,8 @@ private void handleRequestBodyResponse(BodyResponse bodyResponse) {
10971101
}
10981102
}
10991103
super.sendMessage(new KnownLengthInputStream(body));
1100-
if (!config.getObservabilityMode() && currentProcessingMode.getRequestBodyMode() == ProcessingMode.BodySendMode.GRPC) {
1104+
if (!config.getObservabilityMode()
1105+
&& currentProcessingMode.getRequestBodyMode() == ProcessingMode.BodySendMode.GRPC) {
11011106
trySendAccumulatedWindowUpdates();
11021107
}
11031108
}
@@ -1118,29 +1123,74 @@ private void handleResponseBodyResponse(
11181123
StreamedBodyResponse streamed = mutation.getStreamedResponse();
11191124
com.google.protobuf.ByteString body = streamed.getBody();
11201125
final int bodySize = body.size();
1121-
if (!config.getObservabilityMode() && currentProcessingMode.getResponseBodyMode() == ProcessingMode.BodySendMode.GRPC) {
1126+
if (!config.getObservabilityMode()
1127+
&& currentProcessingMode.getResponseBodyMode() == ProcessingMode.BodySendMode.GRPC) {
11221128
synchronized (streamLock) {
11231129
if (sidestreamToDownstreamWindow <= 0) {
11241130
internalOnError(Status.INTERNAL
1125-
.withDescription("Flow control violation: received server body from ext_proc when window is closed")
1131+
.withDescription(
1132+
"Flow control violation: received server body from ext_proc "
1133+
+ "when window is closed")
11261134
.asRuntimeException());
11271135
return;
11281136
}
11291137
sidestreamToDownstreamWindow -= bodySize;
11301138
}
11311139
}
1140+
deliverResponseBody(body, listener);
1141+
}
1142+
}
1143+
}
1144+
1145+
private void deliverResponseBody(ByteString body, DataPlaneListener listener) {
1146+
synchronized (streamLock) {
1147+
if (config.getObservabilityMode()
1148+
|| currentProcessingMode.getResponseBodyMode() != ProcessingMode.BodySendMode.GRPC) {
1149+
callContext.run(() -> listener.onExternalBody(body));
1150+
return;
1151+
}
1152+
1153+
if (downstreamRequestsPending > 0) {
1154+
downstreamRequestsPending--;
1155+
final int bodySize = body.size();
11321156
callContext.run(() -> {
11331157
try {
11341158
listener.onExternalBody(body);
11351159
} finally {
1136-
if (!config.getObservabilityMode() && currentProcessingMode.getResponseBodyMode() == ProcessingMode.BodySendMode.GRPC) {
1160+
synchronized (streamLock) {
1161+
accumulatedWindowUpdateSidestreamToDownstream += bodySize;
1162+
}
1163+
trySendAccumulatedWindowUpdates();
1164+
}
1165+
});
1166+
} else {
1167+
pendingMutatedResponseBodies.add(body);
1168+
}
1169+
}
1170+
}
1171+
1172+
private void drainPendingMutatedResponseBodies() {
1173+
synchronized (streamLock) {
1174+
while (downstreamRequestsPending > 0 && !pendingMutatedResponseBodies.isEmpty()) {
1175+
ByteString body = pendingMutatedResponseBodies.poll();
1176+
downstreamRequestsPending--;
1177+
if (pendingRequests.get() > 0) {
1178+
pendingRequests.decrementAndGet();
1179+
}
1180+
final int bodySize = body.size();
1181+
final DataPlaneListener listener = wrappedListener;
1182+
if (listener != null) {
1183+
callContext.run(() -> {
1184+
try {
1185+
listener.onExternalBody(body);
1186+
} finally {
11371187
synchronized (streamLock) {
11381188
accumulatedWindowUpdateSidestreamToDownstream += bodySize;
11391189
}
11401190
trySendAccumulatedWindowUpdates();
11411191
}
1142-
}
1143-
});
1192+
});
1193+
}
11441194
}
11451195
}
11461196
}

0 commit comments

Comments
 (0)