Skip to content

Commit 1ad8aa5

Browse files
committed
xds/ext_proc: Enforce protocol ordering and violation handling in server interceptor
- Replace expectedResponses queue with directional state flags and configuration checks. - Enforce strict directional ordering on ext_proc ProcessingResponse events. - Propagate protocol violation causes on stream failure. - Add Category 21 unit tests covering 17 protocol ordering and violation scenarios. TAG=agy CONV=581f86a6-097d-44a2-9d6e-6f9d2e83cbc0
1 parent 3b4b890 commit 1ad8aa5

2 files changed

Lines changed: 2038 additions & 27 deletions

File tree

‎xds/src/main/java/io/grpc/xds/ExternalProcessorServerInterceptor.java‎

Lines changed: 173 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -66,7 +66,6 @@
6666
import io.grpc.xds.ExternalProcessorFilter.ExternalProcessorFilterConfig;
6767
import io.grpc.xds.Filter.FilterContext;
6868
import io.grpc.xds.internal.extproc.DataPlaneCallState;
69-
import io.grpc.xds.internal.extproc.EventType;
7069
import io.grpc.xds.internal.extproc.ExtProcStreamState;
7170
import io.grpc.xds.internal.extproc.KnownLengthInputStream;
7271
import io.grpc.xds.internal.grpcservice.CachedChannelManager;
@@ -243,7 +242,40 @@ static class DataPlaneServerCall
243242
private final ExternalProcessorFilterConfig config;
244243
private final ScheduledExecutorService scheduler;
245244
final Object streamLock = new Object();
246-
private final Queue<EventType> expectedResponses = new ConcurrentLinkedQueue<>();
245+
private volatile boolean requestHeadersSent;
246+
private volatile boolean responseHeadersSentToExtProc;
247+
private volatile boolean responseTrailersSentToExtProc;
248+
private volatile boolean clientHeadersReceived;
249+
private volatile boolean clientEndOfStreamReceived;
250+
private volatile boolean serverHeadersReceived;
251+
private volatile boolean serverTrailersReceived;
252+
private volatile boolean serverEndOfStreamReceived;
253+
private volatile Throwable streamFailureCause;
254+
255+
private boolean isRequestHeaderConfigured() {
256+
ProcessingMode.HeaderSendMode mode = currentProcessingMode.getRequestHeaderMode();
257+
return mode == ProcessingMode.HeaderSendMode.SEND
258+
|| mode == ProcessingMode.HeaderSendMode.DEFAULT;
259+
}
260+
261+
private boolean isRequestBodyConfigured() {
262+
return currentProcessingMode.getRequestBodyMode() != ProcessingMode.BodySendMode.NONE;
263+
}
264+
265+
private boolean isResponseHeaderConfigured() {
266+
ProcessingMode.HeaderSendMode mode = currentProcessingMode.getResponseHeaderMode();
267+
return mode == ProcessingMode.HeaderSendMode.SEND
268+
|| mode == ProcessingMode.HeaderSendMode.DEFAULT;
269+
}
270+
271+
private boolean isResponseBodyConfigured() {
272+
return currentProcessingMode.getResponseBodyMode() != ProcessingMode.BodySendMode.NONE;
273+
}
274+
275+
private boolean isResponseTrailerConfigured() {
276+
return currentProcessingMode.getResponseTrailerMode() == ProcessingMode.HeaderSendMode.SEND;
277+
}
278+
247279
private volatile ClientCallStreamObserver<ProcessingRequest> extProcClientCallRequestObserver;
248280
private final Queue<InputStream> pendingDrainingOutgoingMessages =
249281
new ConcurrentLinkedQueue<>();
@@ -575,11 +607,11 @@ private void sendToExtProc(ProcessingRequest request) {
575607
}
576608

577609
if (requestToSend.hasRequestHeaders()) {
578-
expectedResponses.add(EventType.REQUEST_HEADERS);
610+
requestHeadersSent = true;
579611
} else if (requestToSend.hasResponseHeaders()) {
580-
expectedResponses.add(EventType.RESPONSE_HEADERS);
612+
responseHeadersSentToExtProc = true;
581613
} else if (requestToSend.hasResponseTrailers()) {
582-
expectedResponses.add(EventType.RESPONSE_TRAILERS);
614+
responseTrailersSentToExtProc = true;
583615
}
584616

585617
extProcClientCallRequestObserver.onNext(requestToSend);
@@ -666,7 +698,6 @@ private void closeExtProcStream() {
666698
extProcClientCallRequestObserver.onCompleted();
667699
}
668700
}
669-
expectedResponses.clear();
670701
}
671702
proceedWithClose();
672703
}
@@ -683,13 +714,12 @@ private void cancelExtProcStream(Throwable t) {
683714
extProcClientCallRequestObserver = null;
684715
}
685716
}
686-
expectedResponses.clear();
687717
proceedWithClose();
688718
}
689719
}
690720

691721
private void internalOnError(Throwable t) {
692-
722+
this.streamFailureCause = t;
693723
if (markExtProcStreamFailed(extProcStreamState)) {
694724
synchronized (streamLock) {
695725
if (extProcClientCallRequestObserver != null) {
@@ -701,7 +731,6 @@ private void internalOnError(Throwable t) {
701731
extProcClientCallRequestObserver = null;
702732
}
703733
}
704-
expectedResponses.clear();
705734
if (config.getObservabilityMode()
706735
|| (config.getFailureModeAllow() && !bodyMessageSentToExtProc.get())) {
707736
handleFailOpen();
@@ -746,37 +775,149 @@ void handleExtProcResponse(ProcessingResponse response) {
746775
return;
747776
}
748777

749-
EventType expected = expectedResponses.peek();
750-
EventType received = null;
751778
if (response.hasRequestHeaders()) {
752-
received = EventType.REQUEST_HEADERS;
779+
if (!isRequestHeaderConfigured()) {
780+
internalOnError(Status.UNAVAILABLE
781+
.withDescription("Protocol error: received response out of order. "
782+
+ "Filter is not configured to send client headers.")
783+
.asRuntimeException());
784+
return;
785+
}
786+
if (!requestHeadersSent) {
787+
internalOnError(Status.UNAVAILABLE
788+
.withDescription("Protocol error: received response out of order. "
789+
+ "Received request_headers before request_headers was sent.")
790+
.asRuntimeException());
791+
return;
792+
}
793+
if (clientEndOfStreamReceived) {
794+
internalOnError(Status.UNAVAILABLE
795+
.withDescription("Protocol error: received response out of order. "
796+
+ "Received request_headers after end-of-stream in "
797+
+ "client-to-server direction.")
798+
.asRuntimeException());
799+
return;
800+
}
801+
if (clientHeadersReceived) {
802+
internalOnError(Status.UNAVAILABLE
803+
.withDescription("Protocol error: received response out of order. "
804+
+ "Duplicate request_headers response.")
805+
.asRuntimeException());
806+
return;
807+
}
808+
clientHeadersReceived = true;
753809
} else if (response.hasResponseHeaders()) {
754-
received = EventType.RESPONSE_HEADERS;
810+
if (!isResponseHeaderConfigured()) {
811+
internalOnError(Status.UNAVAILABLE
812+
.withDescription("Protocol error: received response out of order. "
813+
+ "Filter is not configured to send server headers.")
814+
.asRuntimeException());
815+
return;
816+
}
817+
if (!responseHeadersSentToExtProc) {
818+
internalOnError(Status.UNAVAILABLE
819+
.withDescription("Protocol error: received response out of order. "
820+
+ "Received response_headers before response_headers was sent.")
821+
.asRuntimeException());
822+
return;
823+
}
824+
if (serverTrailersReceived || serverEndOfStreamReceived) {
825+
internalOnError(Status.UNAVAILABLE
826+
.withDescription("Protocol error: received response out of order. "
827+
+ "Received response_headers after trailers or end-of-stream.")
828+
.asRuntimeException());
829+
return;
830+
}
831+
if (serverHeadersReceived) {
832+
internalOnError(Status.UNAVAILABLE
833+
.withDescription("Protocol error: received response out of order. "
834+
+ "Duplicate response_headers response.")
835+
.asRuntimeException());
836+
return;
837+
}
838+
serverHeadersReceived = true;
755839
} else if (response.hasResponseTrailers()) {
756-
received = EventType.RESPONSE_TRAILERS;
757-
}
758-
759-
if (received != null) {
760-
if (expected == null || expected != received) {
840+
if (!isResponseTrailerConfigured()) {
761841
internalOnError(Status.UNAVAILABLE
762-
.withDescription("Protocol error: received response out of order. Expected: "
763-
+ expected + ", Received: " + received)
842+
.withDescription("Protocol error: received response out of order. "
843+
+ "Filter is not configured to send server trailers.")
764844
.asRuntimeException());
765845
return;
766846
}
767-
expectedResponses.poll();
847+
if (isResponseHeaderConfigured() && !serverHeadersReceived) {
848+
internalOnError(Status.UNAVAILABLE
849+
.withDescription("Protocol error: received response out of order. "
850+
+ "Received response_trailers before response_headers response.")
851+
.asRuntimeException());
852+
return;
853+
}
854+
if (!responseTrailersSentToExtProc) {
855+
internalOnError(Status.UNAVAILABLE
856+
.withDescription("Protocol error: received response out of order. "
857+
+ "Received response_trailers before response_trailers was sent.")
858+
.asRuntimeException());
859+
return;
860+
}
861+
if (serverTrailersReceived) {
862+
internalOnError(Status.UNAVAILABLE
863+
.withDescription("Protocol error: received response out of order. "
864+
+ "Duplicate response_trailers response.")
865+
.asRuntimeException());
866+
return;
867+
}
868+
if (serverEndOfStreamReceived) {
869+
internalOnError(Status.UNAVAILABLE
870+
.withDescription("Protocol error: received response out of order. "
871+
+ "Received response_trailers after end-of-stream.")
872+
.asRuntimeException());
873+
return;
874+
}
875+
serverTrailersReceived = true;
876+
serverEndOfStreamReceived = true;
768877
} else if (response.hasRequestBody()) {
769-
if (expected == EventType.REQUEST_HEADERS) {
878+
if (!isRequestBodyConfigured()) {
879+
internalOnError(Status.UNAVAILABLE
880+
.withDescription(
881+
"Protocol error: received request_body when filter is not configured "
882+
+ "to send client messages.")
883+
.asRuntimeException());
884+
return;
885+
}
886+
if (isRequestHeaderConfigured() && !clientHeadersReceived) {
770887
internalOnError(Status.UNAVAILABLE
771888
.withDescription(
772889
"Protocol error: received request_body before request_headers response.")
773890
.asRuntimeException());
774891
return;
775892
}
893+
if (clientEndOfStreamReceived) {
894+
internalOnError(Status.UNAVAILABLE
895+
.withDescription(
896+
"Protocol error: received request_body after end-of-stream in "
897+
+ "client-to-server direction.")
898+
.asRuntimeException());
899+
return;
900+
}
776901
} else if (response.hasResponseBody()) {
777-
if (expected == EventType.RESPONSE_HEADERS) {
902+
if (!isResponseBodyConfigured()) {
778903
internalOnError(Status.UNAVAILABLE
779-
.withDescription("Protocol error: received response_body before headers response.")
904+
.withDescription(
905+
"Protocol error: received response_body when filter is not configured "
906+
+ "to send server messages.")
907+
.asRuntimeException());
908+
return;
909+
}
910+
if (isResponseHeaderConfigured() && !serverHeadersReceived) {
911+
internalOnError(Status.UNAVAILABLE
912+
.withDescription(
913+
"Protocol error: received response_body before headers response.")
914+
.asRuntimeException());
915+
return;
916+
}
917+
if (serverTrailersReceived || serverEndOfStreamReceived) {
918+
internalOnError(Status.UNAVAILABLE
919+
.withDescription(
920+
"Protocol error: received response_body after trailers or end-of-stream.")
780921
.asRuntimeException());
781922
return;
782923
}
@@ -828,6 +969,7 @@ else if (response.hasResponseHeaders()) {
828969
}
829970
}
830971
if (trailersOnly.get()) {
972+
serverEndOfStreamReceived = true;
831973
proceedWithClose();
832974
} else {
833975
proceedWithSendHeaders();
@@ -858,6 +1000,7 @@ else if (response.hasResponseTrailers()) {
8581000
}
8591001

8601002
void handleExtProcError(Throwable t) {
1003+
this.streamFailureCause = t;
8611004
if (markExtProcStreamFailed(extProcStreamState)) {
8621005
synchronized (streamLock) {
8631006
extProcClientCallRequestObserver = null;
@@ -1122,7 +1265,7 @@ public void close(Status status, Metadata trailers) {
11221265
if (markDataPlaneCallClosed(dataPlaneCallState)) {
11231266
proceedWithClose(
11241267
Status.INTERNAL.withDescription("External processor stream failed")
1125-
.withCause(status.getCause()),
1268+
.withCause(streamFailureCause != null ? streamFailureCause : status.getCause()),
11261269
new Metadata());
11271270
}
11281271
return;
@@ -1341,6 +1484,9 @@ private void handleRequestBodyResponse(BodyResponse bodyResponse) {
13411484
BodyMutation mutation = bodyResponse.getResponse().getBodyMutation();
13421485
if (mutation.hasStreamedResponse()) {
13431486
StreamedBodyResponse streamed = mutation.getStreamedResponse();
1487+
if (streamed.getEndOfStream()) {
1488+
clientEndOfStreamReceived = true;
1489+
}
13441490
final int bodySize = streamed.getBody().size();
13451491
synchronized (streamLock) {
13461492
sidestreamToUpstreamWindow -= bodySize;
@@ -1638,8 +1784,8 @@ private void handleFailOpen() {
16381784
*/
16391785
private void checkEndOfStream() {
16401786
if (terminationTriggered.get() && isRequestSideCompleted()
1641-
&& !expectedResponses.contains(EventType.RESPONSE_HEADERS)
1642-
&& !expectedResponses.contains(EventType.RESPONSE_TRAILERS)) {
1787+
&& (!isResponseHeaderConfigured() || serverHeadersReceived)
1788+
&& (!isResponseTrailerConfigured() || serverTrailersReceived || trailersOnly.get())) {
16431789
unblockAfterStreamComplete();
16441790
closeExtProcStream();
16451791
}

0 commit comments

Comments
 (0)