Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@
package io.grpc.xds;

import static com.google.common.base.Preconditions.checkNotNull;
import static com.google.common.base.Preconditions.checkState;
import static io.grpc.xds.internal.extproc.ExternalProcessorUtil.applyHeaderMutations;
import static io.grpc.xds.internal.extproc.ExternalProcessorUtil.collectAttributes;
import static io.grpc.xds.internal.extproc.ExternalProcessorUtil.markDataPlaneCallClosed;
Expand Down Expand Up @@ -366,6 +367,8 @@ private static class DataPlaneClientCall<ReqT, RespT>
final AtomicBoolean isProcessingTrailers = new AtomicBoolean(false);
final AtomicBoolean pendingHalfClose = new AtomicBoolean(false);
final AtomicBoolean bodyMessageSentToExtProc = new AtomicBoolean(false);
final AtomicBoolean aborted = new AtomicBoolean(false);
final AtomicBoolean cancelCalled = new AtomicBoolean(false);

protected DataPlaneClientCall(
DataPlaneDelayedCall<ReqT, RespT> delayedCall,
Expand Down Expand Up @@ -994,26 +997,30 @@ public void request(int numMessages) {
@SuppressWarnings("unchecked")
@Override
public void sendMessage(ReqT message) {
checkState(!cancelCalled.get(), "call was cancelled");
if (aborted.get()) {
return;
}
if (requestSideClosed.get()) {
// External processor already closed the request stream. Discard further messages.
return;
}

if (passThroughMode.get()) {
super.sendMessage(message);
proceedWithSendMessage(message);
return;
}

synchronized (streamLock) {
if (passThroughMode.get()) {
super.sendMessage(message);
proceedWithSendMessage(message);
return;
}

ExtProcStreamState state = extProcStreamState.get();
if (state.isDraining() || state.isCompleted()) {
if (currentProcessingMode.getRequestBodyMode() == ProcessingMode.BodySendMode.NONE) {
super.sendMessage(message);
proceedWithSendMessage(message);
return;
}
try {
Expand All @@ -1026,7 +1033,7 @@ public void sendMessage(ReqT message) {
}

if (currentProcessingMode.getRequestBodyMode() == ProcessingMode.BodySendMode.NONE) {
super.sendMessage(message);
proceedWithSendMessage(message);
return;
}

Expand All @@ -1041,7 +1048,7 @@ public void sendMessage(ReqT message) {
.build())
.build());
bodyMessageSentToExtProc.set(true);
super.sendMessage((ReqT) new KnownLengthInputStream(bodyByteString));
proceedWithSendMessage((ReqT) new KnownLengthInputStream(bodyByteString));
} else {
if (downstreamToSidestreamWindow <= 0 || !pendingRequestBodyMessages.isEmpty()) {
pendingRequestBodyMessages.add(bodyByteString);
Expand Down Expand Up @@ -1075,11 +1082,25 @@ private void drainPendingRequestBodyMessages() {
sendRequestBodyToExtProc(body);
}
if (pendingRequestBodyMessages.isEmpty() && pendingHalfClose.compareAndSet(true, false)) {
halfClose();
internalHalfClose();
}
}

private boolean isCancelledOrAborted() {
return cancelCalled.get() || aborted.get();
}

private void proceedWithSendMessage(ReqT message) {
if (isCancelledOrAborted()) {
return;
}
super.sendMessage(message);
}

private void proceedWithHalfClose() {
if (isCancelledOrAborted()) {
return;
}
if (clientHalfCloseStartNanos > 0) {
long durationNanos = System.nanoTime() - clientHalfCloseStartNanos;
recordDuration(clientHalfCloseDuration, durationNanos);
Expand All @@ -1090,6 +1111,14 @@ private void proceedWithHalfClose() {

@Override
public void halfClose() {
checkState(!cancelCalled.get(), "call was cancelled");
internalHalfClose();
}

private void internalHalfClose() {
if (isCancelledOrAborted()) {
return;
}
if (appHalfClosed.compareAndSet(false, true)) {
clientHalfCloseStartNanos = System.nanoTime();
}
Expand Down Expand Up @@ -1159,12 +1188,14 @@ public void halfClose() {
}

void cancelDownstream(@Nullable String message, @Nullable Throwable cause) {

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Is there a potential race between cancelDownstream and cancel for the value of aborted? cancelDownstream could set aborted to true after it's ben set to false by cancel.

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I have changed it now so that:

  1. cancel() only sets cancelCalled.set(true) and calls delayedCall.cancel(message, cause) directly, while cancelDownstream() only sets aborted.set(true) and calls delayedCall.cancel(message, cause) (so neither flag ever overwrites the other).
  2. sendMessage() and halfClose() check checkState(!cancelCalled.get(), "call was cancelled") before if (aborted.get()) { return; }, so if both cancelDownstream() and cancel() occur, explicit application cancellation always takes precedence without needing cancel() to reset aborted.

aborted.set(true);
delayedCall.cancel(message, cause);
}

@Override
public void cancel(@Nullable String message, @Nullable Throwable cause) {
synchronized (streamLock) {
cancelCalled.set(true);
if (markExtProcStreamFailed(extProcStreamState)) {
if (extProcClientCallRequestObserver != null) {
extProcClientCallRequestObserver.onError(
Expand All @@ -1176,7 +1207,7 @@ public void cancel(@Nullable String message, @Nullable Throwable cause) {
}
}
}
cancelDownstream(message, cause);
delayedCall.cancel(message, cause);
}

@SuppressWarnings("unchecked")
Expand All @@ -1201,7 +1232,7 @@ private void handleRequestBodyResponse(BodyResponse bodyResponse) {
}
}
if (sendImmediately) {
super.sendMessage((ReqT) new KnownLengthInputStream(body));
proceedWithSendMessage((ReqT) new KnownLengthInputStream(body));
trySendAccumulatedWindowUpdates();
}
}
Expand Down Expand Up @@ -1319,7 +1350,7 @@ void drainPendingUpstreamBodyMessages() {
if (body == null) {
break;
}
super.sendMessage((ReqT) new KnownLengthInputStream(body));
proceedWithSendMessage((ReqT) new KnownLengthInputStream(body));
trySendAccumulatedWindowUpdates();
if (triggerHalfClose) {
if (requestSideClosed.compareAndSet(false, true)) {
Expand Down Expand Up @@ -1402,9 +1433,9 @@ private void drainPendingDrainingMessages() {
}

if (isMutated) {
super.sendMessage((ReqT) new KnownLengthInputStream((ByteString) msg));
proceedWithSendMessage((ReqT) new KnownLengthInputStream((ByteString) msg));
} else {
super.sendMessage((ReqT) msg);
proceedWithSendMessage((ReqT) msg);
}
}
}
Expand Down
62 changes: 38 additions & 24 deletions xds/src/main/java/io/grpc/xds/XdsNameResolver.java
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@

import static com.google.common.base.Preconditions.checkArgument;
import static com.google.common.base.Preconditions.checkNotNull;
import static com.google.common.base.Preconditions.checkState;
import static io.grpc.xds.client.Bootstrapper.XDSTP_SCHEME;

import com.google.common.annotations.VisibleForTesting;
Expand Down Expand Up @@ -1283,35 +1284,43 @@ public void onReady() {
};
}

private static <ReqT> void sendSerializedMessage(
ClientCall<InputStream, ?> rawCall,
MethodDescriptor<ReqT, ?> method,
ReqT message) {
ByteString byteString;
try {
InputStream stream = method.getRequestMarshaller().stream(message);
private abstract static class PartialForwardingClientCall<ReqT, RespT>
extends ClientCall<ReqT, RespT> {
private boolean aborted;
private boolean cancelCalled;

protected abstract ClientCall<?, ?> delegate();

protected void sendSerializedMessage(
ClientCall<InputStream, ?> rawCall,
MethodDescriptor<ReqT, ?> method,
ReqT message) {
checkState(!cancelCalled, "call was cancelled");
if (aborted) {
return;
}
ByteString byteString;
try {
byteString = ExternalProcessorUtil.outboundStreamToByteString(stream);
} finally {
InputStream stream = method.getRequestMarshaller().stream(message);
try {
stream.close();
} catch (IOException ignored) {
// ignore
byteString = ExternalProcessorUtil.outboundStreamToByteString(stream);
} finally {
try {
stream.close();
} catch (IOException ignored) {
// ignore
}
}
} catch (IOException | RuntimeException e) {
aborted = true;
rawCall.cancel("Failed to read message for raw message interceptor", e);
return;
} catch (Error e) {
rawCall.cancel("Client sendMessage() failed with Error", e);
throw e;
}
} catch (IOException | RuntimeException e) {
rawCall.cancel("Failed to read message for raw message interceptor", e);
return;
} catch (Error e) {
rawCall.cancel("Client sendMessage() failed with Error", e);
throw e;
rawCall.sendMessage(new KnownLengthInputStream(byteString));
}
rawCall.sendMessage(new KnownLengthInputStream(byteString));
}

private abstract static class PartialForwardingClientCall<ReqT, RespT>
extends ClientCall<ReqT, RespT> {
protected abstract ClientCall<?, ?> delegate();

@Override
public void request(int numMessages) {
Expand All @@ -1320,11 +1329,16 @@ public void request(int numMessages) {

@Override
public void cancel(@Nullable String message, @Nullable Throwable cause) {
cancelCalled = true;
delegate().cancel(message, cause);
}

@Override
public void halfClose() {
checkState(!cancelCalled, "call was cancelled");
if (aborted) {
return;
}
delegate().halfClose();
}

Expand Down
Loading
Loading