Skip to content

Commit 032d292

Browse files
shivaspeaksshivaspeaks
authored
xds: ORCA to LRS propagation changes (grpc#12203)
Implements gRFC A85 (grpc/proposal#454).
1 parent 349a35a commit 032d292

20 files changed

Lines changed: 707 additions & 68 deletions

‎xds/src/main/java/io/grpc/xds/CdsLoadBalancer2.java‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -122,15 +122,17 @@ public Status acceptResolvedAddresses(ResolvedAddresses resolvedAddresses) {
122122
result.maxConcurrentRequests(),
123123
result.upstreamTlsContext(),
124124
result.filterMetadata(),
125-
result.outlierDetection());
125+
result.outlierDetection(),
126+
result.backendMetricPropagation());
126127
} else {
127128
instance = DiscoveryMechanism.forLogicalDns(
128129
clusterName,
129130
result.dnsHostName(),
130131
result.lrsServerInfo(),
131132
result.maxConcurrentRequests(),
132133
result.upstreamTlsContext(),
133-
result.filterMetadata());
134+
result.filterMetadata(),
135+
result.backendMetricPropagation());
134136
}
135137
gracefulConfig = GracefulSwitchLoadBalancer.createLoadBalancingPolicyConfig(
136138
lbRegistry.getProvider(CLUSTER_RESOLVER_POLICY_NAME),

‎xds/src/main/java/io/grpc/xds/ClusterImplLoadBalancer.java‎

Lines changed: 21 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@
1717
package io.grpc.xds;
1818

1919
import static com.google.common.base.Preconditions.checkNotNull;
20+
import static io.grpc.xds.client.LoadStatsManager2.isEnabledOrcaLrsPropagation;
2021

2122
import com.google.common.annotations.VisibleForTesting;
2223
import com.google.common.base.MoreObjects;
@@ -46,6 +47,7 @@
4647
import io.grpc.xds.EnvoyServerProtoData.UpstreamTlsContext;
4748
import io.grpc.xds.ThreadSafeRandom.ThreadSafeRandomImpl;
4849
import io.grpc.xds.XdsNameResolverProvider.CallCounterProvider;
50+
import io.grpc.xds.client.BackendMetricPropagation;
4951
import io.grpc.xds.client.Bootstrapper.ServerInfo;
5052
import io.grpc.xds.client.LoadStatsManager2.ClusterDropStats;
5153
import io.grpc.xds.client.LoadStatsManager2.ClusterLocalityStats;
@@ -149,6 +151,7 @@ public Status acceptResolvedAddresses(ResolvedAddresses resolvedAddresses) {
149151
childLbHelper.updateMaxConcurrentRequests(config.maxConcurrentRequests);
150152
childLbHelper.updateSslContextProviderSupplier(config.tlsContext);
151153
childLbHelper.updateFilterMetadata(config.filterMetadata);
154+
childLbHelper.updateBackendMetricPropagation(config.backendMetricPropagation);
152155

153156
childSwitchLb.handleResolvedAddresses(
154157
resolvedAddresses.toBuilder()
@@ -209,6 +212,8 @@ private final class ClusterImplLbHelper extends ForwardingLoadBalancerHelper {
209212
private Map<String, Struct> filterMetadata = ImmutableMap.of();
210213
@Nullable
211214
private final ServerInfo lrsServerInfo;
215+
@Nullable
216+
private BackendMetricPropagation backendMetricPropagation;
212217

213218
private ClusterImplLbHelper(AtomicLong inFlights, @Nullable ServerInfo lrsServerInfo) {
214219
this.inFlights = checkNotNull(inFlights, "inFlights");
@@ -321,7 +326,7 @@ private ClusterLocality createClusterLocalityFromAttributes(Attributes addressAt
321326
(lrsServerInfo == null)
322327
? null
323328
: xdsClient.addClusterLocalityStats(lrsServerInfo, cluster,
324-
edsServiceName, locality);
329+
edsServiceName, locality, backendMetricPropagation);
325330

326331
return new ClusterLocality(localityStats, localityName);
327332
}
@@ -371,6 +376,11 @@ private void updateFilterMetadata(Map<String, Struct> filterMetadata) {
371376
this.filterMetadata = ImmutableMap.copyOf(filterMetadata);
372377
}
373378

379+
private void updateBackendMetricPropagation(
380+
@Nullable BackendMetricPropagation backendMetricPropagation) {
381+
this.backendMetricPropagation = backendMetricPropagation;
382+
}
383+
374384
private class RequestLimitingSubchannelPicker extends SubchannelPicker {
375385
private final SubchannelPicker delegate;
376386
private final List<DropOverload> dropPolicies;
@@ -506,11 +516,19 @@ private OrcaPerRpcListener(ClusterLocalityStats stats) {
506516
}
507517

508518
/**
509-
* Copies {@link MetricReport#getNamedMetrics()} to {@link ClusterLocalityStats} such that it is
510-
* included in the snapshot for the LRS report sent to the LRS server.
519+
* Copies ORCA metrics from {@link MetricReport} to {@link ClusterLocalityStats}
520+
* such that they are included in the snapshot for the LRS report sent to the LRS server.
521+
* This includes both top-level metrics (CPU, memory, application utilization) and named
522+
* metrics, filtered according to the backend metric propagation configuration.
511523
*/
512524
@Override
513525
public void onLoadReport(MetricReport report) {
526+
if (isEnabledOrcaLrsPropagation) {
527+
stats.recordTopLevelMetrics(
528+
report.getCpuUtilization(),
529+
report.getMemoryUtilization(),
530+
report.getApplicationUtilization());
531+
}
514532
stats.recordBackendLoadMetricStats(report.getNamedMetrics());
515533
}
516534
}

‎xds/src/main/java/io/grpc/xds/ClusterImplLoadBalancerProvider.java‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,7 @@
3131
import io.grpc.Status;
3232
import io.grpc.xds.Endpoints.DropOverload;
3333
import io.grpc.xds.EnvoyServerProtoData.UpstreamTlsContext;
34+
import io.grpc.xds.client.BackendMetricPropagation;
3435
import io.grpc.xds.client.Bootstrapper.ServerInfo;
3536
import java.util.ArrayList;
3637
import java.util.Collections;
@@ -98,11 +99,14 @@ static final class ClusterImplConfig {
9899
// Provides the direct child policy and its config.
99100
final Object childConfig;
100101
final Map<String, Struct> filterMetadata;
102+
@Nullable
103+
final BackendMetricPropagation backendMetricPropagation;
101104

102105
ClusterImplConfig(String cluster, @Nullable String edsServiceName,
103106
@Nullable ServerInfo lrsServerInfo, @Nullable Long maxConcurrentRequests,
104107
List<DropOverload> dropCategories, Object childConfig,
105-
@Nullable UpstreamTlsContext tlsContext, Map<String, Struct> filterMetadata) {
108+
@Nullable UpstreamTlsContext tlsContext, Map<String, Struct> filterMetadata,
109+
@Nullable BackendMetricPropagation backendMetricPropagation) {
106110
this.cluster = checkNotNull(cluster, "cluster");
107111
this.edsServiceName = edsServiceName;
108112
this.lrsServerInfo = lrsServerInfo;
@@ -112,6 +116,7 @@ static final class ClusterImplConfig {
112116
this.dropCategories = Collections.unmodifiableList(
113117
new ArrayList<>(checkNotNull(dropCategories, "dropCategories")));
114118
this.childConfig = checkNotNull(childConfig, "childConfig");
119+
this.backendMetricPropagation = backendMetricPropagation;
115120
}
116121

117122
@Override

‎xds/src/main/java/io/grpc/xds/ClusterResolverLoadBalancer.java‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,7 @@
4444
import io.grpc.xds.PriorityLoadBalancerProvider.PriorityLbConfig.PriorityChildConfig;
4545
import io.grpc.xds.XdsConfig.XdsClusterConfig;
4646
import io.grpc.xds.XdsEndpointResource.EdsUpdate;
47+
// import io.grpc.xds.client.BackendMetricPropagation;]
4748
import io.grpc.xds.client.Locality;
4849
import io.grpc.xds.client.XdsLogger;
4950
import io.grpc.xds.client.XdsLogger.XdsLogLevel;
@@ -336,7 +337,7 @@ private static Map<String, PriorityChildConfig> generatePriorityChildConfigs(
336337
new ClusterImplConfig(
337338
discovery.cluster, discovery.edsServiceName, discovery.lrsServerInfo,
338339
discovery.maxConcurrentRequests, dropOverloads, endpointLbConfig,
339-
discovery.tlsContext, discovery.filterMetadata);
340+
discovery.tlsContext, discovery.filterMetadata, discovery.backendMetricPropagation);
340341
LoadBalancerProvider clusterImplLbProvider =
341342
lbRegistry.getProvider(XdsLbPolicies.CLUSTER_IMPL_POLICY_NAME);
342343
Object priorityChildPolicy = GracefulSwitchLoadBalancer.createLoadBalancingPolicyConfig(

‎xds/src/main/java/io/grpc/xds/ClusterResolverLoadBalancerProvider.java‎

Lines changed: 17 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,7 @@
3030
import io.grpc.Status;
3131
import io.grpc.xds.EnvoyServerProtoData.OutlierDetection;
3232
import io.grpc.xds.EnvoyServerProtoData.UpstreamTlsContext;
33+
import io.grpc.xds.client.BackendMetricPropagation;
3334
import io.grpc.xds.client.Bootstrapper.ServerInfo;
3435
import java.util.Map;
3536
import java.util.Objects;
@@ -152,6 +153,8 @@ static final class DiscoveryMechanism {
152153
@Nullable
153154
final OutlierDetection outlierDetection;
154155
final Map<String, Struct> filterMetadata;
156+
@Nullable
157+
final BackendMetricPropagation backendMetricPropagation;
155158

156159
enum Type {
157160
EDS,
@@ -161,7 +164,8 @@ enum Type {
161164
private DiscoveryMechanism(String cluster, Type type, @Nullable String edsServiceName,
162165
@Nullable String dnsHostName, @Nullable ServerInfo lrsServerInfo,
163166
@Nullable Long maxConcurrentRequests, @Nullable UpstreamTlsContext tlsContext,
164-
Map<String, Struct> filterMetadata, @Nullable OutlierDetection outlierDetection) {
167+
Map<String, Struct> filterMetadata, @Nullable OutlierDetection outlierDetection,
168+
@Nullable BackendMetricPropagation backendMetricPropagation) {
165169
this.cluster = checkNotNull(cluster, "cluster");
166170
this.type = checkNotNull(type, "type");
167171
this.edsServiceName = edsServiceName;
@@ -171,27 +175,33 @@ private DiscoveryMechanism(String cluster, Type type, @Nullable String edsServic
171175
this.tlsContext = tlsContext;
172176
this.filterMetadata = ImmutableMap.copyOf(checkNotNull(filterMetadata, "filterMetadata"));
173177
this.outlierDetection = outlierDetection;
178+
this.backendMetricPropagation = backendMetricPropagation;
174179
}
175180

176181
static DiscoveryMechanism forEds(String cluster, @Nullable String edsServiceName,
177182
@Nullable ServerInfo lrsServerInfo, @Nullable Long maxConcurrentRequests,
178183
@Nullable UpstreamTlsContext tlsContext, Map<String, Struct> filterMetadata,
179-
OutlierDetection outlierDetection) {
180-
return new DiscoveryMechanism(cluster, Type.EDS, edsServiceName, null, lrsServerInfo,
181-
maxConcurrentRequests, tlsContext, filterMetadata, outlierDetection);
184+
OutlierDetection outlierDetection,
185+
@Nullable BackendMetricPropagation backendMetricPropagation) {
186+
return new DiscoveryMechanism(cluster, Type.EDS, edsServiceName,
187+
null, lrsServerInfo, maxConcurrentRequests, tlsContext,
188+
filterMetadata, outlierDetection, backendMetricPropagation);
182189
}
183190

184191
static DiscoveryMechanism forLogicalDns(String cluster, String dnsHostName,
185192
@Nullable ServerInfo lrsServerInfo, @Nullable Long maxConcurrentRequests,
186-
@Nullable UpstreamTlsContext tlsContext, Map<String, Struct> filterMetadata) {
193+
@Nullable UpstreamTlsContext tlsContext, Map<String, Struct> filterMetadata,
194+
@Nullable BackendMetricPropagation backendMetricPropagation) {
187195
return new DiscoveryMechanism(cluster, Type.LOGICAL_DNS, null, dnsHostName,
188-
lrsServerInfo, maxConcurrentRequests, tlsContext, filterMetadata, null);
196+
lrsServerInfo, maxConcurrentRequests, tlsContext, filterMetadata, null,
197+
backendMetricPropagation);
189198
}
190199

191200
@Override
192201
public int hashCode() {
193202
return Objects.hash(cluster, type, lrsServerInfo, maxConcurrentRequests, tlsContext,
194-
edsServiceName, dnsHostName, filterMetadata, outlierDetection);
203+
edsServiceName, dnsHostName, filterMetadata,
204+
outlierDetection, backendMetricPropagation);
195205
}
196206

197207
@Override

‎xds/src/main/java/io/grpc/xds/XdsClusterResource.java‎

Lines changed: 26 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@
1818

1919
import static com.google.common.base.Preconditions.checkNotNull;
2020
import static io.grpc.xds.client.Bootstrapper.ServerInfo;
21+
import static io.grpc.xds.client.LoadStatsManager2.isEnabledOrcaLrsPropagation;
2122

2223
import com.google.auto.value.AutoValue;
2324
import com.google.common.annotations.VisibleForTesting;
@@ -47,6 +48,7 @@
4748
import io.grpc.xds.EnvoyServerProtoData.OutlierDetection;
4849
import io.grpc.xds.EnvoyServerProtoData.UpstreamTlsContext;
4950
import io.grpc.xds.XdsClusterResource.CdsUpdate;
51+
import io.grpc.xds.client.BackendMetricPropagation;
5052
import io.grpc.xds.client.XdsClient.ResourceUpdate;
5153
import io.grpc.xds.client.XdsResourceType;
5254
import io.grpc.xds.internal.security.CommonTlsContextUtil;
@@ -227,6 +229,12 @@ private static StructOrError<CdsUpdate.Builder> parseNonAggregateCluster(
227229
UpstreamTlsContext upstreamTlsContext = null;
228230
OutlierDetection outlierDetection = null;
229231
boolean isHttp11ProxyAvailable = false;
232+
BackendMetricPropagation backendMetricPropagation = null;
233+
234+
if (isEnabledOrcaLrsPropagation) {
235+
backendMetricPropagation = BackendMetricPropagation.fromMetricSpecs(
236+
cluster.getLrsReportEndpointMetricsList());
237+
}
230238
if (cluster.hasLrsServer()) {
231239
if (!cluster.getLrsServer().hasSelf()) {
232240
return StructOrError.fromError(
@@ -326,7 +334,7 @@ private static StructOrError<CdsUpdate.Builder> parseNonAggregateCluster(
326334

327335
return StructOrError.fromStruct(CdsUpdate.forEds(
328336
clusterName, edsServiceName, lrsServerInfo, maxConcurrentRequests, upstreamTlsContext,
329-
outlierDetection, isHttp11ProxyAvailable));
337+
outlierDetection, isHttp11ProxyAvailable, backendMetricPropagation));
330338
} else if (type.equals(Cluster.DiscoveryType.LOGICAL_DNS)) {
331339
if (!cluster.hasLoadAssignment()) {
332340
return StructOrError.fromError(
@@ -362,7 +370,7 @@ private static StructOrError<CdsUpdate.Builder> parseNonAggregateCluster(
362370
Locale.US, "%s:%d", socketAddress.getAddress(), socketAddress.getPortValue());
363371
return StructOrError.fromStruct(CdsUpdate.forLogicalDns(
364372
clusterName, dnsHostName, lrsServerInfo, maxConcurrentRequests,
365-
upstreamTlsContext, isHttp11ProxyAvailable));
373+
upstreamTlsContext, isHttp11ProxyAvailable, backendMetricPropagation));
366374
}
367375
return StructOrError.fromError(
368376
"Cluster " + clusterName + ": unsupported built-in discovery type: " + type);
@@ -614,6 +622,9 @@ abstract static class CdsUpdate implements ResourceUpdate {
614622

615623
abstract ImmutableMap<String, Object> parsedMetadata();
616624

625+
@Nullable
626+
abstract BackendMetricPropagation backendMetricPropagation();
627+
617628
private static Builder newBuilder(String clusterName) {
618629
return new AutoValue_XdsClusterResource_CdsUpdate.Builder()
619630
.clusterName(clusterName)
@@ -622,7 +633,8 @@ private static Builder newBuilder(String clusterName) {
622633
.choiceCount(0)
623634
.filterMetadata(ImmutableMap.of())
624635
.parsedMetadata(ImmutableMap.of())
625-
.isHttp11ProxyAvailable(false);
636+
.isHttp11ProxyAvailable(false)
637+
.backendMetricPropagation(null);
626638
}
627639

628640
static Builder forAggregate(String clusterName, List<String> prioritizedClusterNames) {
@@ -636,29 +648,33 @@ static Builder forEds(String clusterName, @Nullable String edsServiceName,
636648
@Nullable ServerInfo lrsServerInfo, @Nullable Long maxConcurrentRequests,
637649
@Nullable UpstreamTlsContext upstreamTlsContext,
638650
@Nullable OutlierDetection outlierDetection,
639-
boolean isHttp11ProxyAvailable) {
651+
boolean isHttp11ProxyAvailable,
652+
BackendMetricPropagation backendMetricPropagation) {
640653
return newBuilder(clusterName)
641654
.clusterType(ClusterType.EDS)
642655
.edsServiceName(edsServiceName)
643656
.lrsServerInfo(lrsServerInfo)
644657
.maxConcurrentRequests(maxConcurrentRequests)
645658
.upstreamTlsContext(upstreamTlsContext)
646659
.outlierDetection(outlierDetection)
647-
.isHttp11ProxyAvailable(isHttp11ProxyAvailable);
660+
.isHttp11ProxyAvailable(isHttp11ProxyAvailable)
661+
.backendMetricPropagation(backendMetricPropagation);
648662
}
649663

650664
static Builder forLogicalDns(String clusterName, String dnsHostName,
651665
@Nullable ServerInfo lrsServerInfo,
652666
@Nullable Long maxConcurrentRequests,
653667
@Nullable UpstreamTlsContext upstreamTlsContext,
654-
boolean isHttp11ProxyAvailable) {
668+
boolean isHttp11ProxyAvailable,
669+
BackendMetricPropagation backendMetricPropagation) {
655670
return newBuilder(clusterName)
656671
.clusterType(ClusterType.LOGICAL_DNS)
657672
.dnsHostName(dnsHostName)
658673
.lrsServerInfo(lrsServerInfo)
659674
.maxConcurrentRequests(maxConcurrentRequests)
660675
.upstreamTlsContext(upstreamTlsContext)
661-
.isHttp11ProxyAvailable(isHttp11ProxyAvailable);
676+
.isHttp11ProxyAvailable(isHttp11ProxyAvailable)
677+
.backendMetricPropagation(backendMetricPropagation);
662678
}
663679

664680
enum ClusterType {
@@ -749,6 +765,9 @@ Builder leastRequestLbPolicy(Integer choiceCount) {
749765

750766
protected abstract Builder parsedMetadata(ImmutableMap<String, Object> parsedMetadata);
751767

768+
protected abstract Builder backendMetricPropagation(
769+
BackendMetricPropagation backendMetricPropagation);
770+
752771
abstract CdsUpdate build();
753772
}
754773
}

0 commit comments

Comments
 (0)