From eecfc2373a2330bcb3cff20d789f4d918d15ba86 Mon Sep 17 00:00:00 2001 From: Rajkumar Rangaraj Date: Thu, 12 Nov 2026 14:37:46 -0800 Subject: [PATCH 1/3] Stop using OpenTelemetry SDK internal AttributesProcessor for pre-aggregated metrics AttributesProcessor and ViewBuilder.addAttributesProcessor() were removed in OpenTelemetry SDK 1.67.0 (open-telemetry/opentelemetry-java#8818), which breaks the build for the OpenTelemetry dependency updates (#4876, #4888). Add the Application Insights specific attributes (connection string and role name overrides, synthetic) and mark the span as pre-aggregated from an InstrumenterCustomizer instead, and limit the metric attributes using the public View attribute filter. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../AiInstrumenterCustomizerProvider.java | 10 +- .../agent/internal/init/SecondEntryPoint.java | 2 +- .../agent/internal/preagg/AiViewRegistry.java | 23 +++ .../agent/internal/preagg}/MetricView.java | 21 ++- .../agent/internal/preagg/MetricsState.java | 68 ++++++++ .../PreAggregatedMetricsCustomizer.java | 129 +++++++++++++++ .../agent/internal/preagg}/UserAgents.java | 2 +- .../sdk/metrics/ViewBuilderAccessor.java | 15 -- .../metrics/internal/view/AiViewRegistry.java | 38 ----- .../view/MetricViewAttributesProcessor.java | 70 -------- .../PreAggregatedMetricsTest.java | 149 +++++++++++++++++- 11 files changed, 386 insertions(+), 141 deletions(-) create mode 100644 agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/preagg/AiViewRegistry.java rename agent/agent-tooling/src/main/java/{io/opentelemetry/sdk/metrics/internal/view => com/microsoft/applicationinsights/agent/internal/preagg}/MetricView.java (77%) create mode 100644 agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/preagg/MetricsState.java create mode 100644 agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/preagg/PreAggregatedMetricsCustomizer.java rename agent/agent-tooling/src/main/java/{io/opentelemetry/sdk/metrics/internal/view => com/microsoft/applicationinsights/agent/internal/preagg}/UserAgents.java (87%) delete mode 100644 agent/agent-tooling/src/main/java/io/opentelemetry/sdk/metrics/ViewBuilderAccessor.java delete mode 100644 agent/agent-tooling/src/main/java/io/opentelemetry/sdk/metrics/internal/view/AiViewRegistry.java delete mode 100644 agent/agent-tooling/src/main/java/io/opentelemetry/sdk/metrics/internal/view/MetricViewAttributesProcessor.java rename agent/agent-tooling/src/test/java/com/microsoft/applicationinsights/agent/internal/{ => preagg}/PreAggregatedMetricsTest.java (70%) diff --git a/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/init/AiInstrumenterCustomizerProvider.java b/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/init/AiInstrumenterCustomizerProvider.java index beaf6f2600d..6b2dc64fb99 100644 --- a/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/init/AiInstrumenterCustomizerProvider.java +++ b/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/init/AiInstrumenterCustomizerProvider.java @@ -5,12 +5,15 @@ import com.google.auto.service.AutoService; import com.microsoft.applicationinsights.agent.bootstrap.preagg.AiContextCustomizerHolder; +import com.microsoft.applicationinsights.agent.internal.configuration.Configuration; +import com.microsoft.applicationinsights.agent.internal.preagg.PreAggregatedMetricsCustomizer; import io.opentelemetry.instrumentation.api.incubator.instrumenter.InstrumenterCustomizer; import io.opentelemetry.instrumentation.api.incubator.instrumenter.InstrumenterCustomizerProvider; /** * InstrumenterCustomizerProvider implementation that adds Application Insights context customizer - * to all instrumenters. This replaces the need for copying the InstrumenterBuilder class. + * to all instrumenters, and the pre-aggregated metrics customizer to http and rpc instrumenters. + * This replaces the need for copying the InstrumenterBuilder class. */ @AutoService(InstrumenterCustomizerProvider.class) public class AiInstrumenterCustomizerProvider implements InstrumenterCustomizerProvider { @@ -20,5 +23,10 @@ public void customize(InstrumenterCustomizer customizer) { // Add the Application Insights context customizer to all instrumenters // This was previously done by modifying the InstrumenterBuilder constructor customizer.addContextCustomizer(AiContextCustomizerHolder.getInstance()); + + Configuration configuration = FirstEntryPoint.getConfiguration(); + if (configuration != null && configuration.internal.preAggregatedStandardMetrics.enabled) { + PreAggregatedMetricsCustomizer.customize(customizer); + } } } diff --git a/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/init/SecondEntryPoint.java b/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/init/SecondEntryPoint.java index 6c1cce0cb45..4998cd50bb6 100644 --- a/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/init/SecondEntryPoint.java +++ b/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/init/SecondEntryPoint.java @@ -39,6 +39,7 @@ import com.microsoft.applicationinsights.agent.internal.exporter.AgentSpanExporter; import com.microsoft.applicationinsights.agent.internal.httpclient.LazyHttpClient; import com.microsoft.applicationinsights.agent.internal.legacyheaders.AiLegacyHeaderSpanProcessor; +import com.microsoft.applicationinsights.agent.internal.preagg.AiViewRegistry; import com.microsoft.applicationinsights.agent.internal.processors.ExporterWithLogProcessor; import com.microsoft.applicationinsights.agent.internal.processors.ExporterWithSpanProcessor; import com.microsoft.applicationinsights.agent.internal.processors.LogExporterWithAttributeProcessor; @@ -65,7 +66,6 @@ import io.opentelemetry.sdk.metrics.SdkMeterProviderBuilder; import io.opentelemetry.sdk.metrics.View; import io.opentelemetry.sdk.metrics.export.MetricExporter; -import io.opentelemetry.sdk.metrics.internal.view.AiViewRegistry; import io.opentelemetry.sdk.trace.SdkTracerProviderBuilder; import io.opentelemetry.sdk.trace.export.SpanExporter; import io.opentelemetry.sdk.trace.samplers.SamplingDecision; diff --git a/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/preagg/AiViewRegistry.java b/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/preagg/AiViewRegistry.java new file mode 100644 index 00000000000..d990dca0ab6 --- /dev/null +++ b/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/preagg/AiViewRegistry.java @@ -0,0 +1,23 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.microsoft.applicationinsights.agent.internal.preagg; + +import io.opentelemetry.sdk.metrics.InstrumentSelector; +import io.opentelemetry.sdk.metrics.SdkMeterProviderBuilder; +import io.opentelemetry.sdk.metrics.View; + +// the Application Insights specific attributes are added by PreAggregatedMetricsCustomizer, +// these views only limit the metric attributes to the ones used by the pre-aggregated metrics +public class AiViewRegistry { + + public static void registerViews(SdkMeterProviderBuilder builder) { + for (MetricView view : MetricView.values()) { + builder.registerView( + InstrumentSelector.builder().setName(view.getInstrumentName()).build(), + View.builder().setAttributeFilter(view.getRetainedAttributeKeys()).build()); + } + } + + private AiViewRegistry() {} +} diff --git a/agent/agent-tooling/src/main/java/io/opentelemetry/sdk/metrics/internal/view/MetricView.java b/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/preagg/MetricView.java similarity index 77% rename from agent/agent-tooling/src/main/java/io/opentelemetry/sdk/metrics/internal/view/MetricView.java rename to agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/preagg/MetricView.java index ad57a6de8ae..747f3131029 100644 --- a/agent/agent-tooling/src/main/java/io/opentelemetry/sdk/metrics/internal/view/MetricView.java +++ b/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/preagg/MetricView.java @@ -1,8 +1,9 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. -package io.opentelemetry.sdk.metrics.internal.view; +package com.microsoft.applicationinsights.agent.internal.preagg; +import com.azure.monitor.opentelemetry.autoconfigure.implementation.AiSemanticAttributes; import io.opentelemetry.api.common.AttributeKey; import io.opentelemetry.semconv.HttpAttributes; import io.opentelemetry.semconv.ServerAttributes; @@ -34,12 +35,18 @@ String getInstrumentName() { return instrumentName; } - Set> getAttributeKeys() { - return attributeKeys; - } - - boolean isCaptureSynthetic() { - return captureSynthetic; + // the view attribute keys plus the internal attributes added by PreAggregatedMetricsCustomizer + Set getRetainedAttributeKeys() { + Set keys = new HashSet<>(); + for (AttributeKey key : attributeKeys) { + keys.add(key.getKey()); + } + keys.add(AiSemanticAttributes.INTERNAL_CONNECTION_STRING.getKey()); + keys.add(AiSemanticAttributes.INTERNAL_ROLE_NAME.getKey()); + if (captureSynthetic) { + keys.add(AiSemanticAttributes.IS_SYNTHETIC.getKey()); + } + return keys; } private static Set> httpClientDurationAttributeKeys() { diff --git a/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/preagg/MetricsState.java b/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/preagg/MetricsState.java new file mode 100644 index 00000000000..b2bc767def4 --- /dev/null +++ b/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/preagg/MetricsState.java @@ -0,0 +1,68 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.microsoft.applicationinsights.agent.internal.preagg; + +import io.opentelemetry.context.Context; +import io.opentelemetry.context.ContextKey; +import io.opentelemetry.instrumentation.api.incubator.semconv.rpc.RpcClientMetrics; +import io.opentelemetry.instrumentation.api.incubator.semconv.rpc.RpcServerMetrics; +import io.opentelemetry.instrumentation.api.semconv.http.HttpClientMetrics; +import io.opentelemetry.instrumentation.api.semconv.http.HttpServerMetrics; +import java.lang.reflect.Field; +import javax.annotation.Nullable; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Detects whether the upstream duration metric will be recorded for an operation. + * + *

Some instrumenters have http or rpc attributes but don't record the duration metrics (e.g. AWS + * SDK, RMI). The upstream metrics store their per-operation state in the context in onStart() and + * only record the duration in onEnd() when that state is present, so checking for the state tells + * whether the pre-aggregated metric will be recorded. + */ +enum MetricsState { + HTTP_SERVER(HttpServerMetrics.class, "HTTP_SERVER_METRICS_STATE"), + HTTP_CLIENT(HttpClientMetrics.class, "HTTP_CLIENT_REQUEST_METRICS_STATE"), + RPC_SERVER(RpcServerMetrics.class, "RPC_SERVER_REQUEST_METRICS_STATE"), + RPC_CLIENT(RpcClientMetrics.class, "RPC_CLIENT_REQUEST_METRICS_STATE"); + + private static final Logger logger = LoggerFactory.getLogger(MetricsState.class); + + @SuppressWarnings("ImmutableEnumChecker") // ContextKey is immutable + @Nullable + private final ContextKey contextKey; + + MetricsState(Class metricsClass, String fieldName) { + this.contextKey = lookupContextKey(metricsClass, fieldName); + } + + boolean isResolved() { + return contextKey != null; + } + + boolean isPresent(Context context) { + if (contextKey == null) { + // fall back to assuming the metric will be recorded + return true; + } + return context.get(contextKey) != null; + } + + @Nullable + private static ContextKey lookupContextKey(Class metricsClass, String fieldName) { + try { + Field field = metricsClass.getDeclaredField(fieldName); + field.setAccessible(true); + return (ContextKey) field.get(null); + } catch (ReflectiveOperationException | RuntimeException e) { + logger.warn( + "Unable to find {}.{}, pre-aggregated metrics may be inaccurate", + metricsClass.getName(), + fieldName, + e); + return null; + } + } +} diff --git a/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/preagg/PreAggregatedMetricsCustomizer.java b/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/preagg/PreAggregatedMetricsCustomizer.java new file mode 100644 index 00000000000..9536265133d --- /dev/null +++ b/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/preagg/PreAggregatedMetricsCustomizer.java @@ -0,0 +1,129 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.microsoft.applicationinsights.agent.internal.preagg; + +import static io.opentelemetry.instrumentation.api.incubator.instrumenter.InstrumenterCustomizer.InstrumentationType.HTTP_CLIENT; +import static io.opentelemetry.instrumentation.api.incubator.instrumenter.InstrumenterCustomizer.InstrumentationType.HTTP_SERVER; +import static io.opentelemetry.instrumentation.api.incubator.instrumenter.InstrumenterCustomizer.InstrumentationType.RPC_CLIENT; +import static io.opentelemetry.instrumentation.api.incubator.instrumenter.InstrumenterCustomizer.InstrumentationType.RPC_SERVER; + +import com.azure.monitor.opentelemetry.autoconfigure.implementation.AiSemanticAttributes; +import com.microsoft.applicationinsights.agent.internal.init.AiContextKeys; +import io.opentelemetry.api.common.Attributes; +import io.opentelemetry.api.common.AttributesBuilder; +import io.opentelemetry.api.trace.Span; +import io.opentelemetry.context.Context; +import io.opentelemetry.context.ContextKey; +import io.opentelemetry.instrumentation.api.incubator.instrumenter.InstrumenterCustomizer; +import io.opentelemetry.instrumentation.api.instrumenter.AttributesExtractor; +import io.opentelemetry.instrumentation.api.instrumenter.ContextCustomizer; +import java.util.ArrayList; +import java.util.List; +import javax.annotation.Nullable; + +/** + * Adds the Application Insights specific attributes to the pre-aggregated http and rpc duration + * metrics, and marks the span as pre-aggregated so that the exporter doesn't also generate standard + * metrics from it. + * + *

The attributes are added in {@link #onEnd}, which runs right before the upstream metrics + * record the duration with the same context and attributes. + */ +public final class PreAggregatedMetricsCustomizer + implements ContextCustomizer, AttributesExtractor { + + private static final ContextKey IS_SYNTHETIC = + ContextKey.named("applicationinsights.internal.is_synthetic"); + + private final List metricsStates; + private final boolean captureSynthetic; + + public static void customize(InstrumenterCustomizer customizer) { + List metricsStates = new ArrayList<>(); + if (customizer.hasType(HTTP_SERVER)) { + metricsStates.add(MetricsState.HTTP_SERVER); + } + if (customizer.hasType(HTTP_CLIENT)) { + metricsStates.add(MetricsState.HTTP_CLIENT); + } + if (customizer.hasType(RPC_SERVER)) { + metricsStates.add(MetricsState.RPC_SERVER); + } + if (customizer.hasType(RPC_CLIENT)) { + metricsStates.add(MetricsState.RPC_CLIENT); + } + if (metricsStates.isEmpty()) { + return; + } + boolean captureSynthetic = metricsStates.contains(MetricsState.HTTP_SERVER); + PreAggregatedMetricsCustomizer instance = + new PreAggregatedMetricsCustomizer(metricsStates, captureSynthetic); + if (captureSynthetic) { + customizer.addContextCustomizer(instance); + } + customizer.addAttributesExtractor(instance); + } + + PreAggregatedMetricsCustomizer(List metricsStates, boolean captureSynthetic) { + this.metricsStates = metricsStates; + this.captureSynthetic = captureSynthetic; + } + + // context customizers receive the start attributes from all of the attributes extractors, + // which is needed to get the user agent + @Override + public Context onStart(Context parentContext, Object request, Attributes startAttributes) { + boolean synthetic = UserAgents.isBot(startAttributes); + if (synthetic == Boolean.TRUE.equals(parentContext.get(IS_SYNTHETIC))) { + return parentContext; + } + return parentContext.with(IS_SYNTHETIC, synthetic); + } + + @Override + public void onStart(AttributesBuilder attributes, Context parentContext, Object request) {} + + @Override + public void onEnd( + AttributesBuilder attributes, + Context context, + Object request, + @Nullable Object response, + @Nullable Throwable error) { + + if (!isMetricRecorded(context)) { + return; + } + + // this is needed for detecting telemetry signals that will trigger pre-aggregated metrics via + // auto instrumentations + Span.fromContext(context).setAttribute(AiSemanticAttributes.IS_PRE_AGGREGATED, true); + + String connectionString = context.get(AiContextKeys.CONNECTION_STRING); + if (connectionString != null) { + // support for connectionStringOverrides + // and for Classic SDK's setConnectionString() + attributes.put(AiSemanticAttributes.INTERNAL_CONNECTION_STRING, connectionString); + } + String roleName = context.get(AiContextKeys.ROLE_NAME); + if (roleName != null) { + // support for roleNameOverrides and for Classic SDK's setConnectionString() + // and Classic SDK set... + attributes.put(AiSemanticAttributes.INTERNAL_ROLE_NAME, roleName); + } + if (captureSynthetic) { + attributes.put( + AiSemanticAttributes.IS_SYNTHETIC, Boolean.TRUE.equals(context.get(IS_SYNTHETIC))); + } + } + + private boolean isMetricRecorded(Context context) { + for (MetricsState metricsState : metricsStates) { + if (metricsState.isPresent(context)) { + return true; + } + } + return false; + } +} diff --git a/agent/agent-tooling/src/main/java/io/opentelemetry/sdk/metrics/internal/view/UserAgents.java b/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/preagg/UserAgents.java similarity index 87% rename from agent/agent-tooling/src/main/java/io/opentelemetry/sdk/metrics/internal/view/UserAgents.java rename to agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/preagg/UserAgents.java index c50028b7b87..a897995357c 100644 --- a/agent/agent-tooling/src/main/java/io/opentelemetry/sdk/metrics/internal/view/UserAgents.java +++ b/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/preagg/UserAgents.java @@ -1,7 +1,7 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. -package io.opentelemetry.sdk.metrics.internal.view; +package com.microsoft.applicationinsights.agent.internal.preagg; import io.opentelemetry.api.common.Attributes; import io.opentelemetry.semconv.UserAgentAttributes; diff --git a/agent/agent-tooling/src/main/java/io/opentelemetry/sdk/metrics/ViewBuilderAccessor.java b/agent/agent-tooling/src/main/java/io/opentelemetry/sdk/metrics/ViewBuilderAccessor.java deleted file mode 100644 index bcf5b849c18..00000000000 --- a/agent/agent-tooling/src/main/java/io/opentelemetry/sdk/metrics/ViewBuilderAccessor.java +++ /dev/null @@ -1,15 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -package io.opentelemetry.sdk.metrics; - -import io.opentelemetry.sdk.metrics.internal.view.AttributesProcessor; - -public class ViewBuilderAccessor { - - public static void add(ViewBuilder viewBuilder, AttributesProcessor attributeProcessor) { - viewBuilder.addAttributesProcessor(attributeProcessor); - } - - private ViewBuilderAccessor() {} -} diff --git a/agent/agent-tooling/src/main/java/io/opentelemetry/sdk/metrics/internal/view/AiViewRegistry.java b/agent/agent-tooling/src/main/java/io/opentelemetry/sdk/metrics/internal/view/AiViewRegistry.java deleted file mode 100644 index 1fe76bff193..00000000000 --- a/agent/agent-tooling/src/main/java/io/opentelemetry/sdk/metrics/internal/view/AiViewRegistry.java +++ /dev/null @@ -1,38 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -package io.opentelemetry.sdk.metrics.internal.view; - -import io.opentelemetry.api.common.AttributeKey; -import io.opentelemetry.sdk.metrics.InstrumentSelector; -import io.opentelemetry.sdk.metrics.SdkMeterProviderBuilder; -import io.opentelemetry.sdk.metrics.View; -import io.opentelemetry.sdk.metrics.ViewBuilder; -import io.opentelemetry.sdk.metrics.ViewBuilderAccessor; -import java.util.Set; - -// there's already a ViewRegistry class in this SDK package -// (and we have to hijack the package to get access to package-private details for now) -public class AiViewRegistry { - - public static void registerViews(SdkMeterProviderBuilder builder) { - for (MetricView view : MetricView.values()) { - registerView( - builder, view.getInstrumentName(), view.getAttributeKeys(), view.isCaptureSynthetic()); - } - } - - private static void registerView( - SdkMeterProviderBuilder builder, - String instrumentName, - Set> attributeKeys, - boolean captureSynthetic) { - ViewBuilder viewBuilder = View.builder(); - ViewBuilderAccessor.add( - viewBuilder, new MetricViewAttributesProcessor(attributeKeys, captureSynthetic)); - builder.registerView( - InstrumentSelector.builder().setName(instrumentName).build(), viewBuilder.build()); - } - - private AiViewRegistry() {} -} diff --git a/agent/agent-tooling/src/main/java/io/opentelemetry/sdk/metrics/internal/view/MetricViewAttributesProcessor.java b/agent/agent-tooling/src/main/java/io/opentelemetry/sdk/metrics/internal/view/MetricViewAttributesProcessor.java deleted file mode 100644 index c9effa69308..00000000000 --- a/agent/agent-tooling/src/main/java/io/opentelemetry/sdk/metrics/internal/view/MetricViewAttributesProcessor.java +++ /dev/null @@ -1,70 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -package io.opentelemetry.sdk.metrics.internal.view; - -import com.azure.monitor.opentelemetry.autoconfigure.implementation.AiSemanticAttributes; -import com.microsoft.applicationinsights.agent.internal.init.AiContextKeys; -import io.opentelemetry.api.common.AttributeKey; -import io.opentelemetry.api.common.Attributes; -import io.opentelemetry.api.common.AttributesBuilder; -import io.opentelemetry.api.trace.Span; -import io.opentelemetry.context.Context; -import java.util.Set; -import java.util.function.BiConsumer; - -@SuppressWarnings("rawtypes") // safe use of raw type - generic type verified by runtime context -class MetricViewAttributesProcessor extends AttributesProcessor { - - private final Set> attributeKeys; - private final boolean captureSynthetic; - - MetricViewAttributesProcessor(Set> attributeKeys, boolean captureSynthetic) { - this.attributeKeys = attributeKeys; - this.captureSynthetic = captureSynthetic; - } - - @Override - public Attributes process(Attributes incoming, Context context) { - - // this is needed for detecting telemetry signals that will trigger pre-aggregated metrics via - // auto instrumentations - Span.fromContext(context).setAttribute(AiSemanticAttributes.IS_PRE_AGGREGATED, true); - - AttributesBuilder builder = Attributes.builder(); - String connectionString = context.get(AiContextKeys.CONNECTION_STRING); - if (connectionString != null) { - // support for connectionStringOverrides - // and for Classic SDK's setConnectionString() - builder.put(AiSemanticAttributes.INTERNAL_CONNECTION_STRING, connectionString); - } - String roleName = context.get(AiContextKeys.ROLE_NAME); - if (roleName != null) { - // support for roleNameOverrides and for Classic SDK's setConnectionString() - // and Classic SDK set... - builder.put(AiSemanticAttributes.INTERNAL_ROLE_NAME, roleName); - } - applyView(builder, incoming, attributeKeys); - if (captureSynthetic) { - builder.put(AiSemanticAttributes.IS_SYNTHETIC, UserAgents.isBot(incoming)); - } - return builder.build(); - } - - @Override - public boolean usesContext() { - return true; - } - - @SuppressWarnings("unchecked") // safe unchecked cast - type verified by runtime context - private static void applyView( - AttributesBuilder filtered, Attributes attributes, Set> view) { - attributes.forEach( - (BiConsumer) - (key, value) -> { - if (view.contains(key)) { - filtered.put(key, value); - } - }); - } -} diff --git a/agent/agent-tooling/src/test/java/com/microsoft/applicationinsights/agent/internal/PreAggregatedMetricsTest.java b/agent/agent-tooling/src/test/java/com/microsoft/applicationinsights/agent/internal/preagg/PreAggregatedMetricsTest.java similarity index 70% rename from agent/agent-tooling/src/test/java/com/microsoft/applicationinsights/agent/internal/PreAggregatedMetricsTest.java rename to agent/agent-tooling/src/test/java/com/microsoft/applicationinsights/agent/internal/preagg/PreAggregatedMetricsTest.java index 70c1f71c6fc..362ef8aaad7 100644 --- a/agent/agent-tooling/src/test/java/com/microsoft/applicationinsights/agent/internal/PreAggregatedMetricsTest.java +++ b/agent/agent-tooling/src/test/java/com/microsoft/applicationinsights/agent/internal/preagg/PreAggregatedMetricsTest.java @@ -1,7 +1,7 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. -package com.microsoft.applicationinsights.agent.internal; +package com.microsoft.applicationinsights.agent.internal.preagg; import static com.azure.monitor.opentelemetry.autoconfigure.implementation.preaggregatedmetrics.DependencyExtractor.DEPENDENCIES_DURATION; import static com.azure.monitor.opentelemetry.autoconfigure.implementation.preaggregatedmetrics.DependencyExtractor.DEPENDENCY_RESULT_CODE; @@ -19,12 +19,15 @@ import static io.opentelemetry.sdk.testing.assertj.OpenTelemetryAssertions.assertThat; import static io.opentelemetry.sdk.testing.assertj.OpenTelemetryAssertions.equalTo; +import com.azure.monitor.opentelemetry.autoconfigure.implementation.AiSemanticAttributes; import com.azure.monitor.opentelemetry.autoconfigure.implementation.MetricDataMapper; import com.azure.monitor.opentelemetry.autoconfigure.implementation.builders.MetricTelemetryBuilder; import com.azure.monitor.opentelemetry.autoconfigure.implementation.models.MetricsData; import com.azure.monitor.opentelemetry.autoconfigure.implementation.models.TelemetryItem; +import com.microsoft.applicationinsights.agent.internal.init.AiContextKeys; import io.opentelemetry.api.common.AttributeKey; import io.opentelemetry.api.common.Attributes; +import io.opentelemetry.api.common.AttributesBuilder; import io.opentelemetry.api.trace.Span; import io.opentelemetry.api.trace.SpanContext; import io.opentelemetry.api.trace.TraceFlags; @@ -37,12 +40,17 @@ import io.opentelemetry.sdk.metrics.SdkMeterProvider; import io.opentelemetry.sdk.metrics.SdkMeterProviderBuilder; import io.opentelemetry.sdk.metrics.data.MetricData; -import io.opentelemetry.sdk.metrics.internal.view.AiViewRegistry; import io.opentelemetry.sdk.testing.exporter.InMemoryMetricReader; +import io.opentelemetry.sdk.testing.exporter.InMemorySpanExporter; +import io.opentelemetry.sdk.trace.SdkTracerProvider; +import io.opentelemetry.sdk.trace.data.SpanData; +import io.opentelemetry.sdk.trace.export.SimpleSpanProcessor; import io.opentelemetry.semconv.HttpAttributes; import io.opentelemetry.semconv.ServerAttributes; +import io.opentelemetry.semconv.UserAgentAttributes; import io.opentelemetry.semconv.incubating.RpcIncubatingAttributes; import java.util.Collection; +import java.util.Collections; import java.util.Comparator; import java.util.HashMap; import java.util.Map; @@ -50,11 +58,17 @@ import java.util.stream.Collectors; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.EnumSource; public class PreAggregatedMetricsTest { + private static final Object REQUEST = new Object(); + private InMemoryMetricReader metricReader; private SdkMeterProvider meterProvider; + private InMemorySpanExporter spanExporter; + private SdkTracerProvider tracerProvider; @BeforeEach void setup() { @@ -62,6 +76,11 @@ void setup() { SdkMeterProviderBuilder builder = SdkMeterProvider.builder(); AiViewRegistry.registerViews(builder); meterProvider = builder.registerMetricReader(metricReader).build(); + spanExporter = InMemorySpanExporter.create(); + tracerProvider = + SdkTracerProvider.builder() + .addSpanProcessor(SimpleSpanProcessor.create(spanExporter)) + .build(); } @SuppressWarnings("SystemOut") // legitimate use of System.out for logging/output @@ -99,8 +118,8 @@ void generateHttpClientMetrics() { TraceFlags.getSampled(), TraceState.getDefault()))); - Context context1 = listener.onStart(parent, requestAttributes, nanos(100)); - listener.onEnd(context1, responseAttributes, nanos(250)); + Context context1 = start(MetricsState.HTTP_CLIENT, false, listener, parent, requestAttributes); + end(MetricsState.HTTP_CLIENT, false, listener, context1, responseAttributes); Collection metricDataCollection = metricReader.collectAllMetrics(); metricDataCollection = @@ -169,11 +188,11 @@ void generateRpcClientMetrics() { TraceFlags.getSampled(), TraceState.getDefault()))); - Context context1 = listener.onStart(parent, requestAttributes, nanos(100)); + Context context1 = start(MetricsState.RPC_CLIENT, false, listener, parent, requestAttributes); assertThat(metricReader.collectAllMetrics()).isEmpty(); - listener.onEnd(context1, responseAttributes1, nanos(250)); + end(MetricsState.RPC_CLIENT, false, listener, context1, responseAttributes1); Collection metricDataCollection = metricReader.collectAllMetrics(); for (MetricData metricData : metricDataCollection) { @@ -243,8 +262,8 @@ void generateHttpServerMetrics() { TraceFlags.getSampled(), TraceState.getDefault()); Context parent1 = Context.root().with(Span.wrap(spanContext1)); - Context context1 = listener.onStart(parent1, requestAttributes, nanos(100)); - listener.onEnd(context1, responseAttributes, nanos(250)); + Context context1 = start(MetricsState.HTTP_SERVER, true, listener, parent1, requestAttributes); + end(MetricsState.HTTP_SERVER, true, listener, context1, responseAttributes); Collection metricDataCollection = metricReader.collectAllMetrics(); MetricData target = null; @@ -287,6 +306,120 @@ void generateHttpServerMetrics() { .containsExactlyInAnyOrderEntriesOf(generateExpectedRequestCustomDimensions("http")); } + @Test + void httpServerMetricsCaptureSyntheticAndOverrides() { + OperationListener listener = HttpServerMetrics.get().create(meterProvider.get("test")); + + Attributes requestAttributes = + Attributes.builder() + .put("http.request.method", "GET") + .put("url.path", "/") + .put(UserAgentAttributes.USER_AGENT_ORIGINAL, "Mozilla/5.0 (compatible; AlwaysOn)") + .build(); + Attributes responseAttributes = + Attributes.builder().put("http.response.status_code", 200).build(); + + Span span = tracerProvider.get("test").spanBuilder("GET /").startSpan(); + Context parent = + Context.root() + .with(span) + .with(AiContextKeys.CONNECTION_STRING, "InstrumentationKey=override") + .with(AiContextKeys.ROLE_NAME, "role-override"); + Context context = start(MetricsState.HTTP_SERVER, true, listener, parent, requestAttributes); + end(MetricsState.HTTP_SERVER, true, listener, context, responseAttributes); + span.end(); + + assertThat(getMetric("http.server.request.duration")) + .hasHistogramSatisfying(histogram -> + histogram.hasPointsSatisfying(point -> + point.hasAttributesSatisfyingExactly( + equalTo(HttpAttributes.HTTP_RESPONSE_STATUS_CODE, 200), + equalTo(AiSemanticAttributes.IS_SYNTHETIC, true), + equalTo( + AiSemanticAttributes.INTERNAL_CONNECTION_STRING, + "InstrumentationKey=override"), + equalTo(AiSemanticAttributes.INTERNAL_ROLE_NAME, "role-override")))); + + assertThat(getSingleSpan().getAttributes().get(AiSemanticAttributes.IS_PRE_AGGREGATED)) + .isTrue(); + } + + // e.g. the AWS SDK and RMI instrumenters have rpc/http attributes but don't record the duration + // metrics, so their spans must not be marked as pre-aggregated + @ParameterizedTest + @EnumSource(MetricsState.class) + void spanIsNotPreAggregatedWhenDurationMetricIsNotRecorded(MetricsState metricsState) { + PreAggregatedMetricsCustomizer customizer = + new PreAggregatedMetricsCustomizer( + Collections.singletonList(metricsState), metricsState == MetricsState.HTTP_SERVER); + + Span span = tracerProvider.get("test").spanBuilder("span").startSpan(); + Context context = + customizer.onStart( + Context.root().with(span).with(AiContextKeys.ROLE_NAME, "role-override"), + REQUEST, + Attributes.empty()); + AttributesBuilder endAttributes = Attributes.builder(); + customizer.onEnd(endAttributes, context, REQUEST, null, null); + span.end(); + + assertThat(endAttributes.build().isEmpty()).isTrue(); + assertThat(getSingleSpan().getAttributes().get(AiSemanticAttributes.IS_PRE_AGGREGATED)) + .isNull(); + } + + // MetricsState looks up private fields of the upstream metrics classes, this fails if they are + // renamed when updating the OpenTelemetry instrumentation version + @ParameterizedTest + @EnumSource(MetricsState.class) + void metricsStateIsResolved(MetricsState metricsState) { + assertThat(metricsState.isResolved()).isTrue(); + } + + // mirrors the order in which Instrumenter calls the context customizers, attributes extractors + // and operation listeners + private static Context start( + MetricsState metricsState, + boolean captureSynthetic, + OperationListener listener, + Context parentContext, + Attributes startAttributes) { + PreAggregatedMetricsCustomizer customizer = + new PreAggregatedMetricsCustomizer( + Collections.singletonList(metricsState), captureSynthetic); + Context context = parentContext; + if (captureSynthetic) { + context = customizer.onStart(context, REQUEST, startAttributes); + } + return listener.onStart(context, startAttributes, nanos(100)); + } + + private static void end( + MetricsState metricsState, + boolean captureSynthetic, + OperationListener listener, + Context context, + Attributes endAttributes) { + PreAggregatedMetricsCustomizer customizer = + new PreAggregatedMetricsCustomizer( + Collections.singletonList(metricsState), captureSynthetic); + AttributesBuilder attributes = endAttributes.toBuilder(); + customizer.onEnd(attributes, context, REQUEST, null, null); + listener.onEnd(context, attributes.build(), nanos(250)); + } + + private MetricData getMetric(String name) { + return metricReader.collectAllMetrics().stream() + .filter(metricData -> metricData.getName().equals(name)) + .findFirst() + .orElseThrow(() -> new AssertionError("metric not found: " + name)); + } + + private SpanData getSingleSpan() { + assertThat(spanExporter.getFinishedSpanItems()).hasSize(1); + return spanExporter.getFinishedSpanItems().get(0); + } + private static long nanos(int millis) { return TimeUnit.MILLISECONDS.toNanos(millis); } From 1ce5abe2a55e1253ca0aa4300a96ccb4d7de0038 Mon Sep 17 00:00:00 2001 From: Rajkumar Rangaraj Date: Thu, 12 Nov 2026 15:25:52 -0800 Subject: [PATCH 2/3] Address review feedback on pre-aggregated metrics customizer - Don't customize rpc instrumenters when only stable rpc semconv is emitted, since rpc.*.call.duration are not pre-aggregated standard metrics - Only mark the span when the metrics state was created by the current operation, not inherited from a parent operation - Don't use the static logger from the enum constructors, it isn't initialized yet - Add tests using real Instrumenters Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../agent/internal/preagg/MetricsState.java | 55 +++- .../PreAggregatedMetricsCustomizer.java | 44 ++- .../PreAggregatedMetricsInstrumenterTest.java | 252 ++++++++++++++++++ .../preagg/PreAggregatedMetricsTest.java | 54 +++- 4 files changed, 376 insertions(+), 29 deletions(-) create mode 100644 agent/agent-tooling/src/test/java/com/microsoft/applicationinsights/agent/internal/preagg/PreAggregatedMetricsInstrumenterTest.java diff --git a/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/preagg/MetricsState.java b/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/preagg/MetricsState.java index b2bc767def4..ddbd8be1a46 100644 --- a/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/preagg/MetricsState.java +++ b/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/preagg/MetricsState.java @@ -11,7 +11,6 @@ import io.opentelemetry.instrumentation.api.semconv.http.HttpServerMetrics; import java.lang.reflect.Field; import javax.annotation.Nullable; -import org.slf4j.Logger; import org.slf4j.LoggerFactory; /** @@ -21,6 +20,10 @@ * SDK, RMI). The upstream metrics store their per-operation state in the context in onStart() and * only record the duration in onEnd() when that state is present, so checking for the state tells * whether the pre-aggregated metric will be recorded. + * + *

The state is inherited by nested operations, so the state that was already in the parent + * context is captured before the operation listeners run (see {@link #captureInherited}), and only + * a state created by the current operation counts. */ enum MetricsState { HTTP_SERVER(HttpServerMetrics.class, "HTTP_SERVER_METRICS_STATE"), @@ -28,41 +31,71 @@ enum MetricsState { RPC_SERVER(RpcServerMetrics.class, "RPC_SERVER_REQUEST_METRICS_STATE"), RPC_CLIENT(RpcClientMetrics.class, "RPC_CLIENT_REQUEST_METRICS_STATE"); - private static final Logger logger = LoggerFactory.getLogger(MetricsState.class); - @SuppressWarnings("ImmutableEnumChecker") // ContextKey is immutable @Nullable private final ContextKey contextKey; + @SuppressWarnings("ImmutableEnumChecker") // ContextKey is immutable + private final ContextKey inheritedContextKey; + MetricsState(Class metricsClass, String fieldName) { this.contextKey = lookupContextKey(metricsClass, fieldName); + this.inheritedContextKey = + ContextKey.named("applicationinsights.internal.inherited_metrics_state." + fieldName); } boolean isResolved() { return contextKey != null; } - boolean isPresent(Context context) { + // needs to be called before the operation listeners' onStart() + Context captureInherited(Context parentContext) { + if (contextKey == null) { + return parentContext; + } + return parentContext.with(inheritedContextKey, new Inherited(parentContext.get(contextKey))); + } + + // identity comparison is intended, the upstream state is a value class + @SuppressWarnings("ReferenceEquality") + boolean isRecorded(Context context) { if (contextKey == null) { // fall back to assuming the metric will be recorded return true; } - return context.get(contextKey) != null; + Object state = context.get(contextKey); + if (state == null) { + return false; + } + Inherited inherited = context.get(inheritedContextKey); + // inherited is null when the operation was not started by a customized instrumenter + return inherited == null || state != inherited.state; } @Nullable - private static ContextKey lookupContextKey(Class metricsClass, String fieldName) { + static ContextKey lookupContextKey(Class metricsClass, String fieldName) { try { Field field = metricsClass.getDeclaredField(fieldName); field.setAccessible(true); return (ContextKey) field.get(null); } catch (ReflectiveOperationException | RuntimeException e) { - logger.warn( - "Unable to find {}.{}, pre-aggregated metrics may be inaccurate", - metricsClass.getName(), - fieldName, - e); + // not using a static logger field, since this is called from the enum constructors which run + // before the other static fields are initialized + LoggerFactory.getLogger(MetricsState.class) + .warn( + "Unable to find {}.{}, pre-aggregated metrics may be inaccurate", + metricsClass.getName(), + fieldName, + e); return null; } } + + private static final class Inherited { + @Nullable private final Object state; + + private Inherited(@Nullable Object state) { + this.state = state; + } + } } diff --git a/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/preagg/PreAggregatedMetricsCustomizer.java b/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/preagg/PreAggregatedMetricsCustomizer.java index 9536265133d..cff0f822f9f 100644 --- a/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/preagg/PreAggregatedMetricsCustomizer.java +++ b/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/preagg/PreAggregatedMetricsCustomizer.java @@ -18,6 +18,7 @@ import io.opentelemetry.instrumentation.api.incubator.instrumenter.InstrumenterCustomizer; import io.opentelemetry.instrumentation.api.instrumenter.AttributesExtractor; import io.opentelemetry.instrumentation.api.instrumenter.ContextCustomizer; +import io.opentelemetry.instrumentation.api.internal.SemconvStability; import java.util.ArrayList; import java.util.List; import javax.annotation.Nullable; @@ -29,6 +30,10 @@ * *

The attributes are added in {@link #onEnd}, which runs right before the upstream metrics * record the duration with the same context and attributes. + * + *

Note: this only applies to the agent's instrumenters. Duration metrics recorded by + * OpenTelemetry instrumentation libraries bundled in the application (through the OpenTelemetry API + * bridge) only go through the views, so their spans are not marked as pre-aggregated. */ public final class PreAggregatedMetricsCustomizer implements ContextCustomizer, AttributesExtractor { @@ -40,6 +45,11 @@ public final class PreAggregatedMetricsCustomizer private final boolean captureSynthetic; public static void customize(InstrumenterCustomizer customizer) { + customize(customizer, SemconvStability.emitOldRpcSemconv()); + } + + // visible for testing + static void customize(InstrumenterCustomizer customizer, boolean emitOldRpcSemconv) { List metricsStates = new ArrayList<>(); if (customizer.hasType(HTTP_SERVER)) { metricsStates.add(MetricsState.HTTP_SERVER); @@ -47,21 +57,21 @@ public static void customize(InstrumenterCustomizer customizer) { if (customizer.hasType(HTTP_CLIENT)) { metricsStates.add(MetricsState.HTTP_CLIENT); } - if (customizer.hasType(RPC_SERVER)) { + // with stable rpc semconv only, the upstream rpc metrics record rpc.*.call.duration instead of + // rpc.*.duration, which are not pre-aggregated standard metrics + if (emitOldRpcSemconv && customizer.hasType(RPC_SERVER)) { metricsStates.add(MetricsState.RPC_SERVER); } - if (customizer.hasType(RPC_CLIENT)) { + if (emitOldRpcSemconv && customizer.hasType(RPC_CLIENT)) { metricsStates.add(MetricsState.RPC_CLIENT); } if (metricsStates.isEmpty()) { return; } - boolean captureSynthetic = metricsStates.contains(MetricsState.HTTP_SERVER); PreAggregatedMetricsCustomizer instance = - new PreAggregatedMetricsCustomizer(metricsStates, captureSynthetic); - if (captureSynthetic) { - customizer.addContextCustomizer(instance); - } + new PreAggregatedMetricsCustomizer( + metricsStates, metricsStates.contains(MetricsState.HTTP_SERVER)); + customizer.addContextCustomizer(instance); customizer.addAttributesExtractor(instance); } @@ -70,15 +80,21 @@ public static void customize(InstrumenterCustomizer customizer) { this.captureSynthetic = captureSynthetic; } - // context customizers receive the start attributes from all of the attributes extractors, - // which is needed to get the user agent + // context customizers run before the operation listeners' onStart(), and receive the start + // attributes from all of the attributes extractors, which is needed to get the user agent @Override public Context onStart(Context parentContext, Object request, Attributes startAttributes) { - boolean synthetic = UserAgents.isBot(startAttributes); - if (synthetic == Boolean.TRUE.equals(parentContext.get(IS_SYNTHETIC))) { - return parentContext; + Context context = parentContext; + for (MetricsState metricsState : metricsStates) { + context = metricsState.captureInherited(context); + } + if (captureSynthetic) { + boolean synthetic = UserAgents.isBot(startAttributes); + if (synthetic != Boolean.TRUE.equals(context.get(IS_SYNTHETIC))) { + context = context.with(IS_SYNTHETIC, synthetic); + } } - return parentContext.with(IS_SYNTHETIC, synthetic); + return context; } @Override @@ -120,7 +136,7 @@ public void onEnd( private boolean isMetricRecorded(Context context) { for (MetricsState metricsState : metricsStates) { - if (metricsState.isPresent(context)) { + if (metricsState.isRecorded(context)) { return true; } } diff --git a/agent/agent-tooling/src/test/java/com/microsoft/applicationinsights/agent/internal/preagg/PreAggregatedMetricsInstrumenterTest.java b/agent/agent-tooling/src/test/java/com/microsoft/applicationinsights/agent/internal/preagg/PreAggregatedMetricsInstrumenterTest.java new file mode 100644 index 00000000000..9b69c9c804b --- /dev/null +++ b/agent/agent-tooling/src/test/java/com/microsoft/applicationinsights/agent/internal/preagg/PreAggregatedMetricsInstrumenterTest.java @@ -0,0 +1,252 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.microsoft.applicationinsights.agent.internal.preagg; + +import static io.opentelemetry.sdk.testing.assertj.OpenTelemetryAssertions.assertThat; +import static io.opentelemetry.sdk.testing.assertj.OpenTelemetryAssertions.equalTo; +import static java.util.Collections.emptyList; +import static java.util.Collections.singletonList; + +import com.azure.monitor.opentelemetry.autoconfigure.implementation.AiSemanticAttributes; +import com.microsoft.applicationinsights.agent.internal.init.AiContextKeys; +import io.opentelemetry.context.Context; +import io.opentelemetry.instrumentation.api.incubator.instrumenter.internal.InstrumenterCustomizerImpl; +import io.opentelemetry.instrumentation.api.incubator.semconv.rpc.RpcAttributesGetter; +import io.opentelemetry.instrumentation.api.incubator.semconv.rpc.RpcClientAttributesExtractor; +import io.opentelemetry.instrumentation.api.incubator.semconv.rpc.RpcClientMetrics; +import io.opentelemetry.instrumentation.api.instrumenter.Instrumenter; +import io.opentelemetry.instrumentation.api.instrumenter.InstrumenterBuilder; +import io.opentelemetry.instrumentation.api.instrumenter.SpanKindExtractor; +import io.opentelemetry.instrumentation.api.internal.InternalInstrumenterCustomizerProvider; +import io.opentelemetry.instrumentation.api.internal.InternalInstrumenterCustomizerUtil; +import io.opentelemetry.instrumentation.api.semconv.http.HttpServerAttributesExtractor; +import io.opentelemetry.instrumentation.api.semconv.http.HttpServerAttributesGetter; +import io.opentelemetry.instrumentation.api.semconv.http.HttpServerMetrics; +import io.opentelemetry.sdk.OpenTelemetrySdk; +import io.opentelemetry.sdk.metrics.SdkMeterProvider; +import io.opentelemetry.sdk.metrics.SdkMeterProviderBuilder; +import io.opentelemetry.sdk.metrics.data.MetricData; +import io.opentelemetry.sdk.testing.exporter.InMemoryMetricReader; +import io.opentelemetry.sdk.testing.exporter.InMemorySpanExporter; +import io.opentelemetry.sdk.trace.SdkTracerProvider; +import io.opentelemetry.sdk.trace.data.SpanData; +import io.opentelemetry.sdk.trace.export.SimpleSpanProcessor; +import io.opentelemetry.semconv.HttpAttributes; +import io.opentelemetry.semconv.incubating.RpcIncubatingAttributes; +import java.util.List; +import javax.annotation.Nullable; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +// uses real Instrumenters, so that the order in which the context customizers, attributes +// extractors and operation listeners are called is the one from the OpenTelemetry instrumentation +class PreAggregatedMetricsInstrumenterTest { + + private List originalCustomizerProviders; + private boolean emitOldRpcSemconv = true; + + private InMemoryMetricReader metricReader; + private InMemorySpanExporter spanExporter; + private OpenTelemetrySdk openTelemetry; + + @BeforeEach + void setup() { + originalCustomizerProviders = + InternalInstrumenterCustomizerUtil.getInstrumenterCustomizerProviders(); + InternalInstrumenterCustomizerUtil.setInstrumenterCustomizerProviders( + singletonList(customizer -> + PreAggregatedMetricsCustomizer.customize( + new InstrumenterCustomizerImpl(customizer), emitOldRpcSemconv))); + + metricReader = InMemoryMetricReader.create(); + SdkMeterProviderBuilder meterProviderBuilder = SdkMeterProvider.builder(); + AiViewRegistry.registerViews(meterProviderBuilder); + spanExporter = InMemorySpanExporter.create(); + openTelemetry = + OpenTelemetrySdk.builder() + .setTracerProvider( + SdkTracerProvider.builder() + .addSpanProcessor(SimpleSpanProcessor.create(spanExporter)) + .build()) + .setMeterProvider(meterProviderBuilder.registerMetricReader(metricReader).build()) + .build(); + } + + @AfterEach + void cleanup() { + InternalInstrumenterCustomizerUtil.setInstrumenterCustomizerProviders( + originalCustomizerProviders); + openTelemetry.close(); + } + + @Test + void httpServer() { + Instrumenter instrumenter = + Instrumenter.builder(openTelemetry, "test", request -> "GET") + .addAttributesExtractor(HttpServerAttributesExtractor.create(new HttpGetter())) + .addOperationMetrics(HttpServerMetrics.get()) + .buildInstrumenter(SpanKindExtractor.alwaysServer()); + + TestRequest request = new TestRequest("Mozilla/5.0 (compatible; AlwaysOn)"); + Context parentContext = + Context.root() + .with(AiContextKeys.CONNECTION_STRING, "InstrumentationKey=override") + .with(AiContextKeys.ROLE_NAME, "role-override"); + Context context = instrumenter.start(parentContext, request); + instrumenter.end(context, request, 200, null); + + assertThat(getMetric("http.server.request.duration")) + .hasHistogramSatisfying(histogram -> + histogram.hasPointsSatisfying(point -> + point.hasAttributesSatisfyingExactly( + equalTo(HttpAttributes.HTTP_RESPONSE_STATUS_CODE, 200), + equalTo(AiSemanticAttributes.IS_SYNTHETIC, true), + equalTo( + AiSemanticAttributes.INTERNAL_CONNECTION_STRING, + "InstrumentationKey=override"), + equalTo(AiSemanticAttributes.INTERNAL_ROLE_NAME, "role-override")))); + assertThat(getSingleSpan()).hasAttribute(AiSemanticAttributes.IS_PRE_AGGREGATED, true); + } + + @Test + void rpcClient() { + Instrumenter instrumenter = + rpcClientInstrumenterBuilder() + .addOperationMetrics(RpcClientMetrics.get()) + .buildInstrumenter(SpanKindExtractor.alwaysClient()); + + runRpcClient(instrumenter); + + assertThat(getMetric("rpc.client.duration")) + .hasHistogramSatisfying(histogram -> + histogram.hasPointsSatisfying(point -> + point.hasAttributesSatisfyingExactly( + equalTo(RpcIncubatingAttributes.RPC_SYSTEM, "test-rpc")))); + assertThat(getSingleSpan()).hasAttribute(AiSemanticAttributes.IS_PRE_AGGREGATED, true); + } + + // e.g. the RMI and AWS SDK instrumenters + @Test + void rpcClientWithoutMetrics() { + Instrumenter instrumenter = + rpcClientInstrumenterBuilder().buildInstrumenter(SpanKindExtractor.alwaysClient()); + + runRpcClient(instrumenter); + + assertThat(metricReader.collectAllMetrics()).isEmpty(); + assertThat(getSingleSpan().getAttributes().get(AiSemanticAttributes.IS_PRE_AGGREGATED)) + .isNull(); + } + + // with otel.semconv-stability.opt-in=rpc, the upstream rpc metrics record rpc.*.call.duration, + // which are not pre-aggregated standard metrics + @Test + void rpcClientIsNotCustomizedWithStableRpcSemconvOnly() { + emitOldRpcSemconv = false; + Instrumenter instrumenter = + rpcClientInstrumenterBuilder() + .addOperationMetrics(RpcClientMetrics.get()) + .buildInstrumenter(SpanKindExtractor.alwaysClient()); + + runRpcClient(instrumenter); + + assertThat(getSingleSpan().getAttributes().get(AiSemanticAttributes.IS_PRE_AGGREGATED)) + .isNull(); + } + + private InstrumenterBuilder rpcClientInstrumenterBuilder() { + return Instrumenter.builder(openTelemetry, "test", request -> "rpc") + .addAttributesExtractor(RpcClientAttributesExtractor.create(new RpcGetter())); + } + + private static void runRpcClient(Instrumenter instrumenter) { + TestRequest request = new TestRequest(null); + Context context = instrumenter.start(Context.root(), request); + instrumenter.end(context, request, null, null); + } + + private MetricData getMetric(String name) { + return metricReader.collectAllMetrics().stream() + .filter(metricData -> metricData.getName().equals(name)) + .findFirst() + .orElseThrow(() -> new AssertionError("metric not found: " + name)); + } + + private SpanData getSingleSpan() { + assertThat(spanExporter.getFinishedSpanItems()).hasSize(1); + return spanExporter.getFinishedSpanItems().get(0); + } + + private static class TestRequest { + @Nullable private final String userAgent; + + private TestRequest(@Nullable String userAgent) { + this.userAgent = userAgent; + } + } + + private static class HttpGetter implements HttpServerAttributesGetter { + + @Override + public String getHttpRequestMethod(TestRequest request) { + return "GET"; + } + + @Override + public List getHttpRequestHeader(TestRequest request, String name) { + if (name.equals("user-agent") && request.userAgent != null) { + return singletonList(request.userAgent); + } + return emptyList(); + } + + @Override + @Nullable + public Integer getHttpResponseStatusCode( + TestRequest request, Integer response, @Nullable Throwable error) { + return response; + } + + @Override + public List getHttpResponseHeader(TestRequest request, Integer response, String name) { + return emptyList(); + } + + @Override + public String getUrlScheme(TestRequest request) { + return "http"; + } + + @Override + public String getUrlPath(TestRequest request) { + return "/"; + } + + @Override + @Nullable + public String getUrlQuery(TestRequest request) { + return null; + } + } + + @SuppressWarnings("deprecation") // getSystem() and getMethod() are deprecated in newer versions + private static class RpcGetter implements RpcAttributesGetter { + + @Override + public String getSystem(TestRequest request) { + return "test-rpc"; + } + + @Override + public String getService(TestRequest request) { + return "TestService"; + } + + @Override + public String getMethod(TestRequest request) { + return "testMethod"; + } + } +} diff --git a/agent/agent-tooling/src/test/java/com/microsoft/applicationinsights/agent/internal/preagg/PreAggregatedMetricsTest.java b/agent/agent-tooling/src/test/java/com/microsoft/applicationinsights/agent/internal/preagg/PreAggregatedMetricsTest.java index 362ef8aaad7..10456ad01c3 100644 --- a/agent/agent-tooling/src/test/java/com/microsoft/applicationinsights/agent/internal/preagg/PreAggregatedMetricsTest.java +++ b/agent/agent-tooling/src/test/java/com/microsoft/applicationinsights/agent/internal/preagg/PreAggregatedMetricsTest.java @@ -376,6 +376,55 @@ void metricsStateIsResolved(MetricsState metricsState) { assertThat(metricsState.isResolved()).isTrue(); } + @Test + void missingMetricsStateFieldFallsBackGracefully() { + assertThat(MetricsState.lookupContextKey(HttpServerMetrics.class, "DOES_NOT_EXIST")).isNull(); + } + + // e.g. when span suppression is disabled, a nested http client instrumenter that doesn't record + // the duration metric inherits the metrics state of its parent + @Test + void nestedSpanWithoutMetricsIsNotMarkedFromInheritedMetricsState() { + OperationListener listener = HttpClientMetrics.get().create(meterProvider.get("test")); + Attributes attributes = Attributes.builder().put("http.response.status_code", 200).build(); + + Span parentSpan = tracerProvider.get("test").spanBuilder("parent").startSpan(); + Context parentContext = + start( + MetricsState.HTTP_CLIENT, + false, + listener, + Context.root().with(parentSpan), + Attributes.empty()); + + PreAggregatedMetricsCustomizer childCustomizer = + new PreAggregatedMetricsCustomizer( + Collections.singletonList(MetricsState.HTTP_CLIENT), false); + Span childSpan = + tracerProvider.get("test").spanBuilder("child").setParent(parentContext).startSpan(); + Context childContext = + childCustomizer.onStart(parentContext.with(childSpan), REQUEST, Attributes.empty()); + AttributesBuilder childEndAttributes = Attributes.builder(); + childCustomizer.onEnd(childEndAttributes, childContext, REQUEST, null, null); + childSpan.end(); + + end(MetricsState.HTTP_CLIENT, false, listener, parentContext, attributes); + parentSpan.end(); + + assertThat(spanExporter.getFinishedSpanItems()) + .satisfiesExactlyInAnyOrder( + span -> + assertThat(span) + .hasName("child") + .satisfies(s -> + assertThat(s.getAttributes().get(AiSemanticAttributes.IS_PRE_AGGREGATED)) + .isNull()), + span -> + assertThat(span) + .hasName("parent") + .hasAttribute(AiSemanticAttributes.IS_PRE_AGGREGATED, true)); + } + // mirrors the order in which Instrumenter calls the context customizers, attributes extractors // and operation listeners private static Context start( @@ -387,10 +436,7 @@ private static Context start( PreAggregatedMetricsCustomizer customizer = new PreAggregatedMetricsCustomizer( Collections.singletonList(metricsState), captureSynthetic); - Context context = parentContext; - if (captureSynthetic) { - context = customizer.onStart(context, REQUEST, startAttributes); - } + Context context = customizer.onStart(parentContext, REQUEST, startAttributes); return listener.onStart(context, startAttributes, nanos(100)); } From 5114ca65da1f1ebb5887dcb51399fad92d3b6d7c Mon Sep 17 00:00:00 2001 From: Rajkumar Rangaraj Date: Thu, 12 Nov 2026 22:11:27 -0800 Subject: [PATCH 3/3] Only mark spans as pre-aggregated when the Azure Monitor metric exporter is used With otel.metrics.exporter=none (or otlp only), the pre-aggregated metrics don't reach Application Insights, so the spans must not be marked as pre-aggregated, otherwise the standard metrics are not generated from them either. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../agent/internal/init/SecondEntryPoint.java | 4 ++ .../PreAggregatedMetricsCustomizer.java | 12 ++++++ .../PreAggregatedMetricsInstrumenterTest.java | 40 +++++++++++++++++++ 3 files changed, 56 insertions(+) diff --git a/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/init/SecondEntryPoint.java b/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/init/SecondEntryPoint.java index 4998cd50bb6..0f02e05dd8f 100644 --- a/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/init/SecondEntryPoint.java +++ b/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/init/SecondEntryPoint.java @@ -40,6 +40,7 @@ import com.microsoft.applicationinsights.agent.internal.httpclient.LazyHttpClient; import com.microsoft.applicationinsights.agent.internal.legacyheaders.AiLegacyHeaderSpanProcessor; import com.microsoft.applicationinsights.agent.internal.preagg.AiViewRegistry; +import com.microsoft.applicationinsights.agent.internal.preagg.PreAggregatedMetricsCustomizer; import com.microsoft.applicationinsights.agent.internal.processors.ExporterWithLogProcessor; import com.microsoft.applicationinsights.agent.internal.processors.ExporterWithSpanProcessor; import com.microsoft.applicationinsights.agent.internal.processors.LogExporterWithAttributeProcessor; @@ -276,6 +277,9 @@ public void customize(AutoConfigurationCustomizer autoConfiguration) { }) .addMetricExporterCustomizer((metricExporter, configProperties) -> { if (metricExporter instanceof AzureMonitorMetricExporterProvider.MarkerMetricExporter) { + // the metric exporters are created when the SDK is built, which is before any + // instrumenter is built + PreAggregatedMetricsCustomizer.setAzureMonitorMetricExporterEnabled(true); return buildMetricExporter( configuration, telemetryClient, metricFilters, configProperties); } else { diff --git a/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/preagg/PreAggregatedMetricsCustomizer.java b/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/preagg/PreAggregatedMetricsCustomizer.java index cff0f822f9f..8ee3da864fe 100644 --- a/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/preagg/PreAggregatedMetricsCustomizer.java +++ b/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/preagg/PreAggregatedMetricsCustomizer.java @@ -41,10 +41,22 @@ public final class PreAggregatedMetricsCustomizer private static final ContextKey IS_SYNTHETIC = ContextKey.named("applicationinsights.internal.is_synthetic"); + private static volatile boolean azureMonitorMetricExporterEnabled; + private final List metricsStates; private final boolean captureSynthetic; + // the pre-aggregated metrics only reach Application Insights through the Azure Monitor metric + // exporter, e.g. with otel.metrics.exporter=none the duration metrics are not collected at all, + // so the spans must not be marked as pre-aggregated + public static void setAzureMonitorMetricExporterEnabled(boolean enabled) { + azureMonitorMetricExporterEnabled = enabled; + } + public static void customize(InstrumenterCustomizer customizer) { + if (!azureMonitorMetricExporterEnabled) { + return; + } customize(customizer, SemconvStability.emitOldRpcSemconv()); } diff --git a/agent/agent-tooling/src/test/java/com/microsoft/applicationinsights/agent/internal/preagg/PreAggregatedMetricsInstrumenterTest.java b/agent/agent-tooling/src/test/java/com/microsoft/applicationinsights/agent/internal/preagg/PreAggregatedMetricsInstrumenterTest.java index 9b69c9c804b..91ac52f6a78 100644 --- a/agent/agent-tooling/src/test/java/com/microsoft/applicationinsights/agent/internal/preagg/PreAggregatedMetricsInstrumenterTest.java +++ b/agent/agent-tooling/src/test/java/com/microsoft/applicationinsights/agent/internal/preagg/PreAggregatedMetricsInstrumenterTest.java @@ -78,9 +78,49 @@ void setup() { void cleanup() { InternalInstrumenterCustomizerUtil.setInstrumenterCustomizerProviders( originalCustomizerProviders); + PreAggregatedMetricsCustomizer.setAzureMonitorMetricExporterEnabled(false); openTelemetry.close(); } + // e.g. otel.metrics.exporter=none or otlp only, the pre-aggregated metrics don't reach + // Application Insights, so the standard metrics need to be generated from the spans + @Test + void notCustomizedWithoutAzureMonitorMetricExporter() { + useAgentCustomizerEntryPoint(); + + runHttpServer(); + + assertThat(getSingleSpan().getAttributes().get(AiSemanticAttributes.IS_PRE_AGGREGATED)) + .isNull(); + } + + @Test + void customizedWithAzureMonitorMetricExporter() { + useAgentCustomizerEntryPoint(); + PreAggregatedMetricsCustomizer.setAzureMonitorMetricExporterEnabled(true); + + runHttpServer(); + + assertThat(getSingleSpan()).hasAttribute(AiSemanticAttributes.IS_PRE_AGGREGATED, true); + } + + private static void useAgentCustomizerEntryPoint() { + InternalInstrumenterCustomizerUtil.setInstrumenterCustomizerProviders( + singletonList(customizer -> + PreAggregatedMetricsCustomizer.customize(new InstrumenterCustomizerImpl(customizer)))); + } + + private void runHttpServer() { + Instrumenter instrumenter = + Instrumenter.builder(openTelemetry, "test", request -> "GET") + .addAttributesExtractor(HttpServerAttributesExtractor.create(new HttpGetter())) + .addOperationMetrics(HttpServerMetrics.get()) + .buildInstrumenter(SpanKindExtractor.alwaysServer()); + TestRequest request = new TestRequest(null); + Context context = instrumenter.start(Context.root(), request); + instrumenter.end(context, request, 200, null); + } + @Test void httpServer() { Instrumenter instrumenter =