Skip to content

Commit c071d5c

Browse files
Merge pull request #16 from gitcommitankit/phase-6
Phase 6
2 parents 00bceac + 2de702e commit c071d5c

6 files changed

Lines changed: 331 additions & 17 deletions

File tree

‎cmd/main.go‎

Lines changed: 37 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@ import (
2222
"crypto/tls"
2323
"errors"
2424
"flag"
25+
"log/slog"
2526
"net/http"
2627
"os"
2728
"time"
@@ -40,7 +41,6 @@ import (
4041
"sigs.k8s.io/controller-runtime/pkg/cache"
4142
"sigs.k8s.io/controller-runtime/pkg/client"
4243
"sigs.k8s.io/controller-runtime/pkg/healthz"
43-
"sigs.k8s.io/controller-runtime/pkg/log/zap"
4444
"sigs.k8s.io/controller-runtime/pkg/metrics/filters"
4545
metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server"
4646
"sigs.k8s.io/controller-runtime/pkg/webhook"
@@ -51,6 +51,7 @@ import (
5151
agentraxv1alpha1 "github.com/gitcommitankit/agentrax/api/v1alpha1"
5252
"github.com/gitcommitankit/agentrax/internal/controller"
5353
"github.com/gitcommitankit/agentrax/internal/metrics"
54+
"github.com/gitcommitankit/agentrax/internal/observability"
5455
"github.com/gitcommitankit/agentrax/internal/quota"
5556
"github.com/gitcommitankit/agentrax/internal/registry"
5657
"github.com/gitcommitankit/agentrax/internal/rollout"
@@ -124,6 +125,9 @@ func main() {
124125
var gatewayName string
125126
var gatewayNamespace string
126127
var registryAddr string
128+
var otlpEndpoint string
129+
var otlpInsecure bool
130+
var logLevel string
127131
var tlsOpts []func(*tls.Config)
128132
flag.StringVar(&metricsAddr, "metrics-bind-address", "0", "The address the metrics endpoint binds to. "+
129133
"Use :8443 for HTTPS or :8080 for HTTP, or leave as 0 to disable the metrics service.")
@@ -146,13 +150,40 @@ func main() {
146150
"Namespace of the Gateway API Gateway object used for canary traffic splitting.")
147151
flag.StringVar(&registryAddr, "registry-bind-address", ":9090",
148152
"The address the MCP discovery registry HTTP endpoint binds to.")
149-
opts := zap.Options{
150-
Development: false,
151-
}
152-
opts.BindFlags(flag.CommandLine)
153+
flag.StringVar(&otlpEndpoint, "otlp-endpoint", "",
154+
"gRPC endpoint for the OpenTelemetry trace exporter (e.g. localhost:4317). "+
155+
"Leave empty to disable tracing.")
156+
flag.BoolVar(&otlpInsecure, "otlp-insecure", false,
157+
"Use plaintext (insecure) gRPC connection for OpenTelemetry trace exporter. "+
158+
"Only use for local development; remote collectors use TLS by default.")
159+
flag.StringVar(&logLevel, "log-level", "info",
160+
"Minimum log level to emit. One of: debug, info, warn, error.")
153161
flag.Parse()
154162

155-
ctrl.SetLogger(zap.New(zap.UseFlagOptions(&opts)))
163+
// Configure structured JSON logging via log/slog, bridged to controller-runtime
164+
// through the logr interface. This replaces the default Zap logger.
165+
var slogLevel slog.Level
166+
if err := slogLevel.UnmarshalText([]byte(logLevel)); err != nil {
167+
slogLevel = slog.LevelInfo
168+
}
169+
slogLogger := slog.New(slog.NewJSONHandler(os.Stdout, &slog.HandlerOptions{Level: slogLevel}))
170+
ctrl.SetLogger(observability.NewLogr(slogLogger))
171+
172+
// Initialize OTel TracerProvider. The returned Shutdown must be deferred so
173+
// buffered spans are flushed before the process exits.
174+
startCtx := context.Background()
175+
tpShutdown, err := observability.InitTracerProvider(startCtx, otlpEndpoint, otlpInsecure)
176+
if err != nil {
177+
setupLog.Error(err, "unable to initialize OpenTelemetry TracerProvider")
178+
os.Exit(1)
179+
}
180+
defer func() {
181+
shutCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
182+
defer cancel()
183+
if shutErr := tpShutdown(shutCtx); shutErr != nil {
184+
setupLog.Error(shutErr, "error shutting down OTel TracerProvider")
185+
}
186+
}()
156187

157188
// if the enable-http2 flag is false (the default), http/2 should be disabled
158189
// due to its vulnerabilities. More specifically, disabling http/2 will

‎go.mod‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -8,8 +8,11 @@ require (
88
github.com/onsi/gomega v1.33.1
99
github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring v0.75.0
1010
github.com/prometheus/client_golang v1.19.1
11-
github.com/prometheus/client_model v0.6.1
1211
github.com/stretchr/testify v1.10.0
12+
go.opentelemetry.io/otel v1.34.0
13+
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.27.0
14+
go.opentelemetry.io/otel/sdk v1.34.0
15+
go.opentelemetry.io/otel/trace v1.34.0
1316
k8s.io/api v0.31.0
1417
k8s.io/apiextensions-apiserver v0.31.0
1518
k8s.io/apimachinery v0.31.0
@@ -57,6 +60,7 @@ require (
5760
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
5861
github.com/pkg/errors v0.9.1 // indirect
5962
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
63+
github.com/prometheus/client_model v0.6.1 // indirect
6064
github.com/prometheus/common v0.55.0 // indirect
6165
github.com/prometheus/procfs v0.15.1 // indirect
6266
github.com/spf13/cobra v1.8.1 // indirect
@@ -65,12 +69,8 @@ require (
6569
github.com/x448/float16 v0.8.4 // indirect
6670
go.opentelemetry.io/auto/sdk v1.1.0 // indirect
6771
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.53.0 // indirect
68-
go.opentelemetry.io/otel v1.34.0 // indirect
6972
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.28.0 // indirect
70-
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.27.0 // indirect
7173
go.opentelemetry.io/otel/metric v1.34.0 // indirect
72-
go.opentelemetry.io/otel/sdk v1.34.0 // indirect
73-
go.opentelemetry.io/otel/trace v1.34.0 // indirect
7474
go.opentelemetry.io/proto/otlp v1.3.1 // indirect
7575
go.uber.org/multierr v1.11.0 // indirect
7676
go.uber.org/zap v1.26.0 // indirect

‎internal/controller/agentdeployment_controller.go‎

Lines changed: 55 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,9 @@ import (
4040

4141
"github.com/go-logr/logr"
4242
monitoringv1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1"
43+
"go.opentelemetry.io/otel/attribute"
44+
"go.opentelemetry.io/otel/codes"
45+
"go.opentelemetry.io/otel/trace"
4346
apimeta "k8s.io/apimachinery/pkg/api/meta"
4447

4548
agentraxv1alpha1 "github.com/gitcommitankit/agentrax/api/v1alpha1"
@@ -125,7 +128,23 @@ type AgentDeploymentReconciler struct {
125128
// It creates and self-heals a Deployment, Service, and (when Prometheus Operator is present)
126129
// a ServiceMonitor as owned child resources, then updates status conditions.
127130
func (r *AgentDeploymentReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
131+
// Start a root OTel span for the entire reconcile loop. The span is ended via
132+
// defer so all exit paths (early return, error, requeue) are covered.
133+
ctx, span := observability.Tracer.Start(ctx, "reconcile",
134+
trace.WithAttributes(
135+
attribute.String("tenant", req.Namespace),
136+
attribute.String("name", req.Name),
137+
),
138+
)
139+
defer span.End()
140+
141+
// Enrich the controller-runtime logger with trace_id/span_id so every log
142+
// line emitted inside this reconcile cycle is correlated to the active OTel trace.
128143
logger := log.FromContext(ctx)
144+
if sc := span.SpanContext(); sc.IsValid() {
145+
logger = logger.WithValues("trace_id", sc.TraceID().String(), "span_id", sc.SpanID().String())
146+
}
147+
ctx = log.IntoContext(ctx, logger)
129148

130149
// Observe reconcile wall-clock duration on every exit path, including early
131150
// returns, errors, and requeues. The tenant label uses the request namespace
@@ -138,13 +157,19 @@ func (r *AgentDeploymentReconciler) Reconcile(ctx context.Context, req ctrl.Requ
138157
}()
139158

140159
// 1. Fetch the AgentDeployment; return immediately if it has been deleted.
160+
fetchCtx, fetchSpan := observability.Tracer.Start(ctx, "fetch_crd")
141161
ad := &agentraxv1alpha1.AgentDeployment{}
142-
if err := r.Get(ctx, req.NamespacedName, ad); err != nil {
162+
if err := r.Get(fetchCtx, req.NamespacedName, ad); err != nil {
143163
if apierrors.IsNotFound(err) {
164+
fetchSpan.End()
144165
return ctrl.Result{}, nil
145166
}
167+
fetchSpan.RecordError(err)
168+
fetchSpan.SetStatus(codes.Error, "fetch_crd failed")
169+
fetchSpan.End()
146170
return ctrl.Result{}, fmt.Errorf("fetching AgentDeployment: %w", err)
147171
}
172+
fetchSpan.End()
148173

149174
// 2. Handle finalizer lifecycle.
150175
if ad.DeletionTimestamp.IsZero() {
@@ -202,37 +227,61 @@ func (r *AgentDeploymentReconciler) Reconcile(ctx context.Context, req ctrl.Requ
202227
}
203228
}
204229

230+
// 4–7. Reconcile child resources (Deployment, Service, ServiceMonitor, HPA)
231+
// under a single span. Each helper propagates ctx so sub-operations can be
232+
// correlated if they are instrumented in future phases.
233+
childrenCtx, childrenSpan := observability.Tracer.Start(ctx, "reconcile_children")
234+
205235
// 4. Reconcile child Deployment.
206-
if err := r.reconcileDeployment(ctx, ad); err != nil {
236+
if err := r.reconcileDeployment(childrenCtx, ad); err != nil {
237+
childrenSpan.RecordError(err)
238+
childrenSpan.SetStatus(codes.Error, "reconcile_children failed")
239+
childrenSpan.End()
207240
return ctrl.Result{}, fmt.Errorf("reconciling deployment: %w", err)
208241
}
209242

210243
// 5. Reconcile child Service.
211-
if err := r.reconcileService(ctx, ad); err != nil {
244+
if err := r.reconcileService(childrenCtx, ad); err != nil {
245+
childrenSpan.RecordError(err)
246+
childrenSpan.SetStatus(codes.Error, "reconcile_children failed")
247+
childrenSpan.End()
212248
return ctrl.Result{}, fmt.Errorf("reconciling service: %w", err)
213249
}
214250

215251
// 6. Reconcile ServiceMonitor when Prometheus Operator is present.
216-
if err := r.reconcileServiceMonitor(ctx, ad); err != nil {
252+
if err := r.reconcileServiceMonitor(childrenCtx, ad); err != nil {
253+
childrenSpan.RecordError(err)
254+
childrenSpan.SetStatus(codes.Error, "reconcile_children failed")
255+
childrenSpan.End()
217256
return ctrl.Result{}, fmt.Errorf("reconciling servicemonitor: %w", err)
218257
}
219258

220259
// 7. Reconcile the managed HPA (skip during active canary — Phase 4 owns it).
221260
// reconcileHPA also returns the quota evaluation state so updateStatus can
222261
// write the correct QuotaLimited condition onto the freshly re-fetched object.
223-
hpaResult, qs, err := r.reconcileHPA(ctx, ad)
262+
hpaResult, qs, err := r.reconcileHPA(childrenCtx, ad)
224263
if err != nil {
264+
childrenSpan.RecordError(err)
265+
childrenSpan.SetStatus(codes.Error, "reconcile_children failed")
266+
childrenSpan.End()
225267
return ctrl.Result{}, fmt.Errorf("reconciling hpa: %w", err)
226268
}
269+
childrenSpan.End()
227270

228271
// 8. Derive status from the live Deployment and update it — always last.
229272
// We continue into updateStatus even when hpaResult requests a requeue so
230273
// that the QuotaLimited condition is written in the same reconcile cycle.
231274
// Return the shorter of the two requeue intervals.
232-
statusResult, err := r.updateStatus(ctx, ad, logger, qs)
275+
statusCtx, statusSpan := observability.Tracer.Start(ctx, "update_status")
276+
statusResult, err := r.updateStatus(statusCtx, ad, logger, qs)
233277
if err != nil {
278+
statusSpan.RecordError(err)
279+
statusSpan.SetStatus(codes.Error, "update_status failed")
280+
statusSpan.End()
234281
return statusResult, fmt.Errorf("updating status: %w", err)
235282
}
283+
statusSpan.End()
284+
236285
if hpaResult.RequeueAfter > 0 {
237286
if statusResult.RequeueAfter == 0 || hpaResult.RequeueAfter < statusResult.RequeueAfter {
238287
return hpaResult, nil

‎internal/observability/logging.go‎

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,40 @@
1+
package observability
2+
3+
import (
4+
"context"
5+
"io"
6+
"log/slog"
7+
8+
"github.com/go-logr/logr"
9+
"go.opentelemetry.io/otel/trace"
10+
)
11+
12+
// NewJSONLogger returns a [*slog.Logger] that writes structured JSON records to
13+
// w. Pass os.Stdout in production; pass a [*bytes.Buffer] in tests.
14+
func NewJSONLogger(w io.Writer) *slog.Logger {
15+
return slog.New(slog.NewJSONHandler(w, &slog.HandlerOptions{
16+
Level: slog.LevelInfo,
17+
}))
18+
}
19+
20+
// WithTraceContext returns a child logger pre-populated with "trace_id" and
21+
// "span_id" fields extracted from the active OpenTelemetry span in ctx. If ctx
22+
// carries no valid span the original logger is returned unchanged.
23+
func WithTraceContext(ctx context.Context, logger *slog.Logger) *slog.Logger {
24+
span := trace.SpanFromContext(ctx)
25+
sc := span.SpanContext()
26+
if !sc.IsValid() {
27+
return logger
28+
}
29+
return logger.With(
30+
"trace_id", sc.TraceID().String(),
31+
"span_id", sc.SpanID().String(),
32+
)
33+
}
34+
35+
// NewLogr wraps a [*slog.Logger] as a [logr.Logger] so it can be passed to
36+
// controller-runtime's [ctrl.SetLogger]. All controller-runtime log output
37+
// (including reconcile errors) then flows through the slog JSON handler.
38+
func NewLogr(logger *slog.Logger) logr.Logger {
39+
return logr.FromSlogHandler(logger.Handler())
40+
}

‎internal/observability/tracing.go‎

Lines changed: 81 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,81 @@
1+
// Package observability provides OpenTelemetry tracing initialization for the Agentrax operator.
2+
//
3+
// Use [InitTracerProvider] in main() to configure the global OTel tracer. When
4+
// endpoint is empty the function installs a no-op provider so callers need not
5+
// guard on whether tracing is enabled. The returned Shutdown function must be
6+
// deferred in main() to flush buffered spans before process exit.
7+
package observability
8+
9+
import (
10+
"context"
11+
"fmt"
12+
13+
"go.opentelemetry.io/otel"
14+
"go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc"
15+
"go.opentelemetry.io/otel/propagation"
16+
"go.opentelemetry.io/otel/sdk/resource"
17+
sdktrace "go.opentelemetry.io/otel/sdk/trace"
18+
semconv "go.opentelemetry.io/otel/semconv/v1.26.0"
19+
"go.opentelemetry.io/otel/trace"
20+
"go.opentelemetry.io/otel/trace/noop"
21+
)
22+
23+
// TracerName is the instrumentation library name used for all Agentrax OTel spans.
24+
const TracerName = "agentrax.io/controller"
25+
26+
// Tracer is the package-level tracer used by controller code. It is set by
27+
// [InitTracerProvider] and defaults to the global no-op tracer so it is always
28+
// safe to call, even when tracing is disabled.
29+
var Tracer trace.Tracer = noop.NewTracerProvider().Tracer(TracerName)
30+
31+
// InitTracerProvider configures the global OpenTelemetry TracerProvider and
32+
// installs a W3C TraceContext propagator. When endpoint is empty it installs a
33+
// no-op provider (tracing disabled). TLS is used by default for exporter connections;
34+
// set insecure to true only for local development with plaintext collectors.
35+
// The returned Shutdown function flushes and stops the exporter; it must be
36+
// deferred in main().
37+
func InitTracerProvider(ctx context.Context, endpoint string, insecure bool) (func(context.Context) error, error) {
38+
if endpoint == "" {
39+
// No-op: tracing disabled. Global tracer stays as the default no-op.
40+
return func(context.Context) error { return nil }, nil
41+
}
42+
43+
opts := []otlptracegrpc.Option{
44+
otlptracegrpc.WithEndpoint(endpoint),
45+
}
46+
if insecure {
47+
opts = append(opts, otlptracegrpc.WithInsecure())
48+
}
49+
50+
exp, err := otlptracegrpc.New(ctx, opts...)
51+
if err != nil {
52+
return nil, fmt.Errorf("creating OTLP gRPC exporter: %w", err)
53+
}
54+
55+
res, err := resource.Merge(
56+
resource.Default(),
57+
resource.NewSchemaless(
58+
semconv.ServiceName("agentrax-operator"),
59+
),
60+
)
61+
if err != nil {
62+
return nil, fmt.Errorf("creating OTel resource: %w", err)
63+
}
64+
65+
tp := sdktrace.NewTracerProvider(
66+
sdktrace.WithBatcher(exp),
67+
sdktrace.WithResource(res),
68+
)
69+
70+
otel.SetTracerProvider(tp)
71+
// W3C TraceContext + Baggage propagation — required for cross-service correlation.
72+
otel.SetTextMapPropagator(propagation.NewCompositeTextMapPropagator(
73+
propagation.TraceContext{},
74+
propagation.Baggage{},
75+
))
76+
77+
// Update the package-level tracer to use the real provider.
78+
Tracer = tp.Tracer(TracerName)
79+
80+
return tp.Shutdown, nil
81+
}

0 commit comments

Comments
 (0)