Add Tempo tracing backend and fix dashboard queries

Adds a Tempo container (compose + Helm) alongside the OTel collector for TraceQL, service-graph, and span-metrics dashboards, wiring the metrics generator to remote-write into Prometheus. Updates Grafana dashboard datasource UIDs, span-metric attribute names (OTel semconv 1.x: http.response.status_code, messaging.destination.name, etc.), and Prometheus scrape targets/ports for workers.

Rewrites many Grafana panel queries that were previously disabled or stubbed (ADR-002 placeholders) to pull from their now-available source tables (climate_snapshots, motor_snapshots, security_events, drive_telemetry, tesla_battery_passport_ledger, etc.) instead of signal_log placeholders. Adds vehicle/drive picker dropdowns backed by SQL variables in several dashboards, adds an automation-worker service to docker-compose.dev.yml, exposes /metrics on all workers via promhttp, and improves HTTP route attribution in the tracing middleware by tagging spans with chi's matched route after routing completes.

Also fixes the validate_grafana_signal_log.py script to use a relative dashboard path and substitute more template variables.

Author: atulmgupta

.env.example

@@ -36,6 +36,12 @@ GRAFANA_PORT=3001
 GRAFANA_USER=admin
 GRAFANA_PASSWORD=changeme
 
+# Distributed tracing (optional — pair with: docker compose --profile tracing up)
+# Required for the Tempo, service-graph, span-metrics, and SLO trace dashboards.
+OTEL_ENABLED=false
+OTEL_ENDPOINT=otel-collector:4317
+OTEL_INSECURE=true
+
 # MQTT
 MQTT_PORT=1883
 

cmd/automation-worker/main.go

@@ -11,6 +11,7 @@ import (
 	"time"
 
 	pahomqtt "github.com/eclipse/paho.mqtt.golang"
+	"github.com/prometheus/client_golang/prometheus/promhttp"
 	"github.com/rs/zerolog"
 	"github.com/rs/zerolog/log"
 
@@ -295,6 +296,7 @@ func main() {
 	port := healthPort()
 	healthMux := http.NewServeMux()
 	healthMux.HandleFunc("/healthz", healthHandler(db))
+	healthMux.Handle("/metrics", promhttp.Handler())
 	go func() {
 		log.Info().Str("port", port).Msg("health endpoint listening")
 		if err := http.ListenAndServe(":"+port, healthMux); err != nil && err != http.ErrServerClosed {

cmd/export-worker/main.go

@@ -12,6 +12,7 @@ import (
 
 	backupmodel "github.com/ev-dev-labs/teslasync/internal/models/backup"
 
+	"github.com/prometheus/client_golang/prometheus/promhttp"
 	"github.com/rs/zerolog"
 	"github.com/rs/zerolog/log"
 	"go.opentelemetry.io/otel"
@@ -211,6 +212,7 @@ func main() {
 	healthPort := resolveHealthPort()
 	healthMux := http.NewServeMux()
 	healthMux.HandleFunc("/healthz", newHealthHandler(db))
+	healthMux.Handle("/metrics", promhttp.Handler())
 	go func() {
 		log.Info().Str("port", healthPort).Msg("health endpoint listening")
 		if err := http.ListenAndServe(":"+healthPort, healthMux); err != nil && err != http.ErrServerClosed {

cmd/notification-worker/main.go

@@ -15,6 +15,7 @@ import (
 
 	alertmodel "github.com/ev-dev-labs/teslasync/internal/models/alert"
 
+	"github.com/prometheus/client_golang/prometheus/promhttp"
 	"github.com/rs/zerolog"
 	"github.com/rs/zerolog/log"
 	"go.opentelemetry.io/otel"
@@ -384,6 +385,7 @@ func main() {
 	}
 	healthMux := http.NewServeMux()
 	healthMux.HandleFunc("/healthz", healthzHandler(db))
+	healthMux.Handle("/metrics", promhttp.Handler())
 	go func() {
 		log.Info().Str("port", healthPort).Msg("health endpoint listening")
 		if err := http.ListenAndServe(":"+healthPort, healthMux); err != nil && err != http.ErrServerClosed {

cmd/slogen/generate_dashboards.go

@@ -145,7 +145,7 @@ func runGenerateDashboards(args []string) error {
 	return nil
 }
 
-var prometheusDS = datasource{Type: "prometheus", UID: "prometheus"}
+var prometheusDS = datasource{Type: "prometheus", UID: "DS_TESLASYNC_PROMETHEUS"}
 
 func renderSLODashboard(s SLO) (string, error) {
 	var min0 float64 = 0

docker-compose.dev.yml

@@ -1,8 +1,8 @@
 # docker-compose.dev.yml — Development stack with ALL components enabled.
 # Usage:  docker compose -f docker-compose.dev.yml up --build
 #
-# Includes: API, Web (Nginx), Notification Worker, Export Worker,
-#           PostgreSQL, Redis, Mosquitto, Grafana, Jaeger (tracing).
+# Includes: API, Web (Nginx), Notification Worker, Export Worker, Automation Worker,
+#           PostgreSQL, Redis, Mosquitto, Grafana, Tempo + Jaeger (tracing).
 # All tracing and telemetry features are enabled by default.
 #
 # ⚠️  TimescaleDB-HA migration (ADR-007):
@@ -51,7 +51,7 @@ services:
       - FLEET_TELEMETRY_FALLBACK_POLL_INTERVAL=5m
       # Tracing enabled by default in dev
       - OTEL_ENABLED=true
-      - OTEL_ENDPOINT=jaeger:4317
+      - OTEL_ENDPOINT=otel-collector:4317
       - OTEL_SERVICE_NAME=teslasync-api
       - OTEL_INSECURE=true
       - GOOGLE_MAPS_API_KEY=${GOOGLE_MAPS_API_KEY:-}
@@ -63,7 +63,7 @@ services:
         condition: service_started
       redis:
         condition: service_healthy
-      jaeger:
+      otel-collector:
         condition: service_started
     healthcheck:
       test: ["CMD", "/usr/local/bin/teslasync", "healthcheck"]
@@ -104,14 +104,16 @@ services:
       - LOG_LEVEL=error
       - HEALTH_PORT=8081
       - OTEL_ENABLED=true
-      - OTEL_ENDPOINT=jaeger:4317
+      - OTEL_ENDPOINT=otel-collector:4317
       - OTEL_SERVICE_NAME=teslasync-notification-worker
       - OTEL_INSECURE=true
     depends_on:
       postgres:
         condition: service_healthy
       mosquitto:
         condition: service_started
+      otel-collector:
+        condition: service_started
     healthcheck:
       test: ["CMD", "/usr/local/bin/notification-worker", "healthcheck"]
       interval: 15s
@@ -139,21 +141,63 @@ services:
       - LOG_LEVEL=error
       - HEALTH_PORT=8082
       - OTEL_ENABLED=true
-      - OTEL_ENDPOINT=jaeger:4317
+      - OTEL_ENDPOINT=otel-collector:4317
       - OTEL_SERVICE_NAME=teslasync-export-worker
       - OTEL_INSECURE=true
     depends_on:
       postgres:
         condition: service_healthy
       mosquitto:
         condition: service_started
+      otel-collector:
+        condition: service_started
     healthcheck:
       test: ["CMD", "/usr/local/bin/export-worker", "healthcheck"]
       interval: 15s
       timeout: 5s
       retries: 3
       start_period: 10s
 
+  automation-worker:
+    build:
+      context: .
+      dockerfile: Dockerfile.automation
+      args:
+        VERSION: dev
+    container_name: teslasync-automation-worker-dev
+    restart: unless-stopped
+    environment:
+      - DATABASE_HOST=postgres
+      - DATABASE_PORT=5432
+      - DATABASE_USER=teslasync
+      - DATABASE_PASSWORD=teslasync
+      - DATABASE_NAME=teslasync
+      - DATABASE_SSL_MODE=disable
+      - MQTT_HOST=mosquitto
+      - MQTT_PORT=1883
+      - TESLA_CLIENT_ID=${TESLA_CLIENT_ID:-dev-client-id}
+      - TESLA_CLIENT_SECRET=${TESLA_CLIENT_SECRET:-dev-client-secret}
+      - TESLA_API_BASE_URL=${TESLA_API_BASE_URL:-https://fleet-api.prd.na.vn.cloud.tesla.com}
+      - LOG_LEVEL=error
+      - HEALTH_PORT=8083
+      - OTEL_ENABLED=true
+      - OTEL_ENDPOINT=otel-collector:4317
+      - OTEL_SERVICE_NAME=teslasync-automation-worker
+      - OTEL_INSECURE=true
+    depends_on:
+      postgres:
+        condition: service_healthy
+      mosquitto:
+        condition: service_started
+      otel-collector:
+        condition: service_started
+    healthcheck:
+      test: ["CMD", "/usr/local/bin/automation-worker", "healthcheck"]
+      interval: 15s
+      timeout: 5s
+      retries: 3
+      start_period: 10s
+
   # ── Infrastructure ────────────────────────────────────────
 
   postgres:
@@ -229,13 +273,35 @@ services:
       - POSTGRES_USER=teslasync
       - POSTGRES_PASSWORD=teslasync
       - POSTGRES_DB=teslasync
+      - PROMETHEUS_URL=http://prometheus:9090
     volumes:
       - dev_grafana_data:/var/lib/grafana
-      - ./grafana/provisioning:/etc/grafana/provisioning
+      - ./grafana/custom.ini:/etc/grafana/grafana.ini:ro
+      - ./grafana/provisioning/datasources:/etc/grafana/provisioning/datasources:ro
+      - ./grafana/provisioning/dashboards:/etc/grafana/provisioning/dashboards:ro
       - ./grafana/dashboards:/var/lib/grafana/dashboards
     depends_on:
       postgres:
         condition: service_healthy
+      prometheus:
+        condition: service_started
+
+  prometheus:
+    image: prom/prometheus:v2.51.0
+    container_name: teslasync-prometheus-dev
+    restart: unless-stopped
+    ports:
+      - "9099:9090"
+    volumes:
+      - ./prometheus/prometheus.yml:/etc/prometheus/prometheus.yml:ro
+      - ./helm/teslasync/files/prometheus/recording-rules.yaml:/etc/prometheus/rules/recording-rules.yaml:ro
+      - ./helm/teslasync/files/prometheus/alerting-rules.yaml:/etc/prometheus/rules/alerting-rules.yaml:ro
+      - dev_prometheus_data:/prometheus
+    command:
+      - '--config.file=/etc/prometheus/prometheus.yml'
+      - '--storage.tsdb.retention.time=15d'
+      - '--web.enable-remote-write-receiver'
+      - '--enable-feature=exemplar-storage'
 
   # ── Observability ─────────────────────────────────────────
 
@@ -245,13 +311,46 @@ services:
     restart: unless-stopped
     ports:
       - "16686:16686"   # Jaeger UI
-      - "4317:4317"     # OTLP gRPC
-      - "4318:4318"     # OTLP HTTP
     environment:
       - COLLECTOR_OTLP_ENABLED=true
 
+  tempo:
+    image: grafana/tempo:2.7.0
+    container_name: teslasync-tempo-dev
+    restart: unless-stopped
+    command:
+      - "-config.file=/etc/tempo/config.yaml"
+    ports:
+      - "3200:3200"
+    volumes:
+      - ./observability/tempo/config.yaml:/etc/tempo/config.yaml:ro
+      - dev_tempo_data:/var/tempo
+    depends_on:
+      prometheus:
+        condition: service_started
+
+  otel-collector:
+    image: otel/opentelemetry-collector-contrib:0.103.1
+    container_name: teslasync-otel-collector-dev
+    restart: unless-stopped
+    command:
+      - "--config=/etc/otelcol/config.yaml"
+    volumes:
+      - ./observability/otel-collector/config.yaml:/etc/otelcol/config.yaml:ro
+    ports:
+      - "4317:4317"
+      - "4318:4318"
+      - "9464:9464"
+    depends_on:
+      jaeger:
+        condition: service_started
+      tempo:
+        condition: service_started
+
 volumes:
   dev_postgres_data:
   dev_redis_data:
   dev_mosquitto_data:
   dev_grafana_data:
+  dev_prometheus_data:
+  dev_tempo_data:

docker-compose.yml

@@ -295,7 +295,9 @@ services:
       - PROMETHEUS_URL=http://prometheus:9090
     volumes:
       - grafana_data:/var/lib/grafana
-      - ./grafana/provisioning:/etc/grafana/provisioning
+      - ./grafana/custom.ini:/etc/grafana/grafana.ini:ro
+      - ./grafana/provisioning/datasources:/etc/grafana/provisioning/datasources:ro
+      - ./grafana/provisioning/dashboards:/etc/grafana/provisioning/dashboards:ro
       - ./grafana/dashboards:/var/lib/grafana/dashboards
     depends_on:
       postgres:
@@ -329,6 +331,7 @@ services:
       - '--config.file=/etc/prometheus/prometheus.yml'
       - '--storage.tsdb.retention.time=15d'
       - '--web.enable-lifecycle'
+      - '--web.enable-remote-write-receiver'
       # exemplar-storage retains the trace_id/span_id labels attached to
       # histogram observations by internal/api/middleware.go +
       # internal/metrics/exemplar.go so Grafana can render the "View trace"
@@ -503,6 +506,33 @@ services:
         limits:
           memory: 256M
 
+  # Tempo (optional — docker compose --profile tracing up)
+  # Grafana's TraceQL, service-map, and span-metrics dashboards use this
+  # backend. Its metrics generator remote-writes RED and service-graph series
+  # to the Prometheus service above.
+  tempo:
+    image: grafana/tempo:2.7.0
+    container_name: teslasync-tempo
+    restart: unless-stopped
+    profiles:
+      - tracing
+    command:
+      - "-config.file=/etc/tempo/config.yaml"
+    ports:
+      - "${TEMPO_PORT:-3200}:3200"
+    volumes:
+      - ./observability/tempo/config.yaml:/etc/tempo/config.yaml:ro
+      - tempo_data:/var/tempo
+    depends_on:
+      prometheus:
+        condition: service_started
+    deploy:
+      resources:
+        limits:
+          memory: 512M
+        reservations:
+          memory: 128M
+
   # OpenTelemetry Collector (optional — docker compose --profile tracing up)
   # Dev variant of helm/teslasync/files/otel-collector/config.yaml — 100%
   # sampling (no tail policy), CORS-enabled OTLP HTTP receiver for browser
@@ -532,6 +562,8 @@ services:
     depends_on:
       jaeger:
         condition: service_started
+      tempo:
+        condition: service_started
     deploy:
       resources:
         limits:
@@ -697,5 +729,6 @@ volumes:
   mosquitto_data:
   redis_data:
   prometheus_data:
+  tempo_data:
   pyroscope_data:
-  vehicle_photos:
+  vehicle_photos:

docs/deployment/docker.md

@@ -27,7 +27,7 @@ The rest of this page is the longer story — what each service does, when to en
 
 ## What's in `docker-compose.yml`
 
-The Compose file defines **13 services** across five concerns. By default, 10 come up; 3 are opt-in via profiles plus Ollama in the `ai` profile.
+The Compose file defines the core application and data services plus optional profile-gated sidecars.
 
 ### Always-on (10 services)
 
@@ -44,7 +44,7 @@ The Compose file defines **13 services** across five concerns. By default, 10 co
 | `grafana`              | Provisioned dashboards (admin/admin default)     | 3001         |
 | `prometheus`           | Metrics scrape + retention                       | 9099         |
 
-### Profile-gated (3 services)
+### Profile-gated services
 
 Profiles let you opt into heavier sidecars only when you need them. Bring a profile up alongside the default stack:
 
@@ -54,13 +54,24 @@ docker compose --profile <name> up -d
 
 | Profile     | Service                  | What it adds                                                     |
 | ----------- | ------------------------ | ---------------------------------------------------------------- |
-| `tracing`   | `jaeger`                 | OpenTelemetry trace collector + UI on `:16686`                   |
+| `tracing`   | `otel-collector`, `tempo`, `jaeger` | OTLP ingest, TraceQL storage/span metrics, and Jaeger UI on `:16686` |
 | `telemetry` | `fleet-telemetry`        | Tesla Fleet Telemetry server on `:4443`; HTTPS endpoint required |
 | `commands`  | `vehicle-command-proxy`  | Signs commands for vehicles that require it; on `:4443`          |
 | `ai`        | `ollama`                 | Local LLM inference for Helix AI on `:11434`                     |
 
 You can stack profiles: `docker compose --profile telemetry --profile commands --profile ai up -d`.
 
+The tracing dashboards also require the application processes to emit spans. Set
+`OTEL_ENABLED=true` in `.env` before starting the profile, then run:
+
+```bash
+docker compose --profile tracing up -d --build
+```
+
+Changing `OTEL_ENABLED` causes Compose to recreate the API and workers with
+tracing enabled. Starting the profile without that setting only starts the
+backends; it cannot create application traces or span-derived metrics.
+
 ## When to use which profile
 
 | Situation                                                          | Profile to add |

grafana/custom.ini

@@ -3,7 +3,7 @@ enabled = false
 
 [auth.anonymous]
 enabled = true
-org_name = TeslaSync
+org_name = Main Org.
 org_role = Viewer
 
 [dashboards]