feat(gateway): identify gateways in exported traces (#2647)

* feat(gateway): add installation name configuration

Add a first-class operator-assigned gateway name with TOML, CLI, environment,
and Helm configuration surfaces. Local gateways default to openshell, while
Helm defaults to the chart fullname; operators sharing a collector across
namespaces or clusters can set a globally distinct name.

Signed-off-by: Kris Hicks <khicks@nvidia.com>

* feat(gateway): identify gateways in exported traces

Attach the configured gateway installation name and compute driver to the
gateway OpenTelemetry resource so operators can filter traces from multiple
installations that share a collector. Forward the gateway name and OTLP
endpoint to managed external drivers so their distinct service resources carry
the same installation identity.

Keep service.name stable per process type, omit blank resource values, and
leave per-span operation names and request attributes unchanged.

Refs #2507

Signed-off-by: Kris Hicks <khicks@nvidia.com>

---------

Signed-off-by: Kris Hicks <khicks@nvidia.com>

Author: krishicks

architecture/gateway.md

@@ -673,6 +673,15 @@ Driver implementation settings live in the TOML driver tables. See
 `docs/reference/gateway-config.mdx` for worked per-driver examples and RFC
 0003 for the full schema.
 
+Each installation has an operator-assigned gateway name. Configure it with
+`[openshell.gateway].name`, `--name`, or `OPENSHELL_GATEWAY_NAME`.
+The built-in default is `openshell`; the Helm chart defaults it to the chart
+fullname so every replica in one installation reports the same identity.
+Operators must set a globally distinct name when one telemetry collector serves
+installations in multiple Kubernetes namespaces or clusters.
+The name identifies the gateway installation independently of client-side
+aliases, network names, and the sandbox JWT issuer.
+
 `database_url` is env-only and rejected when present in the file
 (`OPENSHELL_DB_URL` / `--db-url`).
 
@@ -721,12 +730,14 @@ between a trace and its log lines. Store and compute-driver spans become
 children of the request span. Reconciliation, provider refresh, and
 driver-watch loops create their own operation spans because they have no
 inbound request to provide a parent. gRPC status is recorded when response
-trailers arrive.
-
-The gateway forwards OTLP configuration and W3C trace context to managed
-external drivers. Built-in drivers use dedicated in-process providers that
-preserve the same RPC trace boundary. Each driver exports to the configured
-collector under its own service name.
+trailers arrive. Gateway spans carry resource attributes for the gateway
+identity and configured compute driver.
+
+The gateway forwards OTLP configuration, its configured gateway name, and W3C
+trace context to managed external drivers. Built-in drivers use dedicated
+in-process providers that preserve the same RPC trace boundary. Each driver
+exports to the configured collector under its own service name and carries the
+gateway name as a resource attribute.
 
 Two invariants shape the failure behavior. Telemetry is diagnostic, so no OTLP
 failure stops the gateway from serving: a malformed endpoint is logged at

crates/openshell-core/src/config.rs

@@ -29,6 +29,9 @@ pub const DEFAULT_SSH_PORT: u16 = 2222;
 /// Default gateway server port.
 pub const DEFAULT_SERVER_PORT: u16 = 17670;
 
+/// Default operator-facing name for a gateway installation.
+pub const DEFAULT_GATEWAY_NAME: &str = "openshell";
+
 /// Default container stop timeout in seconds (SIGTERM → SIGKILL).
 pub const DEFAULT_STOP_TIMEOUT_SECS: u32 = 10;
 
@@ -761,6 +764,9 @@ fn docker_socket_responds(path: &Path) -> bool {
 /// `Deserialize` impls for that purpose).
 #[derive(Debug, Clone)]
 pub struct Config {
+    /// Operator-assigned name for this gateway installation.
+    pub name: String,
+
     /// Address to bind the server to.
     pub bind_address: SocketAddr,
 
@@ -1168,6 +1174,7 @@ impl Config {
     /// Create a new config with optional TLS.
     pub fn new(tls: Option<TlsConfig>) -> Self {
         Self {
+            name: DEFAULT_GATEWAY_NAME.to_string(),
             bind_address: default_bind_address(),
             health_bind_address: None,
             metrics_bind_address: None,
@@ -1195,6 +1202,13 @@ impl Config {
         }
     }
 
+    /// Create a new configuration with the gateway installation name.
+    #[must_use]
+    pub fn with_name(mut self, name: impl Into<String>) -> Self {
+        self.name = name.into();
+        self
+    }
+
     /// Create a new configuration with the given bind address.
     #[must_use]
     pub const fn with_bind_address(mut self, addr: SocketAddr) -> Self {
@@ -1573,6 +1587,15 @@ mod tests {
         assert_eq!(cfg.ttl_secs, 0);
     }
 
+    #[test]
+    fn name_defaults_and_can_be_overridden() {
+        assert_eq!(Config::new(None).name, "openshell");
+        assert_eq!(
+            Config::new(None).with_name("production-us-west").name,
+            "production-us-west"
+        );
+    }
+
     #[test]
     fn gateway_interceptor_failure_policy_rejects_ignore() {
         let err =

crates/openshell-driver-docker/src/main.rs

@@ -38,13 +38,18 @@ struct Args {
 
     #[arg(long, env = "OPENSHELL_OTLP_ENDPOINT")]
     otlp_endpoint: Option<String>,
+
+    #[arg(long, env = "OPENSHELL_GATEWAY_NAME")]
+    gateway_name: Option<String>,
 }
 
 #[tokio::main]
 async fn main() -> Result<()> {
     let args = Args::parse();
-    let (tracer_provider, setup_error) =
-        openshell_driver_docker::otel_tracing::provider_for(args.otlp_endpoint.as_deref());
+    let (tracer_provider, setup_error) = openshell_driver_docker::otel_tracing::provider_for(
+        args.otlp_endpoint.as_deref(),
+        args.gateway_name.as_deref(),
+    );
     tracing_subscriber::registry()
         .with(EnvFilter::try_from_default_env().unwrap_or_else(|_| EnvFilter::new(&args.log_level)))
         .with(tracing_subscriber::fmt::layer())

crates/openshell-driver-docker/src/otel_tracing.rs

@@ -90,13 +90,28 @@ pub(crate) fn compute_driver_rpc_operation(path: &str) -> (&'static str, &'stati
     }
 }
 
+/// Build a tracer provider for the configured OTLP/gRPC endpoint and gateway.
 #[must_use]
-pub fn provider_for(endpoint: Option<&str>) -> (Option<SdkTracerProvider>, Option<SetupError>) {
-    openshell_otel::provider_for(endpoint.map(|endpoint| OtlpTraceConfig {
-        endpoint,
-        service_name: ServiceName::Fixed(SERVICE_NAME),
-        service_version: Some(openshell_core::VERSION),
-        resource_attributes: Vec::new(),
+pub fn provider_for(
+    endpoint: Option<&str>,
+    gateway_name: Option<&str>,
+) -> (Option<SdkTracerProvider>, Option<SetupError>) {
+    openshell_otel::provider_for(endpoint.map(|endpoint| {
+        OtlpTraceConfig {
+            endpoint,
+            service_name: ServiceName::Fixed(SERVICE_NAME),
+            service_version: Some(openshell_core::VERSION),
+            resource_attributes: gateway_name
+                .map(str::trim)
+                .filter(|name| !name.is_empty())
+                .map(|name| {
+                    vec![opentelemetry::KeyValue::new(
+                        "openshell.gateway.name",
+                        name.to_string(),
+                    )]
+                })
+                .unwrap_or_default(),
+        }
     }))
 }
 
@@ -151,11 +166,11 @@ mod tests {
     }
 
     #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
-    async fn tracing_docker_driver_spans_reach_otlp_collector_with_distinct_service_name() {
+    async fn tracing_docker_driver_spans_reach_otlp_collector_with_resource_identity() {
         let _tracing_lock = super::test_lock().await;
         let collector = OtlpTestServer::start().await;
 
-        let (provider, error) = super::provider_for(Some(collector.endpoint()));
+        let (provider, error) = super::provider_for(Some(collector.endpoint()), Some("docker-dev"));
         assert!(error.is_none());
         let provider = provider.expect("provider");
         let subscriber = tracing_subscriber::registry().with(super::layer(&provider));
@@ -175,6 +190,7 @@ mod tests {
                 .iter()
                 .any(|span| span.name == "docker.schedule_sandbox")
         );
+        assert_eq!(received.gateway_names, ["docker-dev"]);
         assert!(
             received
                 .service_names

crates/openshell-driver-kubernetes/Cargo.toml

@@ -19,6 +19,7 @@ openshell-core = { path = "../openshell-core", default-features = false }
 openshell-otel = { path = "../openshell-otel" }
 openshell-policy = { path = "../openshell-policy" }
 
+opentelemetry = { workspace = true }
 tokio = { workspace = true }
 tonic = { workspace = true, features = ["transport"] }
 prost = { workspace = true }
@@ -39,7 +40,6 @@ notify = "8"
 
 [dev-dependencies]
 openshell-otel-test-support = { path = "../openshell-otel-test-support" }
-opentelemetry = { workspace = true }
 opentelemetry_sdk = { workspace = true, features = ["testing"] }
 bytes = { workspace = true }
 http = { workspace = true }

crates/openshell-driver-kubernetes/README.md

@@ -23,7 +23,9 @@ When the gateway configures `[openshell.gateway.otlp]`, Kubernetes
 compute-driver spans export to the same OTLP/gRPC collector with the service
 name `openshell-driver-kubernetes`. The driver preserves the gateway trace
 context and uses the same compute-driver RPC span names in its in-process and
-standalone forms.
+standalone forms. Standalone deployments set `--gateway-name` or
+`OPENSHELL_GATEWAY_NAME` so exported spans carry the same
+`openshell.gateway.name` resource attribute as gateway spans.
 
 When it creates an Agent Sandbox resource, the driver serializes the active W3C
 trace context into the controller-reserved `opentelemetry.io/trace-context`

crates/openshell-driver-kubernetes/src/main.rs

@@ -42,6 +42,9 @@ struct Args {
     #[arg(long, env = "OPENSHELL_OTLP_ENDPOINT")]
     otlp_endpoint: Option<String>,
 
+    #[arg(long, env = "OPENSHELL_GATEWAY_NAME")]
+    gateway_name: Option<String>,
+
     #[arg(long, env = "OPENSHELL_WORKSPACE_MODE", default_value = "shared")]
     workspace_mode: WorkspaceMode,
 
@@ -215,8 +218,10 @@ async fn shutdown_signal() {
 #[tokio::main]
 async fn main() -> Result<()> {
     let args = Args::parse();
-    let (tracer_provider, setup_error) =
-        openshell_driver_kubernetes::otel_tracing::provider_for(args.otlp_endpoint.as_deref());
+    let (tracer_provider, setup_error) = openshell_driver_kubernetes::otel_tracing::provider_for(
+        args.otlp_endpoint.as_deref(),
+        args.gateway_name.as_deref(),
+    );
     tracing_subscriber::registry()
         .with(EnvFilter::try_from_default_env().unwrap_or_else(|_| EnvFilter::new(&args.log_level)))
         .with(tracing_subscriber::fmt::layer())
@@ -349,17 +354,20 @@ mod tests {
     use super::*;
 
     #[test]
-    fn accepts_gateway_otlp_endpoint() {
+    fn accepts_gateway_otlp_configuration() {
         let args = Args::try_parse_from([
             "openshell-driver-kubernetes",
             "--otlp-endpoint",
             "http://collector.example:4317",
+            "--gateway-name",
+            "kubernetes-dev",
         ])
         .expect("OTLP endpoint should parse");
 
         assert_eq!(
             args.otlp_endpoint.as_deref(),
             Some("http://collector.example:4317")
         );
+        assert_eq!(args.gateway_name.as_deref(), Some("kubernetes-dev"));
     }
 }

crates/openshell-driver-kubernetes/src/otel_tracing.rs

@@ -12,13 +12,28 @@ const SERVICE_NAME: &str = "openshell-driver-kubernetes";
 const INSTRUMENTATION_SCOPE: &str = "openshell-driver-kubernetes";
 pub const IN_PROCESS_TARGET_PREFIX: &str = "openshell_driver_kubernetes";
 
+/// Build a tracer provider for the configured OTLP/gRPC endpoint and gateway.
 #[must_use]
-pub fn provider_for(endpoint: Option<&str>) -> (Option<SdkTracerProvider>, Option<SetupError>) {
-    openshell_otel::provider_for(endpoint.map(|endpoint| OtlpTraceConfig {
-        endpoint,
-        service_name: ServiceName::Fixed(SERVICE_NAME),
-        service_version: Some(openshell_core::VERSION),
-        resource_attributes: Vec::new(),
+pub fn provider_for(
+    endpoint: Option<&str>,
+    gateway_name: Option<&str>,
+) -> (Option<SdkTracerProvider>, Option<SetupError>) {
+    openshell_otel::provider_for(endpoint.map(|endpoint| {
+        OtlpTraceConfig {
+            endpoint,
+            service_name: ServiceName::Fixed(SERVICE_NAME),
+            service_version: Some(openshell_core::VERSION),
+            resource_attributes: gateway_name
+                .map(str::trim)
+                .filter(|name| !name.is_empty())
+                .map(|name| {
+                    vec![opentelemetry::KeyValue::new(
+                        "openshell.gateway.name",
+                        name.to_string(),
+                    )]
+                })
+                .unwrap_or_default(),
+        }
     }))
 }
 
@@ -75,10 +90,11 @@ mod tests {
     }
 
     #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
-    async fn tracing_kubernetes_driver_spans_reach_otlp_collector_with_distinct_service_name() {
+    async fn tracing_kubernetes_driver_spans_reach_otlp_collector_with_resource_identity() {
         let _tracing_lock = super::test_lock().await;
         let collector = OtlpTestServer::start().await;
-        let (provider, error) = super::provider_for(Some(collector.endpoint()));
+        let (provider, error) =
+            super::provider_for(Some(collector.endpoint()), Some("kubernetes-dev"));
         assert!(error.is_none());
         let provider = provider.expect("provider");
         let subscriber = tracing_subscriber::registry().with(super::layer(&provider));
@@ -98,6 +114,7 @@ mod tests {
                 .iter()
                 .any(|span| span.name == "kubernetes.create_sandbox")
         );
+        assert_eq!(received.gateway_names, ["kubernetes-dev"]);
         assert!(
             received
                 .service_names

crates/openshell-driver-podman/src/main.rs

@@ -40,6 +40,9 @@ struct Args {
     #[arg(long, env = "OPENSHELL_OTLP_ENDPOINT")]
     otlp_endpoint: Option<String>,
 
+    #[arg(long, env = "OPENSHELL_GATEWAY_NAME")]
+    gateway_name: Option<String>,
+
     /// Path to the Podman API Unix socket.
     #[arg(long, env = "OPENSHELL_PODMAN_SOCKET")]
     podman_socket: Option<PathBuf>,
@@ -174,8 +177,10 @@ struct Args {
 #[tokio::main]
 async fn main() -> Result<()> {
     let args = Args::parse();
-    let (tracer_provider, setup_error) =
-        openshell_driver_podman::otel_tracing::provider_for(args.otlp_endpoint.as_deref());
+    let (tracer_provider, setup_error) = openshell_driver_podman::otel_tracing::provider_for(
+        args.otlp_endpoint.as_deref(),
+        args.gateway_name.as_deref(),
+    );
     tracing_subscriber::registry()
         .with(EnvFilter::try_from_default_env().unwrap_or_else(|_| EnvFilter::new(&args.log_level)))
         .with(tracing_subscriber::fmt::layer())
@@ -306,17 +311,20 @@ mod tests {
     }
 
     #[test]
-    fn accepts_gateway_otlp_endpoint() {
+    fn accepts_gateway_otlp_configuration() {
         let args = Args::try_parse_from([
             "openshell-driver-podman",
             "--otlp-endpoint",
             "http://collector.internal:4317",
+            "--gateway-name",
+            "production-us-west",
         ])
-        .expect("OTLP endpoint should be accepted");
+        .expect("OTLP configuration should be accepted");
 
         assert_eq!(
             args.otlp_endpoint.as_deref(),
             Some("http://collector.internal:4317")
         );
+        assert_eq!(args.gateway_name.as_deref(), Some("production-us-west"));
     }
 }