fix(supervisor-network): keep FORWARD denials in the connect trace

At debug level, FORWARD credential-binding and request middleware
denials opened root deny spans, because the saved connect span exists
there but is not entered. Deny spans also charged the per-destination
limiter even when they were disabled, and a fail-closed WebSocket
middleware finding did not record the denial its HTTP counterparts do.

Those denials now pass or run under the saved connect span, the limiter
is consulted only when INFO spans are enabled, and the fail-closed
WebSocket finding sets action Denied.

Signed-off-by: Kris Hicks <khicks@nvidia.com>

Author: krishicks

crates/openshell-supervisor-network/src/l7/middleware.rs

@@ -388,8 +388,13 @@ fn emit_websocket_invocations(
             .build();
         egress_emit!(event);
         if invocation.failed {
-            let event = DetectionFindingBuilder::new(openshell_ocsf::ctx::ctx())
-                .severity(if invocation.outcome == Outcome::FailClosed {
+            let fail_closed = invocation.outcome == Outcome::FailClosed;
+            let mut finding = DetectionFindingBuilder::new(openshell_ocsf::ctx::ctx());
+            if fail_closed {
+                finding = finding.action(ActionId::Denied);
+            }
+            let event = finding
+                .severity(if fail_closed {
                     SeverityId::High
                 } else {
                     SeverityId::Medium
@@ -1052,6 +1057,40 @@ mod trace_tests {
         }
     }
 
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+    async fn websocket_fail_closed_findings_record_the_denial() {
+        use openshell_supervisor_middleware::{
+            WebSocketInvocation, WebSocketInvocationOutcome as Outcome,
+        };
+        for (outcome, action_id) in [(Outcome::FailClosed, Some(2)), (Outcome::FailOpen, None)] {
+            let (events, _) = exported("warn", || {
+                emit_websocket_invocations(
+                    &L7EvalContext::default(),
+                    &[WebSocketInvocation {
+                        config_name: "guard".to_string(),
+                        implementation: "test/guard".to_string(),
+                        outcome,
+                        sequence: None,
+                        original_size: 0,
+                        replacement_size: None,
+                        transformed: false,
+                        failed: true,
+                        stage_disabled: false,
+                        reason_code: None,
+                    }],
+                );
+            })
+            .await;
+            let finding = events
+                .iter()
+                .find(|event| {
+                    event["finding_info"]["uid"] == "openshell.middleware.websocket_failure"
+                })
+                .unwrap();
+            assert_eq!(finding["action_id"].as_u64(), action_id, "{outcome:?}");
+        }
+    }
+
     #[test]
     fn refusal_findings_record_the_denial() {
         for reason in [

crates/openshell-supervisor-network/src/proxy.rs

@@ -247,15 +247,21 @@ fn build_credential_endpoint_mismatch_event(
         .build()
 }
 
-fn emit_credential_endpoint_mismatch(method: &str, host: &str, port: u16, policy_name: &str) {
+fn emit_credential_endpoint_mismatch(
+    method: &str,
+    host: &str,
+    port: u16,
+    policy_name: &str,
+    connect_span: &tracing::Span,
+) {
     let event = build_credential_endpoint_mismatch_event(method, host, port, policy_name);
     let finding = crate::l7::build_credential_endpoint_mismatch_finding(
         policy_name,
         host,
         None,
         "Provider credential endpoint binding mismatch; request denied",
     );
-    crate::telemetry::emit_decision([event, finding], None);
+    crate::telemetry::emit_decision([event, finding], Some(connect_span));
 }
 
 /// Hostnames injected by compute drivers as `/etc/hosts` aliases for the host
@@ -6490,7 +6496,12 @@ async fn handle_forward_proxy(
             exchange: &middleware_exchange,
             l7_reevaluation,
         };
-        forward_request_bytes = match pipeline.apply(request, client).await? {
+        // Middleware denials join the connect trace, like other FORWARD denials.
+        forward_request_bytes = match pipeline
+            .apply(request, client)
+            .instrument(connect_span.clone())
+            .await?
+        {
             crate::l7::middleware::MiddlewareApplyResult::Allowed(request) => request.raw_header,
             crate::l7::middleware::MiddlewareApplyResult::Denied { denial, .. } => {
                 emit_activity_simple(activity_tx, true, "middleware");
@@ -6671,13 +6682,21 @@ async fn handle_forward_proxy(
                     .await;
             }
             if e.is_placement_mismatch() {
-                crate::l7::relay::reject_credential_resolution(client, &l7_ctx, method, &e).await?;
+                crate::l7::relay::reject_credential_resolution(client, &l7_ctx, method, &e)
+                    .instrument(connect_span.clone())
+                    .await?;
                 client.shutdown().await.into_diagnostic()?;
             } else if e.is_endpoint_mismatch() {
                 if let Some(observer) = endpoint_observer.as_ref() {
                     observer.observe_credential_failure(true);
                 }
-                emit_credential_endpoint_mismatch(method, &host_lc, port, policy_str);
+                emit_credential_endpoint_mismatch(
+                    method,
+                    &host_lc,
+                    port,
+                    policy_str,
+                    &connect_span,
+                );
                 respond(
                     client,
                     &build_json_error_response(

crates/openshell-supervisor-network/src/proxy/trace_tests.rs

@@ -10,7 +10,13 @@ use opentelemetry_proto::tonic::common::v1::any_value::Value;
 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
 async fn paired_credential_denial_shares_one_deny_span() {
     let (events, received) = exported("warn", || {
-        emit_credential_endpoint_mismatch("POST", "blocked.example", 443, "credential-policy");
+        emit_credential_endpoint_mismatch(
+            "POST",
+            "blocked.example",
+            443,
+            "credential-policy",
+            &tracing::Span::none(),
+        );
     })
     .await;
     assert_eq!(events.len(), 2);
@@ -24,16 +30,16 @@ async fn repeated_credential_denials_keep_decision_spans_separate() {
         let (events, received) = exported(level, || {
             let connect =
                 egress::connect_span(&EgressIntent::connect("blocked.example".to_string(), 443));
-            connect.in_scope(|| {
-                for _ in 0..2 {
-                    emit_credential_endpoint_mismatch(
-                        "POST",
-                        "blocked.example",
-                        443,
-                        "credential-policy",
-                    );
-                }
-            });
+            // The FORWARD path passes its saved connect span without entering it.
+            for _ in 0..2 {
+                emit_credential_endpoint_mismatch(
+                    "POST",
+                    "blocked.example",
+                    443,
+                    "credential-policy",
+                    &connect,
+                );
+            }
             let allowed = NetworkActivityBuilder::new(openshell_ocsf::ctx::ctx())
                 .action(ActionId::Allowed)
                 .dst_endpoint(Endpoint::from_domain("allowed.example", 443))

crates/openshell-supervisor-network/src/telemetry.rs

@@ -35,8 +35,9 @@ fn as_http(event: &OcsfEvent) -> Option<&HttpActivityEvent> {
 
 /// Emit all records of one synchronous decision under one deny span. Only HTTP
 /// Activity carries `trace`, so only a denied decision with an HTTP record gets
-/// a span; it is anchored on the denied HTTP record when there is one. A saved
-/// `parent` keeps the span in the connect trace after authorization exits.
+/// a span. The span describes the denied HTTP record, or the first HTTP record
+/// when only a companion finding is denied. A saved `parent` keeps the span in
+/// the connect trace after authorization exits.
 #[allow(clippy::disallowed_macros)]
 pub fn emit_decision(events: impl IntoIterator<Item = OcsfEvent>, parent: Option<&tracing::Span>) {
     let events: Vec<_> = events.into_iter().collect();
@@ -62,8 +63,12 @@ pub fn emit_decision(events: impl IntoIterator<Item = OcsfEvent>, parent: Option
     }
 }
 
-/// Returns `None` when the destination has used up its deny spans.
+/// Returns `None` when deny spans are disabled or the destination has used up
+/// its deny spans. Disabled spans spend no tokens.
 fn deny_span(event: &HttpActivityEvent, parent: Option<&tracing::Span>) -> Option<tracing::Span> {
+    if !tracing::enabled!(kind: tracing::metadata::Kind::SPAN, tracing::Level::INFO) {
+        return None;
+    }
     let endpoint = event.dst_endpoint.as_ref();
     let address =
         endpoint.and_then(|endpoint| endpoint.domain.as_deref().or(endpoint.ip.as_deref()));

crates/openshell-supervisor-network/src/telemetry/tests.rs

@@ -478,3 +478,15 @@ async fn repeated_denials_to_one_destination_export_bounded_spans() {
     assert_eq!(traced, denies, "limited records carry no trace.uid");
     assert_eq!(events.len(), super::DENY_SPAN_BURST as usize + 6);
 }
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+async fn disabled_deny_spans_spend_no_tokens() {
+    let _lock = tracing_test_lock().await;
+    super::reset_deny_span_limiter();
+    let subscriber =
+        tracing_subscriber::registry().with(tracing_subscriber::filter::LevelFilter::WARN);
+    tracing::subscriber::with_default(subscriber, || {
+        super::egress_emit!(http_deny("quiet.example", "denied"));
+    });
+    assert!(super::LIMITER.lock().unwrap().buckets.is_empty());
+}