@@ -46,6 +46,10 @@ const CNI_REGISTRATION_LABEL: &str = "openshell.ai/cni-registration";
4646/// currently enforces on that node, so the gateway can wait for cluster-wide
4747/// acknowledgement of its namespace before it serves sandboxes.
4848const NODE_COVERAGE_ANNOTATION : & str = "openshell.ai/cni-sandbox-namespaces" ;
49+ /// Label selector identifying OpenShell-managed sandbox pods (set by the gateway
50+ /// driver). Used to keep a namespace enforced while its sandboxes are still
51+ /// running even after its registration marker is removed (drain-gated prune).
52+ const SANDBOX_POD_LABEL_SELECTOR : & str = "openshell.ai/managed-by=openshell" ;
4953#[ allow( dead_code) ]
5054const OPENSHELL_TABLE : & str = "openshell_sidecar_bypass" ;
5155#[ allow( dead_code) ]
@@ -188,6 +192,10 @@ trait PodReader {
188192
189193trait RuleInstaller {
190194 fn install ( & self , netns : & Path , proxy_uid : u32 ) -> Result < InstallReport > ;
195+ /// Read-only validation that the bypass-prevention rules are present. Must NOT
196+ /// modify live rules (CNI CHECK runs on a running pod; a destructive reinstall
197+ /// that fails would leave it unenforced).
198+ fn verify ( & self , netns : & Path ) -> Result < ( ) > ;
191199 fn cleanup ( & self , netns : & Path ) -> Result < ( ) > ;
192200}
193201
@@ -254,13 +262,23 @@ pub fn node_ready(args: &[String]) -> Result<()> {
254262
255263/// Entry point for `openshell-cni list-sandbox-namespaces`.
256264///
257- /// Run by the installer reconcile to aggregate every namespace that contains a
258- /// Helm-owned registration marker `ConfigMap` (label
259- /// `openshell.ai/cni-registration=true`) into the plugin allowlist. Prints one
260- /// namespace name per line.
265+ /// Run by the installer reconcile to compute the enforcement allowlist. Prints,
266+ /// one per line, the union of:
267+ /// * namespaces containing a Helm-owned registration marker `ConfigMap`
268+ /// (`openshell.ai/cni-registration=true`) — active registrations; and
269+ /// * namespaces that still contain an OpenShell-managed sandbox pod
270+ /// (`openshell.ai/managed-by=openshell`).
271+ ///
272+ /// The second set makes the allowlist **monotonic while in use**: removing a
273+ /// release's marker does not immediately drop enforcement for a namespace whose
274+ /// sandboxes are still running (which would fail-open their recreated pods). The
275+ /// namespace is pruned only once it is drained — no marker and no sandbox pods.
261276pub fn list_sandbox_namespaces ( ) -> Result < ( ) > {
262277 let client = KubeApiClient :: from_in_cluster ( ) ?;
263- let namespaces = client. list_registration_namespaces ( CNI_REGISTRATION_LABEL ) ?;
278+ let mut namespaces = client. list_registration_namespaces ( CNI_REGISTRATION_LABEL ) ?;
279+ namespaces. extend ( client. list_sandbox_pod_namespaces ( SANDBOX_POD_LABEL_SELECTOR ) ?) ;
280+ namespaces. sort ( ) ;
281+ namespaces. dedup ( ) ;
264282 for ns in namespaces {
265283 println ! ( "{ns}" ) ;
266284 }
@@ -467,7 +485,12 @@ fn handle_command(
467485 let netns = env. netns . as_deref ( ) . ok_or_else ( || {
468486 miette:: miette!( "CNI_NETNS is required for OpenShell CNI CHECK" )
469487 } ) ?;
470- let report = installer. install ( netns, workload. proxy_uid ) ?;
488+ // CHECK is read-only: verify the rules are present without touching
489+ // them, so a check never leaves a running pod momentarily unenforced.
490+ installer. verify ( netns) ?;
491+ let report = InstallReport {
492+ backend : "verified" ,
493+ } ;
471494 log_cni_info (
472495 & config,
473496 env,
@@ -594,6 +617,16 @@ struct KubeApiClient {
594617 client : reqwest:: blocking:: Client ,
595618}
596619
620+ /// Blocking HTTP client builder with bounded connect and request deadlines. The
621+ /// CNI plugin runs synchronously on the CNI ADD path and in the installer's
622+ /// reconcile/coverage loops; without deadlines an API stall would wedge pod
623+ /// creation, invalidate the coverage wait's time bound, and stall reconciliation.
624+ fn kube_client_builder ( ) -> reqwest:: blocking:: ClientBuilder {
625+ reqwest:: blocking:: Client :: builder ( )
626+ . connect_timeout ( std:: time:: Duration :: from_secs ( 5 ) )
627+ . timeout ( std:: time:: Duration :: from_secs ( 15 ) )
628+ }
629+
597630impl KubeApiClient {
598631 fn from_kubeconfig ( path : & Path ) -> Result < Self > {
599632 let contents = std:: fs:: read_to_string ( path)
@@ -630,7 +663,7 @@ impl KubeApiClient {
630663 ) ) ;
631664 }
632665 } ;
633- let mut builder = reqwest :: blocking :: Client :: builder ( ) ;
666+ let mut builder = kube_client_builder ( ) ;
634667 if let Some ( ca) = cluster_certificate_authority ( path, & cluster. cluster ) ? {
635668 builder = builder. add_root_certificate ( ca) ;
636669 }
@@ -663,7 +696,7 @@ impl KubeApiClient {
663696 . into_diagnostic ( )
664697 . wrap_err ( "failed to read in-cluster CA certificate" ) ?;
665698 let ca = reqwest:: Certificate :: from_pem ( & ca_pem) . into_diagnostic ( ) ?;
666- let client = reqwest :: blocking :: Client :: builder ( )
699+ let client = kube_client_builder ( )
667700 . add_root_certificate ( ca)
668701 . build ( )
669702 . into_diagnostic ( ) ?;
@@ -762,6 +795,39 @@ impl KubeApiClient {
762795 Ok ( namespaces)
763796 }
764797
798+ /// Returns the distinct namespaces of all pods matching `selector`
799+ /// cluster-wide (used to keep enforcement for namespaces with live sandboxes).
800+ fn list_sandbox_pod_namespaces ( & self , selector : & str ) -> Result < Vec < String > > {
801+ let url = format ! ( "{}/api/v1/pods" , self . server) ;
802+ let response = self
803+ . client
804+ . get ( url)
805+ . query ( & [ ( "labelSelector" , selector) ] )
806+ . bearer_auth ( & self . token )
807+ . send ( )
808+ . into_diagnostic ( )
809+ . wrap_err ( "failed to list sandbox pods" ) ?;
810+ if !response. status ( ) . is_success ( ) {
811+ return Err ( miette:: miette!(
812+ "Kubernetes API returned {} while listing sandbox pods" ,
813+ response. status( )
814+ ) ) ;
815+ }
816+ let list = response. json :: < Value > ( ) . into_diagnostic ( ) ?;
817+ let mut namespaces: Vec < String > = list[ "items" ]
818+ . as_array ( )
819+ . map ( |items| {
820+ items
821+ . iter ( )
822+ . filter_map ( |item| item[ "metadata" ] [ "namespace" ] . as_str ( ) . map ( str:: to_string) )
823+ . collect ( )
824+ } )
825+ . unwrap_or_default ( ) ;
826+ namespaces. sort ( ) ;
827+ namespaces. dedup ( ) ;
828+ Ok ( namespaces)
829+ }
830+
765831 /// Returns true when at least one node carries `ready_label` and every such
766832 /// node's `coverage_annotation` (a comma-separated list) contains `namespace`.
767833 fn namespace_covered_on_all_ready_nodes (
@@ -965,6 +1031,10 @@ impl RuleInstaller for Runtime {
9651031 install_rules ( netns, proxy_uid)
9661032 }
9671033
1034+ fn verify ( & self , netns : & Path ) -> Result < ( ) > {
1035+ verify_rules ( netns)
1036+ }
1037+
9681038 fn cleanup ( & self , netns : & Path ) -> Result < ( ) > {
9691039 cleanup_rules ( netns)
9701040 }
@@ -1044,11 +1114,50 @@ fn install_rules(netns: &Path, proxy_uid: u32) -> Result<InstallReport> {
10441114
10451115#[ cfg( target_os = "linux" ) ]
10461116fn install_nft_rules ( netns : & Path , proxy_uid : u32 , nft : & str ) -> Result < ( ) > {
1047- let _ = run_nft_args_in_netns ( netns, & nft, & [ "delete" , "table" , "inet" , OPENSHELL_TABLE ] ) ;
1048- let ruleset = generate_sidecar_bypass_ruleset ( proxy_uid, Some ( "openshell:cni-sidecar:" ) ) ;
1117+ // Atomic replace: ensure-exists, delete, recreate in a SINGLE `nft -f`
1118+ // transaction. nft applies the whole file atomically, so a failure never
1119+ // leaves the pod with the table deleted-but-not-recreated (unenforced). The
1120+ // leading `table {}` makes the subsequent `delete table` safe on first apply.
1121+ let body = generate_sidecar_bypass_ruleset ( proxy_uid, Some ( "openshell:cni-sidecar:" ) ) ;
1122+ let ruleset =
1123+ format ! ( "table inet {OPENSHELL_TABLE} {{}}\n delete table inet {OPENSHELL_TABLE}\n {body}" ) ;
10491124 run_nft_ruleset_in_netns ( netns, & nft, & ruleset)
10501125}
10511126
1127+ /// Read-only check that the bypass-prevention rules are present in the netns.
1128+ /// Never modifies live rules. Prefers nft (the table), falls back to the iptables
1129+ /// chain; errors when neither is present so CNI CHECK surfaces the gap.
1130+ #[ cfg( target_os = "linux" ) ]
1131+ fn verify_rules ( netns : & Path ) -> Result < ( ) > {
1132+ if let Some ( nft) = find_nft ( ) {
1133+ if run_nft_args_in_netns ( netns, & nft, & [ "list" , "table" , "inet" , OPENSHELL_TABLE ] ) . is_ok ( ) {
1134+ return Ok ( ( ) ) ;
1135+ }
1136+ }
1137+ if let Some ( iptables) = find_iptables ( ) {
1138+ if run_command_in_netns (
1139+ netns,
1140+ & iptables. ipv4 ,
1141+ & [ "-w" , "-t" , "filter" , "-n" , "-L" , OPENSHELL_IPTABLES_CHAIN ] ,
1142+ )
1143+ . is_ok ( )
1144+ {
1145+ return Ok ( ( ) ) ;
1146+ }
1147+ }
1148+ Err ( miette:: miette!(
1149+ "OpenShell CNI bypass-prevention rules are not present in the pod network namespace"
1150+ ) )
1151+ }
1152+
1153+ #[ cfg( not( target_os = "linux" ) ) ]
1154+ fn verify_rules ( netns : & Path ) -> Result < ( ) > {
1155+ let _ = netns;
1156+ Err ( miette:: miette!(
1157+ "OpenShell CNI rule verification is supported only on Linux nodes"
1158+ ) )
1159+ }
1160+
10521161#[ cfg( not( target_os = "linux" ) ) ]
10531162fn install_rules ( netns : & Path , proxy_uid : u32 ) -> Result < InstallReport > {
10541163 let _ = ( netns, proxy_uid) ;
@@ -1387,6 +1496,7 @@ mod tests {
13871496 #[ derive( Default ) ]
13881497 struct TestInstaller {
13891498 installed : std:: sync:: Mutex < Vec < u32 > > ,
1499+ verified : std:: sync:: Mutex < u32 > ,
13901500 cleaned : std:: sync:: Mutex < u32 > ,
13911501 }
13921502
@@ -1396,6 +1506,11 @@ mod tests {
13961506 Ok ( InstallReport { backend : "test" } )
13971507 }
13981508
1509+ fn verify ( & self , _netns : & Path ) -> Result < ( ) > {
1510+ * self . verified . lock ( ) . unwrap ( ) += 1 ;
1511+ Ok ( ( ) )
1512+ }
1513+
13991514 fn cleanup ( & self , _netns : & Path ) -> Result < ( ) > {
14001515 * self . cleaned . lock ( ) . unwrap ( ) += 1 ;
14011516 Ok ( ( ) )
@@ -1502,6 +1617,19 @@ mod tests {
15021617 assert_eq ! ( * installer. installed. lock( ) . unwrap( ) , vec![ 1337 ] ) ;
15031618 }
15041619
1620+ #[ test]
1621+ fn check_verifies_without_reinstalling ( ) {
1622+ let pods = TestPods {
1623+ annotations : openshell_annotations ( ) ,
1624+ } ;
1625+ let installer = TestInstaller :: default ( ) ;
1626+ handle_command ( & cni_input ( ) , & env ( "CHECK" ) , & pods, & installer) . unwrap ( ) ;
1627+ // CHECK must be read-only: verify, never install (a destructive reinstall
1628+ // that failed would leave the running pod unenforced).
1629+ assert_eq ! ( * installer. verified. lock( ) . unwrap( ) , 1 ) ;
1630+ assert ! ( installer. installed. lock( ) . unwrap( ) . is_empty( ) ) ;
1631+ }
1632+
15051633 #[ test]
15061634 fn add_passes_through_non_openshell_pod ( ) {
15071635 let pods = TestPods {
0 commit comments