Skip to content

Commit e3628be

Browse files
FrostGodTaylorMutch
authored andcommitted
fix(supervisor): enforce app route limits and shutdown fencing
Reject excess service declarations before persistence, fence registration against shutdown, and require the expected application Service count in listener E2E checks. Add focused regression coverage. Signed-off-by: divesh <dgude@nvidia.com> Signed-off-by: Taylor Mutch <taylormutch@gmail.com>
1 parent 0506c7d commit e3628be

7 files changed

Lines changed: 284 additions & 27 deletions

File tree

‎crates/openshell-server/src/grpc/service.rs‎

Lines changed: 177 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -19,11 +19,13 @@ use crate::ServerState;
1919
use crate::auth::workspace_authz::{
2020
AuthorizedWorkspaceScope, MinWorkspaceRole, authorize_list_workspace_selector,
2121
};
22+
use crate::compute::MutationScope;
2223
use crate::pagination::Pagination;
2324
use crate::persistence::{ObjectListQuery, ObjectType, WriteCondition};
2425
use crate::service_routing;
2526

2627
const MAX_SERVICE_NAME_LEN: usize = super::MAX_ROUTABLE_NAME_LEN;
28+
const MAX_APP_SERVICE_ROUTES: u32 = 1024;
2729

2830
#[cfg(test)]
2931
#[derive(Default)]
@@ -92,6 +94,21 @@ pub(super) async fn expose_service_endpoint(
9294
authorization_mode: ServiceAuthorizationMode,
9395
) -> Result<Response<ServiceEndpointResponse>, Status> {
9496
let sandbox_name = sandbox.object_name();
97+
// Count and persist under the same fleet-wide sandbox guard, including
98+
// create-time exposure, so concurrent gateways cannot overfill the allowlist.
99+
let _mutation_guard = if state.config.supervisor_app_listener.enabled {
100+
Some(
101+
state
102+
.compute
103+
.mutation_guard(MutationScope::sandbox(workspace, sandbox.object_id()))
104+
.await
105+
.map_err(|error| {
106+
super::persistence_error_to_status(error, "acquire sandbox mutation lock")
107+
})?,
108+
)
109+
} else {
110+
None
111+
};
95112

96113
let now = crate::persistence::current_time_ms();
97114
let key = service_routing::endpoint_key(sandbox_name, service);
@@ -103,6 +120,15 @@ pub(super) async fn expose_service_endpoint(
103120
.await
104121
.map_err(|e| Status::internal(format!("fetch endpoint failed: {e}")))?;
105122

123+
if state.config.supervisor_app_listener.enabled
124+
&& existing.is_none()
125+
&& app_service_endpoints(state, sandbox).await?.len() >= MAX_APP_SERVICE_ROUTES as usize
126+
{
127+
return Err(Status::resource_exhausted(
128+
"too many application service declarations",
129+
));
130+
}
131+
106132
let (id, created_at_ms, condition, created) = if let Some(existing) = existing {
107133
// Update path: preserve id and created_at, use CAS to prevent conflicts
108134
let resource_version = existing
@@ -383,26 +409,8 @@ pub(super) async fn configured_app_routes(
383409
if !state.config.supervisor_app_listener.enabled {
384410
return Ok(Vec::new());
385411
}
386-
let selector = format!("sandbox={}", sandbox.object_name());
387-
let page = state
388-
.store
389-
.list_message_page::<ServiceEndpoint>(
390-
ObjectListQuery::WorkspaceSelector {
391-
workspace: sandbox.object_workspace(),
392-
label_selector: &selector,
393-
},
394-
None,
395-
1024,
396-
)
397-
.await
398-
.map_err(|_| Status::unavailable("service declarations unavailable"))?;
399-
if page.next_cursor.is_some() {
400-
return Err(Status::resource_exhausted(
401-
"too many application service declarations",
402-
));
403-
}
404-
let mut service_routes: Vec<_> = page
405-
.messages
412+
let mut service_routes: Vec<_> = app_service_endpoints(state, sandbox)
413+
.await?
406414
.into_iter()
407415
.filter(|ep| {
408416
ep.sandbox_id == sandbox.object_id()
@@ -424,6 +432,31 @@ pub(super) async fn configured_app_routes(
424432
Ok(service_routes)
425433
}
426434

435+
async fn app_service_endpoints(
436+
state: &ServerState,
437+
sandbox: &Sandbox,
438+
) -> Result<Vec<ServiceEndpoint>, Status> {
439+
let selector = format!("sandbox={}", sandbox.object_name());
440+
let page = state
441+
.store
442+
.list_message_page::<ServiceEndpoint>(
443+
ObjectListQuery::WorkspaceSelector {
444+
workspace: sandbox.object_workspace(),
445+
label_selector: &selector,
446+
},
447+
None,
448+
MAX_APP_SERVICE_ROUTES,
449+
)
450+
.await
451+
.map_err(|_| Status::unavailable("service declarations unavailable"))?;
452+
if page.next_cursor.is_some() {
453+
return Err(Status::resource_exhausted(
454+
"too many application service declarations",
455+
));
456+
}
457+
Ok(page.messages)
458+
}
459+
427460
async fn get_service_endpoint(
428461
state: &Arc<ServerState>,
429462
workspace: &str,
@@ -536,6 +569,130 @@ mod tests {
536569
state.store.put_message(&sandbox).await.unwrap();
537570
}
538571

572+
async fn app_listener_state() -> Arc<ServerState> {
573+
let mut state = test_server_state().await;
574+
Arc::get_mut(&mut state)
575+
.unwrap()
576+
.config
577+
.supervisor_app_listener = openshell_core::app_listener::AppListenerConfig {
578+
enabled: true,
579+
port: 9443,
580+
domain: "services.example.com".into(),
581+
};
582+
seed_sandbox(&state, "limited").await;
583+
state
584+
}
585+
586+
async fn seed_app_services(state: &Arc<ServerState>, count: u32) {
587+
for index in 0..count {
588+
let name = format!("web-{index}");
589+
state
590+
.store
591+
.put_message(&ServiceEndpoint {
592+
metadata: Some(ObjectMeta {
593+
id: format!("endpoint-{index}"),
594+
name: service_routing::endpoint_key("limited", &name),
595+
workspace: "default".into(),
596+
labels: HashMap::from([("sandbox".into(), "limited".into())]),
597+
..Default::default()
598+
}),
599+
sandbox_id: "sandbox-limited".into(),
600+
sandbox: "limited".into(),
601+
name,
602+
target_port: 8080,
603+
domain: true,
604+
authorization_mode: ServiceAuthorizationMode::Strip as i32,
605+
})
606+
.await
607+
.unwrap();
608+
}
609+
}
610+
611+
async fn expose_limited(
612+
state: &Arc<ServerState>,
613+
name: &str,
614+
) -> Result<Response<ServiceEndpointResponse>, Status> {
615+
handle_expose_service(
616+
state,
617+
authed_request(ExposeServiceRequest {
618+
sandbox: "limited".into(),
619+
name: name.into(),
620+
target_port: 9090,
621+
workspace_scope: Some(openshell_core::proto::workspace_selector("default")),
622+
..Default::default()
623+
}),
624+
)
625+
.await
626+
}
627+
628+
#[tokio::test]
629+
async fn app_listener_rejects_overflow_before_persistence_and_keeps_config_deliverable() {
630+
let state = app_listener_state().await;
631+
seed_app_services(&state, MAX_APP_SERVICE_ROUTES).await;
632+
assert_eq!(
633+
expose_limited(&state, "overflow").await.unwrap_err().code(),
634+
tonic::Code::ResourceExhausted
635+
);
636+
assert!(
637+
get_service_endpoint(&state, "default", "limited", "overflow")
638+
.await
639+
.unwrap()
640+
.is_none()
641+
);
642+
// Editing a route at capacity must remain possible, as must unrelated
643+
// policy changes and the complete configuration that delivers them.
644+
expose_limited(&state, "web-0").await.unwrap();
645+
super::super::policy::handle_update_config(
646+
&state,
647+
authed_request(openshell_core::proto::UpdateConfigRequest {
648+
sandbox: "limited".into(),
649+
workspace_scope: Some(openshell_core::proto::workspace_selector("default")),
650+
policy: Some(openshell_policy::restrictive_default_policy()),
651+
..Default::default()
652+
}),
653+
)
654+
.await
655+
.unwrap();
656+
let sandbox = state
657+
.store
658+
.get_message::<Sandbox>("sandbox-limited")
659+
.await
660+
.unwrap()
661+
.unwrap();
662+
let config = super::super::policy::load_sandbox_config(&state, &sandbox)
663+
.await
664+
.unwrap();
665+
assert_eq!(config.service_routes.len(), MAX_APP_SERVICE_ROUTES as usize);
666+
assert!(config.policy.is_some());
667+
assert_eq!(config.service_routes[0].target_port, 9090);
668+
}
669+
670+
#[tokio::test]
671+
async fn app_listener_concurrent_exposures_cannot_exceed_capacity() {
672+
let state = app_listener_state().await;
673+
seed_app_services(&state, MAX_APP_SERVICE_ROUTES - 1).await;
674+
let barrier = Arc::new(tokio::sync::Barrier::new(2));
675+
let spawn_exposure = |name: &'static str| {
676+
let state = state.clone();
677+
let barrier = barrier.clone();
678+
tokio::spawn(async move {
679+
barrier.wait().await;
680+
expose_limited(&state, name).await
681+
})
682+
};
683+
let (first, second) = tokio::join!(spawn_exposure("first"), spawn_exposure("second"));
684+
let results = [first.unwrap(), second.unwrap()];
685+
assert_eq!(results.iter().filter(|result| result.is_ok()).count(), 1);
686+
assert_eq!(
687+
results
688+
.iter()
689+
.find_map(|result| result.as_ref().err())
690+
.unwrap()
691+
.code(),
692+
tonic::Code::ResourceExhausted
693+
);
694+
}
695+
539696
#[test]
540697
fn validates_good_endpoint_name() {
541698
validate_endpoint_name("service", "web-api", 28).unwrap();

‎crates/openshell-supervisor/src/app_listener.rs‎

Lines changed: 59 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -143,6 +143,14 @@ pub struct Routes {
143143
registration: Option<Arc<Registration>>,
144144
sender: watch::Sender<Snapshot>,
145145
stopped: Arc<AtomicBool>,
146+
#[cfg(test)]
147+
install_probe: Option<Arc<InstallProbe>>,
148+
}
149+
150+
#[cfg(test)]
151+
struct InstallProbe {
152+
checked_running: std::sync::Barrier,
153+
resume: std::sync::Barrier,
146154
}
147155

148156
impl Default for Routes {
@@ -151,6 +159,8 @@ impl Default for Routes {
151159
sender: watch::channel(Snapshot::default()).0,
152160
stopped: Arc::new(AtomicBool::new(false)),
153161
registration: None,
162+
#[cfg(test)]
163+
install_probe: None,
154164
}
155165
}
156166
}
@@ -180,6 +190,11 @@ impl Routes {
180190
if self.stopped.load(Ordering::Acquire) {
181191
return;
182192
}
193+
#[cfg(test)]
194+
if let Some(probe) = &self.install_probe {
195+
probe.checked_running.wait();
196+
probe.resume.wait();
197+
}
183198
let mut routes = BTreeMap::new();
184199
if !result.configuration_admitted {
185200
self.suspend();
@@ -212,7 +227,7 @@ impl Routes {
212227
return;
213228
}
214229
if let Some(registration) = &self.registration
215-
&& !registration.bind(&result.workspace)
230+
&& !registration.bind(&result.workspace, &self.stopped)
216231
{
217232
self.suspend();
218233
return;
@@ -362,12 +377,18 @@ struct Registration {
362377
}
363378

364379
impl Registration {
365-
fn bind(&self, workspace: &str) -> bool {
380+
fn bind(&self, workspace: &str, stopped: &AtomicBool) -> bool {
366381
let next = format!("{workspace}--{}", self.sandbox);
367382
let mut prefix = self
368383
.prefix
369384
.lock()
370385
.unwrap_or_else(std::sync::PoisonError::into_inner);
386+
// Shutdown sets stopped before taking this same prefix lock to remove
387+
// the registration. An install that passed its first check must not
388+
// reinsert the old sender after that removal.
389+
if stopped.load(Ordering::Acquire) {
390+
return false;
391+
}
371392
let mut entries = self
372393
.router
373394
.registrations
@@ -1178,6 +1199,42 @@ mod tests {
11781199
);
11791200
}
11801201

1202+
#[tokio::test]
1203+
async fn install_racing_shutdown_cannot_reinsert_a_stopped_instance() {
1204+
let f = fixture();
1205+
let probe = Arc::new(InstallProbe {
1206+
checked_running: std::sync::Barrier::new(2),
1207+
resume: std::sync::Barrier::new(2),
1208+
});
1209+
let mut installing_routes = f.routes.clone();
1210+
installing_routes.install_probe = Some(probe.clone());
1211+
let snapshot = installed(vec![f.route.clone()]);
1212+
let installer = std::thread::spawn(move || installing_routes.install(&snapshot));
1213+
probe.checked_running.wait();
1214+
// Complete shutdown after install's optimistic stopped check but
1215+
// before its registration insertion, rather than relying on timing.
1216+
drop(f.handle);
1217+
probe.resume.wait();
1218+
installer.join().unwrap();
1219+
assert!(
1220+
!f.host
1221+
.router()
1222+
.registrations
1223+
.lock()
1224+
.unwrap()
1225+
.contains_key("default--sandbox-a")
1226+
);
1227+
let replacement = f.host.router().instance("sandbox-a").unwrap();
1228+
replacement.routes.install(&installed(vec![f.route]));
1229+
assert!(
1230+
replacement
1231+
.routes
1232+
.resolve(HOST, "default--sandbox-a", "svc.openshell.internal")
1233+
.is_some(),
1234+
"the stopped instance must not prevent its replacement from serving"
1235+
);
1236+
}
1237+
11811238
#[tokio::test(start_paused = true)]
11821239
async fn push_freshness_renews_live_routes_but_cannot_revive_rejected_routes() {
11831240
let routes = Routes::default();

‎e2e/kubernetes/app-listener/README.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -56,6 +56,7 @@ Run from the repository root, supplying the public application certificate/CA:
5656
KUBECONFIG="$PWD/kubeconfig" uv run --no-project python \
5757
e2e/kubernetes/app-listener/verify.py \
5858
--namespace listener-review-shared \
59+
--topology shared \
5960
--gateway-endpoint https://localhost:29203 \
6061
--forward-gateway service/listener-review \
6162
--app-ca /path/to/app-ca.crt \

‎e2e/kubernetes/app-listener/run.py‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -247,6 +247,8 @@ def wait_for_shared_hosts():
247247
str(HERE / "verify.py"),
248248
"--namespace",
249249
namespace,
250+
"--topology",
251+
args.topology,
250252
"--gateway-endpoint",
251253
f"https://localhost:{port}",
252254
"--forward-gateway",

0 commit comments

Comments
 (0)