Skip to content

Commit 9563240

Browse files
authored
fix(server): make HA sandbox create and HA e2e tests reliable (NVIDIA#3635)
* fix(server): retry runtime identity persistence on sandbox create With multiple gateway replicas, another replica can update a new sandbox record between the create path's read and its compare-and-swap write of the runtime identity. The create path made one attempt, so the conflict failed the request and deleted the backend sandbox. Persist the identity through the same retrying helper that start and startup recovery use, which rereads the record and retries while the sandbox stays in the same generation and a Provisioning or Ready phase. Signed-off-by: Kris Hicks <khicks@nvidia.com> * ci(e2e): run Kubernetes HA tests one at a time The two HA tests scale and roll the shared gateway Deployment. Their in-process lock does not apply under nextest, which runs each test in its own process, so one test could delete a gateway pod while the other was executing through it. Put both tests in a nextest test group limited to one thread in the e2e-kubernetes profile. The override matches test names because a binary() filter fails in workspaces that lack the HA test binary. Signed-off-by: Kris Hicks <khicks@nvidia.com> --------- Signed-off-by: Kris Hicks <khicks@nvidia.com>
1 parent a408f5d commit 9563240

2 files changed

Lines changed: 81 additions & 18 deletions

File tree

‎.config/nextest.toml‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,16 @@ failure-output = "immediate-final"
1717
# Flag slow tests after 60s and terminate them after five minutes.
1818
slow-timeout = { period = "60s", terminate-after = 5 }
1919

20+
# The HA tests scale and roll the shared gateway Deployment, so they cannot
21+
# overlap. Nextest runs each test in its own process. The filter matches test
22+
# names because binary() fails in workspaces without the HA test binary.
23+
[test-groups]
24+
kubernetes-ha = { max-threads = 1 }
25+
26+
[[profile.e2e-kubernetes.overrides]]
27+
filter = "test(/gateway_(scale_and_rollout|pod_rolls)$/)"
28+
test-group = "kubernetes-ha"
29+
2030
# Relative to the profile store dir (`e2e/rust/target/nextest/e2e-kubernetes/`).
2131
[profile.e2e-kubernetes.junit]
2232
path = "../../../../../results/e2e-kubernetes.xml"

‎crates/openshell-server/src/compute/mod.rs‎

Lines changed: 71 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -1128,21 +1128,14 @@ impl ComputeRuntime {
11281128
.await);
11291129
}
11301130
if self.supports_sandbox_authentication() {
1131-
let driver_name = self.configured_driver_name().to_string();
11321131
let persisted = self
1133-
.store
1134-
.update_message_cas::<Sandbox, _>(&sandbox_id, 0, move |sandbox| {
1135-
if let Some(metadata) = sandbox.metadata.as_mut() {
1136-
metadata.annotations.insert(
1137-
COMPUTE_DRIVER_ANNOTATION.to_string(),
1138-
driver_name.clone(),
1139-
);
1140-
metadata.annotations.insert(
1141-
COMPUTE_RUNTIME_IDENTITY_ANNOTATION.to_string(),
1142-
runtime_identity.clone(),
1143-
);
1144-
}
1145-
})
1132+
.persist_runtime_binding(
1133+
&sandbox_id,
1134+
&sandbox,
1135+
self.configured_driver_name(),
1136+
&runtime_identity,
1137+
&[SandboxPhase::Provisioning, SandboxPhase::Ready],
1138+
)
11461139
.await;
11471140
sandbox = match persisted {
11481141
Ok(sandbox) => sandbox,
@@ -1790,7 +1783,7 @@ impl ComputeRuntime {
17901783
let latest = if self.supports_sandbox_authentication() {
17911784
let driver_name = self.configured_driver_name().to_string();
17921785
let persisted = self
1793-
.persist_start_runtime_binding(
1786+
.persist_runtime_binding(
17941787
&sandbox_id,
17951788
&starting,
17961789
&driver_name,
@@ -1841,7 +1834,7 @@ impl ComputeRuntime {
18411834
}
18421835
}
18431836

1844-
async fn persist_start_runtime_binding(
1837+
async fn persist_runtime_binding(
18451838
&self,
18461839
sandbox_id: &str,
18471840
starting: &Sandbox,
@@ -1900,7 +1893,7 @@ impl ComputeRuntime {
19001893
sandbox_id,
19011894
attempt,
19021895
expected_resource_version,
1903-
"Retrying runtime identity persistence after concurrent start progress"
1896+
"Retrying runtime identity persistence after a concurrent sandbox update"
19041897
);
19051898
}
19061899
Err(error) => return Err(error.to_string()),
@@ -3072,7 +3065,7 @@ impl ComputeRuntime {
30723065
continue;
30733066
}
30743067
match self
3075-
.persist_start_runtime_binding(
3068+
.persist_runtime_binding(
30763069
&sandbox_id,
30773070
&sandbox,
30783071
self.configured_driver_name(),
@@ -7528,6 +7521,66 @@ mod tests {
75287521
assert_eq!(driver.delete_calls(), 0);
75297522
}
75307523

7524+
#[tokio::test]
7525+
async fn create_retries_runtime_binding_after_concurrent_replica_write() {
7526+
let directory = tempfile::tempdir().expect("temporary database directory");
7527+
let database_url = format!("sqlite://{}", directory.path().join("gateway.db").display());
7528+
let store = Arc::new(Store::connect(&database_url).await.expect("connect store"));
7529+
let pool = sqlx::SqlitePool::connect(&database_url)
7530+
.await
7531+
.expect("connect conflict injector");
7532+
sqlx::query("CREATE TABLE injected_conflicts (id TEXT PRIMARY KEY)")
7533+
.execute(&pool)
7534+
.await
7535+
.expect("create conflict marker table");
7536+
// Another replica writes the record between the binding read and its
7537+
// CAS write, once.
7538+
sqlx::query(
7539+
"CREATE TRIGGER inject_replica_write \
7540+
BEFORE UPDATE OF payload ON objects \
7541+
WHEN instr(NEW.payload, CAST('new-runtime-identity' AS BLOB)) > 0 \
7542+
AND NOT EXISTS (SELECT 1 FROM injected_conflicts WHERE id = OLD.id) \
7543+
BEGIN \
7544+
INSERT INTO injected_conflicts (id) VALUES (OLD.id); \
7545+
UPDATE objects SET resource_version = resource_version + 1 \
7546+
WHERE object_type = OLD.object_type AND id = OLD.id; \
7547+
SELECT RAISE(IGNORE); \
7548+
END",
7549+
)
7550+
.execute(&pool)
7551+
.await
7552+
.expect("install conflict trigger");
7553+
pool.close().await;
7554+
7555+
let driver = ControlledDriver::new();
7556+
driver.set_runtime_identity("new-runtime-identity");
7557+
let mut runtime = test_runtime(driver.clone()).await;
7558+
runtime.store = store;
7559+
enable_runtime_identity_binding(&mut runtime);
7560+
let sandbox = sandbox_record(
7561+
"sb-create-replica-race",
7562+
"create-replica-race",
7563+
SandboxPhase::Provisioning,
7564+
);
7565+
7566+
runtime
7567+
.create_sandbox(sandbox.clone(), None, false)
7568+
.await
7569+
.expect("create must retry the runtime binding after a concurrent write");
7570+
7571+
let stored = runtime
7572+
.store
7573+
.get_message::<Sandbox>(sandbox.object_id())
7574+
.await
7575+
.unwrap()
7576+
.unwrap();
7577+
assert_eq!(
7578+
stored.metadata.unwrap().annotations[COMPUTE_RUNTIME_IDENTITY_ANNOTATION],
7579+
"new-runtime-identity"
7580+
);
7581+
assert_eq!(driver.delete_calls(), 0);
7582+
}
7583+
75317584
#[tokio::test]
75327585
async fn empty_start_runtime_identity_stops_backend_and_restores_record() {
75337586
let driver = ControlledDriver::new();

0 commit comments

Comments
 (0)