Skip to content

Commit cf5748a

Browse files
fix(cortex): allocate DAG sequence numbers per author, seeded from the store
`DagAction::seq` is documented as a per-author sequence, and the DAG store keys its author index on `(author, seq)`. The node allocated it from one process-global counter that always started at 1, which broke that contract in two ways — the second one loses history. Gaps: interleaved writes by different authors (the node itself, the tool surface) drew from the same counter, so each author's sequence came out full of holes. Nothing in-tree depends on contiguity, but the number no longer means what the type says it means. Eviction: because the counter restarted at 1 on every process start, a node reopening a persistent DAG re-issued sequence numbers the store already held. `put` inserts into the author index unconditionally, so each new action silently REPLACED the pre-restart action holding that key. Measured on a sled-backed store: four writes across a restart leave four actions on disk and a two-entry author chain. The evicted actions are still stored, still signed and still linked by hash — but they disappear from every view built on the author chain: `/api/v1/dag/chain`, the git-provenance list, the approval list. History that is present and unreadable is not much better than history that is missing. `AppState::next_dag_seq` now allocates per author and seeds an author's counter from its highest recorded `seq` the first time it is asked, so numbering continues the chain instead of colliding with it. All seven allocation sites — the shared triple write path, delete, the two Raft-routed handlers, the GraphQL mutations, custom actions, review approvals and git provenance — go through it. Regression tests cover both symptoms: a restart against a persistent store must leave all writes visible with a contiguous 1..4 sequence, and two authors writing through the same node must each own a gapless sequence.
1 parent 2d18031 commit cf5748a

7 files changed

Lines changed: 191 additions & 41 deletions

File tree

crates/aingle_cortex/src/graphql/resolvers.rs

Lines changed: 2 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -176,9 +176,7 @@ impl MutationRoot {
176176
.dag_author
177177
.clone()
178178
.unwrap_or_else(|| aingle_graph::NodeId::named("node:local"));
179-
let dag_seq = state
180-
.dag_seq_counter
181-
.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
179+
let dag_seq = state.next_dag_seq(&dag_author, Some(dag_store));
182180
let parents = dag_store.tips().unwrap_or_default();
183181

184182
let mut action = aingle_graph::dag::DagAction {
@@ -238,9 +236,7 @@ impl MutationRoot {
238236
.dag_author
239237
.clone()
240238
.unwrap_or_else(|| aingle_graph::NodeId::named("node:local"));
241-
let dag_seq = state
242-
.dag_seq_counter
243-
.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
239+
let dag_seq = state.next_dag_seq(&dag_author, Some(dag_store));
244240
let parents = dag_store.tips().unwrap_or_default();
245241

246242
let mut action = aingle_graph::dag::DagAction {

crates/aingle_cortex/src/rest/dag.rs

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -640,15 +640,15 @@ pub async fn post_create_dag_action(
640640
.unwrap_or_else(|| aingle_graph::NodeId::named("node:local"))
641641
};
642642

643-
let dag_seq = state
644-
.dag_seq_counter
645-
.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
646-
647643
let graph = state.graph.read().await;
648644
let dag_store = graph
649645
.dag_store()
650646
.ok_or_else(|| Error::Internal("DAG not enabled".into()))?;
651647

648+
// Allocated after the store is in hand: the number continues this author's
649+
// recorded chain instead of restarting at 1 and evicting its own history.
650+
let dag_seq = state.next_dag_seq(&dag_author, Some(dag_store));
651+
652652
let parents = dag_store
653653
.tips()
654654
.map_err(|e| Error::Internal(e.to_string()))?;

crates/aingle_cortex/src/rest/triples.rs

Lines changed: 11 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -180,14 +180,14 @@ pub async fn create_triple(
180180
let dag_author = state.dag_author.clone().unwrap_or_else(|| {
181181
aingle_graph::NodeId::named(&format!("node:{}", state.cluster_node_id.unwrap_or(0)))
182182
});
183-
let dag_seq = state
184-
.dag_seq_counter
185-
.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
186-
187-
// Get current tips
188-
let parents = {
183+
// Get current tips, and a sequence number that continues this author's
184+
// recorded chain rather than restarting at 1 and evicting its own history
185+
// from the (author, seq) index.
186+
let (parents, dag_seq) = {
189187
let graph = state.graph.read().await;
190-
graph.dag_tips().unwrap_or_default()
188+
let parents = graph.dag_tips().unwrap_or_default();
189+
let seq = state.next_dag_seq(&dag_author, graph.dag_store());
190+
(parents, seq)
191191
};
192192

193193
let mut action = aingle_graph::dag::DagAction {
@@ -459,19 +459,17 @@ pub async fn delete_triple(
459459
let dag_author = state.dag_author.clone().unwrap_or_else(|| {
460460
aingle_graph::NodeId::named(&format!("node:{}", state.cluster_node_id.unwrap_or(0)))
461461
});
462-
let dag_seq = state
463-
.dag_seq_counter
464-
.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
465-
466-
let (parents, subject_for_dag) = {
462+
let (parents, subject_for_dag, dag_seq) = {
467463
let graph = state.graph.read().await;
468464
let tips = graph.dag_tips().unwrap_or_default();
469465
let subj = graph
470466
.get(&triple_id)
471467
.ok()
472468
.flatten()
473469
.map(|t| crate::service::triples::dag_subject_name(&t.subject));
474-
(tips, subj)
470+
// Continues this author's recorded chain instead of restarting at 1.
471+
let seq = state.next_dag_seq(&dag_author, graph.dag_store());
472+
(tips, subj, seq)
475473
};
476474

477475
let mut action = aingle_graph::dag::DagAction {

crates/aingle_cortex/src/service/git_provenance.rs

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -117,9 +117,7 @@ pub async fn record_git_provenance(
117117
.dag_author
118118
.clone()
119119
.unwrap_or_else(|| aingle_graph::NodeId::named("node:local"));
120-
let seq = state
121-
.dag_seq_counter
122-
.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
120+
let seq = state.next_dag_seq(&author, Some(dag_store));
123121
let parents = dag_store.tips().unwrap_or_default();
124122
let at = chrono::Utc::now();
125123
let summary = match &git_ref.branch {

crates/aingle_cortex/src/service/review.rs

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -52,9 +52,7 @@ pub async fn record_approval(
5252
.dag_author
5353
.clone()
5454
.unwrap_or_else(|| aingle_graph::NodeId::named("node:local"));
55-
let seq = state
56-
.dag_seq_counter
57-
.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
55+
let seq = state.next_dag_seq(&author, Some(dag_store));
5856
let parents = dag_store.tips().unwrap_or_default();
5957
let approved_at = chrono::Utc::now();
6058

crates/aingle_cortex/src/service/triples.rs

Lines changed: 115 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -70,9 +70,7 @@ pub(crate) fn record_insert_action(
7070
origin: Option<&str>,
7171
) -> Result<()> {
7272
let dag_author = dag_action_author(state, origin);
73-
let dag_seq = state
74-
.dag_seq_counter
75-
.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
73+
let dag_seq = state.next_dag_seq(&dag_author, Some(dag_store));
7674
let parents = dag_store.tips().unwrap_or_default();
7775

7876
let mut action = aingle_graph::dag::DagAction {
@@ -399,9 +397,7 @@ pub async fn delete_triple(
399397
if deleted {
400398
if let Some(dag_store) = graph.dag_store() {
401399
let dag_author = dag_action_author(state, origin);
402-
let dag_seq = state
403-
.dag_seq_counter
404-
.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
400+
let dag_seq = state.next_dag_seq(&dag_author, Some(dag_store));
405401
let parents = dag_store.tips().unwrap_or_default();
406402

407403
let mut action = aingle_graph::dag::DagAction {
@@ -781,6 +777,119 @@ mod tests {
781777
}
782778
}
783779

780+
/// A restart must continue each author's sequence, not restart it. The DAG
781+
/// store keys its author index on `(author, seq)`, so re-issuing a number the
782+
/// store already holds evicts the older action from that index: it survives
783+
/// on disk and in the parent chain, but vanishes from every author-chain view
784+
/// — history that is present and unreadable.
785+
#[cfg(feature = "dag")]
786+
#[tokio::test]
787+
async fn a_restart_continues_the_author_sequence_instead_of_evicting_history() {
788+
let dir = tempfile::tempdir().unwrap();
789+
let path = dir.path().join("g.sled");
790+
let p = path.to_str().unwrap().to_string();
791+
792+
let author = NodeId::named("node:local");
793+
794+
// First run: two writes.
795+
{
796+
let state = AppState::with_db_path(&p, None).unwrap();
797+
{
798+
let mut graph = state.graph.write().await;
799+
graph.enable_dag_persistent(&p).unwrap();
800+
}
801+
create_triple(&state, req("ex:a", "ex:p", "ex:1"), None, None)
802+
.await
803+
.unwrap();
804+
create_triple(&state, req("ex:b", "ex:p", "ex:2"), None, None)
805+
.await
806+
.unwrap();
807+
state
808+
.graph
809+
.read()
810+
.await
811+
.dag_store()
812+
.unwrap()
813+
.flush()
814+
.unwrap();
815+
}
816+
817+
// Second run against the same store: a fresh in-memory allocator.
818+
{
819+
let state = AppState::with_db_path(&p, None).unwrap();
820+
{
821+
let mut graph = state.graph.write().await;
822+
graph.enable_dag_persistent(&p).unwrap();
823+
}
824+
create_triple(&state, req("ex:c", "ex:p", "ex:3"), None, None)
825+
.await
826+
.unwrap();
827+
create_triple(&state, req("ex:d", "ex:p", "ex:4"), None, None)
828+
.await
829+
.unwrap();
830+
831+
let graph = state.graph.read().await;
832+
let store = graph.dag_store().unwrap();
833+
let chain = store.chain(&author, 100).unwrap();
834+
assert_eq!(
835+
chain.len(),
836+
4,
837+
"all four writes must remain visible in the author chain after a restart"
838+
);
839+
let mut seqs: Vec<u64> = chain.iter().map(|a| a.seq).collect();
840+
seqs.sort_unstable();
841+
assert_eq!(
842+
seqs,
843+
vec![1, 2, 3, 4],
844+
"an author's sequence must be contiguous, not restarted or gapped"
845+
);
846+
}
847+
}
848+
849+
/// Two authors writing through the same node must not eat each other's
850+
/// numbers: `seq` is documented and indexed per author.
851+
#[cfg(feature = "dag")]
852+
#[tokio::test]
853+
async fn interleaved_authors_get_independent_sequences() {
854+
let state = AppState::with_db_path(":memory:", None).unwrap();
855+
{
856+
let mut graph = state.graph.write().await;
857+
graph.enable_dag();
858+
}
859+
860+
for i in 0..3 {
861+
create_triple(
862+
&state,
863+
req("ex:alice", "ex:knows", &format!("ex:local{i}")),
864+
None,
865+
None,
866+
)
867+
.await
868+
.unwrap();
869+
create_triple(
870+
&state,
871+
req("ex:alice", "ex:knows", &format!("ex:tool{i}")),
872+
None,
873+
Some("mcp"),
874+
)
875+
.await
876+
.unwrap();
877+
}
878+
879+
let graph = state.graph.read().await;
880+
let store = graph.dag_store().unwrap();
881+
for author in ["node:local", "mcp"] {
882+
let mut seqs: Vec<u64> = store
883+
.chain(&NodeId::named(author), 100)
884+
.unwrap()
885+
.iter()
886+
.map(|a| a.seq)
887+
.collect();
888+
seqs.sort_unstable();
889+
assert_eq!(seqs, vec![1, 2, 3], "{author} must own a gapless sequence");
890+
}
891+
}
892+
784893
#[tokio::test]
785894
async fn list_triples_returns_inserted() {
786895
let state = AppState::with_db_path(":memory:", None).unwrap();

crates/aingle_cortex/src/state.rs

Lines changed: 57 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -110,9 +110,22 @@ pub struct AppState {
110110
/// This node's author identity for DAG actions.
111111
#[cfg(feature = "dag")]
112112
pub dag_author: Option<aingle_graph::NodeId>,
113-
/// Per-author monotonic sequence counter for DAG actions.
113+
/// Next DAG sequence number to hand out, per author.
114+
///
115+
/// A `DagAction`'s `seq` is documented — and indexed — as a *per-author*
116+
/// number: the DAG store keys its author index on `(author, seq)`. A single
117+
/// process-wide counter therefore broke in two ways. Interleaved writes by
118+
/// different authors (the node itself, the tool surface) tore holes in each
119+
/// author's sequence; and because the counter always restarted at 1, a
120+
/// restart against a persistent DAG re-issued sequences the store had already
121+
/// used, so each new action silently EVICTED the pre-restart action holding
122+
/// that key — the actions survived on disk and in the chain, but dropped out
123+
/// of every author-chain view (`/dag/chain`, git provenance, approvals).
124+
///
125+
/// Allocate through [`AppState::next_dag_seq`], which seeds an author's
126+
/// counter from the store the first time it is asked.
114127
#[cfg(feature = "dag")]
115-
pub dag_seq_counter: std::sync::Arc<std::sync::atomic::AtomicU64>,
128+
pub dag_seq: std::sync::Arc<std::sync::Mutex<std::collections::HashMap<String, u64>>>,
116129
/// Ed25519 signing key for DAG actions (mandatory in production).
117130
#[cfg(feature = "dag")]
118131
pub dag_signing_key: Option<std::sync::Arc<aingle_graph::dag::DagSigningKey>>,
@@ -189,7 +202,7 @@ impl AppState {
189202
#[cfg(feature = "dag")]
190203
dag_author: None,
191204
#[cfg(feature = "dag")]
192-
dag_seq_counter: std::sync::Arc::new(std::sync::atomic::AtomicU64::new(1)),
205+
dag_seq: std::sync::Arc::new(std::sync::Mutex::new(std::collections::HashMap::new())),
193206
#[cfg(feature = "dag")]
194207
dag_signing_key: None,
195208
#[cfg(feature = "mcp")]
@@ -248,7 +261,7 @@ impl AppState {
248261
#[cfg(feature = "dag")]
249262
dag_author: None,
250263
#[cfg(feature = "dag")]
251-
dag_seq_counter: std::sync::Arc::new(std::sync::atomic::AtomicU64::new(1)),
264+
dag_seq: std::sync::Arc::new(std::sync::Mutex::new(std::collections::HashMap::new())),
252265
#[cfg(feature = "dag")]
253266
dag_signing_key: None,
254267
#[cfg(feature = "mcp")]
@@ -307,7 +320,7 @@ impl AppState {
307320
#[cfg(feature = "dag")]
308321
dag_author: None,
309322
#[cfg(feature = "dag")]
310-
dag_seq_counter: std::sync::Arc::new(std::sync::atomic::AtomicU64::new(1)),
323+
dag_seq: std::sync::Arc::new(std::sync::Mutex::new(std::collections::HashMap::new())),
311324
#[cfg(feature = "dag")]
312325
dag_signing_key: None,
313326
#[cfg(feature = "mcp")]
@@ -483,7 +496,7 @@ impl AppState {
483496
#[cfg(feature = "dag")]
484497
dag_author: None,
485498
#[cfg(feature = "dag")]
486-
dag_seq_counter: std::sync::Arc::new(std::sync::atomic::AtomicU64::new(1)),
499+
dag_seq: std::sync::Arc::new(std::sync::Mutex::new(std::collections::HashMap::new())),
487500
#[cfg(feature = "dag")]
488501
dag_signing_key: None,
489502
#[cfg(feature = "mcp")]
@@ -668,6 +681,44 @@ impl AppState {
668681
.unwrap_or_default()
669682
}
670683

684+
/// Allocate the next DAG sequence number for `author`.
685+
///
686+
/// The DAG store keys its author index on `(author, seq)`, so a sequence
687+
/// number must be unique *within its author* and must not collide with one
688+
/// already on disk. The first request for an author therefore seeds from the
689+
/// store — the author's highest recorded `seq` plus one — instead of assuming
690+
/// an empty history; every later request increments in memory. Reusing a
691+
/// number a persistent DAG already holds silently evicts the older action
692+
/// from the author index, hiding it from every chain view even though it is
693+
/// still stored and still linked.
694+
///
695+
/// `dag_store` is `None` only where the DAG is not enabled, in which case
696+
/// numbering starts at 1 because there is no history to continue.
697+
///
698+
/// A poisoned lock is recovered rather than panicked on: a mutation must not
699+
/// be lost because some unrelated writer panicked while holding this map.
700+
#[cfg(feature = "dag")]
701+
pub fn next_dag_seq(
702+
&self,
703+
author: &aingle_graph::NodeId,
704+
dag_store: Option<&aingle_graph::dag::DagStore>,
705+
) -> u64 {
706+
let key = format!("{author}");
707+
let mut map = self
708+
.dag_seq
709+
.lock()
710+
.unwrap_or_else(std::sync::PoisonError::into_inner);
711+
let seq = match map.get(&key) {
712+
Some(next) => *next,
713+
None => dag_store
714+
.and_then(|s| s.chain(author, 1).ok())
715+
.and_then(|c| c.first().map(|a| a.seq + 1))
716+
.unwrap_or(1),
717+
};
718+
map.insert(key, seq + 1);
719+
seq
720+
}
721+
671722
/// Sets the workspace working-copy root that the note-write tools resolve
672723
/// paths against and that [`crate::service::ingest`] confines every ingest
673724
/// to.

0 commit comments

Comments
 (0)