Skip to content

Commit 8f39961

Browse files
authored
feat(core): run the scan walk at the configured io_priority (#173)
Adopts #170's priority mechanism in #169's parallel scanner - the follow-up called out in #170's PR body ('the broad user-visible win lands when the scanner's dedicated worker threads call apply_to_current_thread'). - The walk's pooled spawn_blocking coordinator takes an RAII begin_background_work guard, so the matcher build's ignore-file collection I/O is shaped too and the pooled thread is restored on scan end. - Each transient walk worker thread (stat + include check + BLAKE3 hashing for deep-verify/coarse-FS) applies the priority once on its first visit; workers are joined at walk end, so a one-shot apply with no restore is correct. - New scan_with_priority entry point; scan/scan_with_latency/scan_with_progress keep their exact signatures (Normal), so no test churn. The orchestrator passes its live PriorityCell value - settings changes apply from the next scan. A sibling PR (in flight) extends shaping to the large-file upload pipeline. Gates: cargo fmt, clippy -p driven-core -D warnings clean, cargo test -p driven-core 393+ green. 🤖 Generated with [Claude Code](https://claude.com/claude-code) https://claude.ai/code/session_01JLB3E2Jm7knNJd37fVpH8X
1 parent a587fed commit 8f39961

2 files changed

Lines changed: 54 additions & 1 deletion

File tree

crates/driven-core/src/orchestrator.rs

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1565,12 +1565,17 @@ impl SyncOrchestrator {
15651565
scanned_total.store(scanned, std::sync::atomic::Ordering::Relaxed);
15661566
Box::pin(self.transition(OrchestratorState::Scanning { source_id, scanned }))
15671567
};
1568-
crate::scanner::scan_with_progress(
1568+
crate::scanner::scan_with_priority(
15691569
source,
15701570
self.state.as_ref(),
15711571
mode,
15721572
self.latency.as_deref(),
15731573
Some(&on_scan_progress),
1574+
// Live SPEC s22 io_priority: the walk's coordinator + worker
1575+
// threads demote themselves so scan stat/hash I/O yields to
1576+
// foreground apps (the same cell the executor's bundle path
1577+
// reads).
1578+
self.priority.get(),
15741579
)
15751580
.await?
15761581
};

crates/driven-core/src/scanner.rs

Lines changed: 48 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -65,6 +65,7 @@ use anyhow::Context;
6565
use ignore::{DirEntry, ParallelVisitor, ParallelVisitorBuilder, WalkState};
6666

6767
use crate::exclude::{build_source_matcher, build_walker_with_matcher, SourceMatcher};
68+
use crate::priority::WorkPriority;
6869
use crate::state::{FileStateRow, SourceRow, StateRepo};
6970
// `PlaceholderPolicy` is consumed only by `should_skip_placeholder`, which is
7071
// cfg-gated the same way; on a non-Windows, non-test lib build the fn (and thus
@@ -310,6 +311,12 @@ struct WalkCtx {
310311
/// Whether to time each file (the reservoir itself lives on the consumer
311312
/// side, which is what actually records the sample).
312313
capture_latency: bool,
314+
/// The backup-work priority (SPEC s22 `io_priority`) each walk worker
315+
/// thread applies to itself on its first visit, so the scan's stat + hash
316+
/// I/O yields to foreground applications. Walker threads live only for the
317+
/// walk (spawned and joined inside `build_parallel().visit()`), so a
318+
/// one-shot apply with no restore is correct - nothing pooled is demoted.
319+
priority: WorkPriority,
313320
}
314321

315322
/// What a worker concluded about ONE file. Deliberately excludes every
@@ -372,6 +379,10 @@ struct WalkVisitor {
372379
/// Set once the consumer has gone away, so the walk stops instead of
373380
/// traversing the rest of the tree with nowhere to send the results.
374381
consumer_gone: bool,
382+
/// Whether this worker thread has applied [`WalkCtx::priority`] to itself
383+
/// yet. `build()` may run on the coordinating thread, so the apply happens
384+
/// on the first `visit()` call, which is guaranteed to run on the worker.
385+
priority_applied: bool,
375386
}
376387

377388
impl WalkVisitor {
@@ -423,6 +434,10 @@ impl Drop for WalkVisitor {
423434

424435
impl ParallelVisitor for WalkVisitor {
425436
fn visit(&mut self, result: Result<DirEntry, ignore::Error>) -> WalkState {
437+
if !self.priority_applied {
438+
self.priority_applied = true;
439+
crate::priority::apply_to_current_thread(self.ctx.priority);
440+
}
426441
let ctx = Arc::clone(&self.ctx);
427442
let entry = match result {
428443
Ok(e) => e,
@@ -517,6 +532,7 @@ impl<'s> ParallelVisitorBuilder<'s> for WalkVisitorBuilder {
517532
items: 0,
518533
last_flush: Instant::now(),
519534
consumer_gone: false,
535+
priority_applied: false,
520536
})
521537
}
522538
}
@@ -667,6 +683,33 @@ pub async fn scan_with_progress(
667683
mode: ScanMode,
668684
latency: Option<&crate::telemetry::LatencyReservoir>,
669685
on_progress: Option<&ScanProgressSink<'_>>,
686+
) -> anyhow::Result<ScanResult> {
687+
scan_with_priority(
688+
source,
689+
state,
690+
mode,
691+
latency,
692+
on_progress,
693+
WorkPriority::Normal,
694+
)
695+
.await
696+
}
697+
698+
/// [`scan_with_progress`] plus the backup-work [`WorkPriority`] (SPEC s22
699+
/// `io_priority`) the walk should run at. The blocking coordinator task takes an
700+
/// RAII [`begin_background_work`](crate::priority::begin_background_work) guard
701+
/// (its thread is POOLED, so the demotion must be undone), and each transient
702+
/// walk worker thread applies the priority once on its first visit (no restore
703+
/// needed - workers die with the walk). `Normal` is exactly the old behaviour;
704+
/// the orchestrator passes its live [`PriorityCell`](crate::priority::PriorityCell)
705+
/// value so a settings change applies from the next scan.
706+
pub async fn scan_with_priority(
707+
source: &SourceRow,
708+
state: &dyn StateRepo,
709+
mode: ScanMode,
710+
latency: Option<&crate::telemetry::LatencyReservoir>,
711+
on_progress: Option<&ScanProgressSink<'_>>,
712+
priority: WorkPriority,
670713
) -> anyhow::Result<ScanResult> {
671714
let known = state
672715
.load_source_file_state(source.id)
@@ -799,6 +842,10 @@ pub async fn scan_with_progress(
799842
let walk_known = Arc::clone(&known);
800843
let capture_latency = latency.is_some_and(|r| r.is_enabled());
801844
let walk_task = tokio::task::spawn_blocking(move || -> anyhow::Result<Arc<SourceMatcher>> {
845+
// This spawn_blocking thread is POOLED, so the demotion is held in an
846+
// RAII guard that restores normal priority when the walk ends; the
847+
// matcher build's ignore-file collection I/O runs demoted too.
848+
let _priority_guard = crate::priority::begin_background_work(priority);
802849
// ONE matcher for the whole scan: the walker's prune predicate, every
803850
// per-entry include check, and the orphan split share this `Arc` rather
804851
// than each rebuilding the cascade (the walker used to build a second,
@@ -814,6 +861,7 @@ pub async fn scan_with_progress(
814861
is_coarse,
815862
last_scan_end_ns,
816863
capture_latency,
864+
priority,
817865
});
818866
let mut wb = build_walker_with_matcher(&walk_source, Arc::clone(&matcher));
819867
wb.threads(walk_threads());

0 commit comments

Comments
 (0)