docs(cargo-codspeed): clarify BENCHNAME and --bench help (#195)

* docs(cargo-codspeed): clarify BENCHNAME and --bench help

The BENCHNAME help claimed a substring match on benchmark names. It is an unanchored regex forwarded to each bench binary, and the matched string differs by mode: the CodSpeed URI in simulation/memory, the framework's own name in walltime. Document that, how to use --exact, and that an empty selection still exits successfully.

The --bench help now mentions it can be repeated.

* fix(criterion): send benchmark markers around iter_manual rounds

Author: not-matthias

crates/cargo-codspeed/src/app.rs

@@ -133,7 +133,7 @@ pub(crate) struct PackageFilters {
 
 #[derive(Args, Clone)]
 pub(crate) struct BenchTargetFilters {
-    /// Select only the specified benchmark target (all benchmark targets by default)
+    /// Select only the specified benchmark target; can be repeated (all benchmark targets by default)
     #[arg(long, help_heading = TARGET_HELP)]
     pub(crate) bench: Option<Vec<String>>,
 }
@@ -200,7 +200,16 @@ enum Commands {
     },
     /// Run the previously built benchmarks
     Run {
-        /// If specified, only run benches containing this string in their names
+        /// Only run benchmarks whose name matches this unanchored regex.
+        ///
+        /// The string matched against depends on the measurement mode:
+        /// - **simulation/memory**: the CodSpeed URI
+        /// - **walltime**: the benchmark framework's own name (can be listed with `-- --list`,
+        ///   and validated with `-- --list '<filter>'`)
+        ///
+        /// To require the whole name to match instead of a regex, pass `-- --exact`. The name format
+        /// still differs between modes.
+        #[arg(verbatim_doc_comment)]
         benchname: Option<String>,
 
         #[command(flatten)]

crates/codspeed/src/instrument_hooks/mod.rs

@@ -279,6 +279,24 @@ pub use linux_impl::InstrumentHooks;
 #[cfg(not(use_instrument_hooks))]
 pub use other_impl::InstrumentHooks;
 
+/// Sends the benchmark start marker when created and the stop marker when
+/// dropped, so every exit path closes the measured region.
+pub struct BenchmarkMarkers(&'static InstrumentHooks);
+
+impl BenchmarkMarkers {
+    pub fn start() -> Self {
+        let hooks = InstrumentHooks::instance();
+        let _ = hooks.start_benchmark();
+        Self(hooks)
+    }
+}
+
+impl Drop for BenchmarkMarkers {
+    fn drop(&mut self) {
+        let _ = self.0.stop_benchmark();
+    }
+}
+
 #[cfg(test)]
 mod tests {
     use super::InstrumentHooks;

crates/criterion_compat/criterion_fork/src/codspeed_iter_manual.rs

@@ -6,7 +6,7 @@
 
 use std::time::{Duration, Instant};
 
-use codspeed::instrument_hooks::InstrumentHooks;
+use codspeed::instrument_hooks::{BenchmarkMarkers, InstrumentHooks};
 
 #[cfg(feature = "async")]
 use crate::async_executor::AsyncExecutor;
@@ -112,6 +112,7 @@ impl<'a, M: Measurement> Bencher<'a, M> {
 
         self.elapsed_time = Duration::ZERO;
         let mut samples = Vec::with_capacity(opts.rounds as usize);
+        let markers = BenchmarkMarkers::start();
         for _ in 0..opts.rounds {
             let bench_start = InstrumentHooks::current_timestamp();
             let round_start = Instant::now();
@@ -126,6 +127,7 @@ impl<'a, M: Measurement> Bencher<'a, M> {
 
             samples.push(self.measurement.to_f64(&value));
         }
+        drop(markers);
 
         self.codspeed_manual = Some(ManualMeasurement {
             samples,
@@ -172,6 +174,7 @@ impl<'a, 'b, A: AsyncExecutor, M: Measurement> AsyncBencher<'a, 'b, A, M> {
 
             b.elapsed_time = Duration::ZERO;
             let mut samples = Vec::with_capacity(opts.rounds as usize);
+            let markers = BenchmarkMarkers::start();
             for _ in 0..opts.rounds {
                 let bench_start = InstrumentHooks::current_timestamp();
                 let round_start = Instant::now();
@@ -186,6 +189,7 @@ impl<'a, 'b, A: AsyncExecutor, M: Measurement> AsyncBencher<'a, 'b, A, M> {
 
                 samples.push(b.measurement.to_f64(&value));
             }
+            drop(markers);
 
             b.codspeed_manual = Some(ManualMeasurement {
                 samples,

crates/criterion_compat/criterion_fork/src/routine.rs

@@ -1,4 +1,4 @@
-use codspeed::instrument_hooks::InstrumentHooks;
+use codspeed::instrument_hooks::BenchmarkMarkers;
 
 use crate::benchmark::BenchmarkConfig;
 use crate::codspeed_iter_manual::ManualMeasurement;
@@ -238,13 +238,8 @@ pub(crate) trait Routine<M: Measurement, T: ?Sized> {
         }
 
         let m_elapsed = {
-            let hooks = InstrumentHooks::instance();
-
-            let _ = hooks.start_benchmark();
-            let value = self.bench(measurement, &m_iters, parameter);
-            let _ = hooks.stop_benchmark();
-
-            value
+            let _markers = BenchmarkMarkers::start();
+            self.bench(measurement, &m_iters, parameter)
         };
         let m_iters_f: Vec<f64> = m_iters.iter().map(|&x| x as f64).collect();