Skip to content

Commit a372a1f

Browse files
committed
perf(pipeline): prove an unchanged re-index on the live generation
Every index_repository call copied the whole existing database into a stage (sqlite3 backup) before routing, and the exact no-op then sealed that copy and renamed it back over the live file. On the Linux kernel index the copy alone was a 19 GB write per unchanged re-index. The copy is now deferred until routing needs it. The exact no-op is decided read-only on the live generation with the same checks the staged route applies to its copy (shallow integrity, format version, current coverage metadata and doc-link layer, byte-identical semantic manifest); the run then publishes nothing and only refreshes the artifact from the live database. Any other outcome copies and routes exactly as before, so every change -- including a same-size edit that keeps the original mtime -- still rebuilds from content hashes. cbm_pipeline_new no longer resolves the Git context the run start always re-resolved (seven git subprocesses per call). No-op index_repository, macOS, median of three runs of n=3: django 840.8 -> 539.0 ms, redis 450.7 -> 310.5 ms, Linux kernel 18449.5 -> 2405.0 ms. Responses and graphs are byte-identical to the baseline for the no-op and for modify/add/delete/rename edits. Signed-off-by: Martin Vogel <martin.vogel.tech@gmail.com>
1 parent 7729a8e commit a372a1f

4 files changed

Lines changed: 367 additions & 40 deletions

File tree

‎src/pipeline/pipeline.c‎

Lines changed: 119 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -260,6 +260,15 @@ struct cbm_pipeline {
260260
* "invalid_existing_db" (#1864). */
261261
bool final_existed;
262262
bool existing_generation;
263+
/* The copy into the stage is deferred until routing needs it: an
264+
* unchanged repository is proven current on the live generation
265+
* (live_db_path, read-only) and never copies, seals or renames a
266+
* database. backup_succeeded records the copy once it ran; live_noop
267+
* marks a run that ended on the live proof with nothing staged. */
268+
bool stage_backup_pending;
269+
bool backup_succeeded;
270+
bool live_noop;
271+
const char *live_db_path; /* borrowed from cbm_pipeline_run */
263272

264273
/* ADR (project_summaries) captured before a full-reindex DB delete, so it
265274
* can be restored after the rebuild. NULL when no ADR existed. Issue #516. */
@@ -848,7 +857,10 @@ cbm_pipeline_t *cbm_pipeline_new(const char *repo_path, const char *db_path,
848857
p->repo_path = strdup(repo_path);
849858
p->db_path = db_path ? strdup(db_path) : NULL;
850859
p->project_name = cbm_project_name_from_path(repo_path);
851-
(void)cbm_git_context_resolve(repo_path, &p->git_ctx);
860+
/* The Git context is resolved once, when the run starts
861+
* (pipeline_refresh_git_context): a context resolved here was always
862+
* replaced before any use, at the cost of seven git subprocesses per
863+
* index call. Until then the branch QN names the working tree. */
852864
p->branch_qn = cbm_git_context_branch_qn(p->project_name, &p->git_ctx);
853865
p->requested_mode = mode;
854866
p->mode = mode;
@@ -2738,12 +2750,69 @@ static int capture_existing_adr(cbm_pipeline_t *p, const char *db_path) {
27382750
return 0;
27392751
}
27402752

2753+
static int remove_stage_files(const char *stage_path);
2754+
2755+
/* Copy the live generation into this run's stage: every route except the
2756+
* live no-op works on that copy. A failed copy behaves as the eager copy it
2757+
* replaces did: the stage files go (the name and its lock stay this run's,
2758+
* the rebuilt generation is renamed over it later) and the run has no
2759+
* previous generation, so it rebuilds in full. */
2760+
static void materialize_stage_backup(cbm_pipeline_t *p) {
2761+
p->stage_backup_pending = false;
2762+
p->backup_succeeded = cbm_store_backup_path(p->live_db_path, p->db_path) == CBM_STORE_OK;
2763+
if (!p->backup_succeeded) {
2764+
cbm_log_warn("pipeline.stage", "action", "backup_failed_full_rebuild", "path",
2765+
p->live_db_path);
2766+
(void)remove_stage_files(p->db_path);
2767+
}
2768+
p->existing_generation = p->backup_succeeded;
2769+
}
2770+
2771+
/* The live generation is provably current: read-only, it passes the shallow
2772+
* integrity and format checks the staged route applies to its copy, and the
2773+
* exact semantic-manifest test the staged route would answer no-op with. A
2774+
* copy of it would route to that no-op and publish the same database again,
2775+
* so the run ends here with nothing copied, sealed or renamed. Any doubt
2776+
* answers false and the run copies and routes exactly as before. */
2777+
static bool live_generation_current(cbm_pipeline_t *p, const cbm_file_hash_t *baseline_manifest,
2778+
int baseline_count) {
2779+
struct stat db_st;
2780+
if (!p->live_db_path || stat(p->live_db_path, &db_st) != 0) {
2781+
return false;
2782+
}
2783+
cbm_store_t *store = cbm_store_open_path_query(p->live_db_path);
2784+
if (!store) {
2785+
return false;
2786+
}
2787+
int fmt = 0;
2788+
bool current = cbm_store_check_integrity(store);
2789+
if (current) {
2790+
cbm_store_get_format_version(store, &fmt);
2791+
current =
2792+
fmt == CBM_INDEX_FORMAT_VERSION &&
2793+
cbm_pipeline_incremental_generation_exact(p, store, baseline_manifest, baseline_count);
2794+
}
2795+
cbm_store_close(store);
2796+
return current;
2797+
}
2798+
27412799
/* Route an existing generation. Full rebuilds never delete the live DB here:
27422800
* publication owns the eventual atomic replacement after every pass and
27432801
* metadata write has succeeded. */
27442802
static int try_incremental_or_delete_db(cbm_pipeline_t *p, cbm_file_info_t *files, int file_count,
27452803
const cbm_file_hash_t *baseline_manifest,
27462804
int baseline_count, bool force_full_on_mismatch) {
2805+
if (p->stage_backup_pending) {
2806+
if (live_generation_current(p, baseline_manifest, baseline_count)) {
2807+
p->stage_backup_pending = false;
2808+
p->live_noop = true;
2809+
cbm_log_info("pipeline.route", "path", "incremental_manifest", "generation", "live");
2810+
cbm_log_info("incremental.noop", "reason", "semantic_manifest_equal", "generation",
2811+
"live");
2812+
return 0;
2813+
}
2814+
materialize_stage_backup(p);
2815+
}
27472816
if (!p->existing_generation) {
27482817
/* Nothing to be incremental against: a first index, or a
27492818
* destination that could not be copied (already reported as
@@ -2846,12 +2915,15 @@ static bool promote_mode_to_existing_coverage(cbm_pipeline_t *p) {
28462915
if (!p || !p->project_name) {
28472916
return false;
28482917
}
2849-
char *db_path = resolve_db_path(p);
2918+
/* Until routing copies it, the stage is an empty placeholder: the
2919+
* coverage the copy would carry is read from the live generation. */
2920+
char *resolved = p->stage_backup_pending ? NULL : resolve_db_path(p);
2921+
const char *db_path = p->stage_backup_pending ? p->live_db_path : resolved;
28502922
if (!db_path) {
28512923
return false;
28522924
}
28532925
cbm_store_t *store = cbm_store_open_path_query(db_path);
2854-
free(db_path);
2926+
free(resolved);
28552927
if (!store) {
28562928
return false;
28572929
}
@@ -4542,6 +4614,29 @@ static void sweep_orphan_stages(const char *final_path) {
45424614
free(dir_path);
45434615
}
45444616

4617+
/* End a run whose exact no-op was proven on the live generation. Nothing was
4618+
* staged, so nothing is sealed or renamed and the live database stays the
4619+
* current generation. The checks publication makes on the run's sources and
4620+
* destination still decide success, and the artifact is refreshed from the
4621+
* live database exactly as it would be from a republished one. */
4622+
static int finish_live_noop(cbm_pipeline_t *p, const char *final_path) {
4623+
cbm_userconfig_t *fresh_config = cbm_userconfig_load_with_source(p->repo_path);
4624+
bool source_matches = cbm_pipeline_project_source_matches(p, fresh_config);
4625+
cbm_userconfig_free(fresh_config);
4626+
struct stat final_st;
4627+
int rc = 0;
4628+
if (!source_matches) {
4629+
cbm_log_error("pipeline.err", "phase", "test_declarations_source_changed");
4630+
rc = CBM_PIPELINE_ABORT_PRESERVE_DB;
4631+
} else if (stat(final_path, &final_st) != 0) {
4632+
/* The destination vanished during the run: someone else owns it. */
4633+
rc = CBM_PIPELINE_ABORT_PRESERVE_DB;
4634+
} else {
4635+
rc = export_after_publish(p, final_path);
4636+
}
4637+
return rc;
4638+
}
4639+
45454640
int cbm_pipeline_run(cbm_pipeline_t *p) {
45464641
/* Per-index attribution: peaks and phase totals are about THIS index, not
45474642
* the process history, so they start clean here. The first mark opens
@@ -4571,21 +4666,12 @@ int cbm_pipeline_run(cbm_pipeline_t *p) {
45714666
return CBM_NOT_FOUND;
45724667
}
45734668

4574-
bool backup_succeeded = false;
4575-
if (final_existed) {
4576-
backup_succeeded = cbm_store_backup_path(final_path, staging_path) == CBM_STORE_OK;
4577-
if (!backup_succeeded) {
4578-
cbm_log_warn("pipeline.stage", "action", "backup_failed_full_rebuild", "path",
4579-
final_path);
4580-
/* The copy is gone but the NAME stays ours: the rebuilt
4581-
* generation is renamed over it by the inner finalize and then
4582-
* published from it below, so its lock is held to the end. */
4583-
remove_stage_files(staging_path);
4584-
}
4585-
}
4586-
4669+
/* The existing generation is copied into the stage only when routing
4670+
* needs a copy (materialize_stage_backup): an exact no-op is proven on
4671+
* the live generation and copies nothing -- on a kernel-scale index the
4672+
* copy alone was a 19 GB write per unchanged re-index. */
45874673
p->final_existed = final_existed;
4588-
p->existing_generation = final_existed && backup_succeeded;
4674+
p->existing_generation = final_existed;
45894675
char *configured_db_path = p->db_path;
45904676
p->db_path = strdup(staging_path);
45914677
if (!p->db_path) {
@@ -4595,10 +4681,19 @@ int cbm_pipeline_run(cbm_pipeline_t *p) {
45954681
free(final_path);
45964682
return CBM_NOT_FOUND;
45974683
}
4684+
p->stage_backup_pending = final_existed;
4685+
p->backup_succeeded = false;
4686+
p->live_noop = false;
4687+
p->live_db_path = final_path;
45984688
cbm_log_debug("pipeline.setup", "step", "staged");
45994689
int rc = cbm_pipeline_run_staged(p);
46004690
free(p->db_path);
46014691
p->db_path = configured_db_path;
4692+
bool backup_succeeded = p->backup_succeeded;
4693+
bool live_noop = p->live_noop;
4694+
p->stage_backup_pending = false;
4695+
p->live_noop = false;
4696+
p->live_db_path = NULL;
46024697

46034698
/* Report WHY the run stopped. Everything below happens before the final
46044699
* rename, so the live database is still the previous generation and an
@@ -4612,12 +4707,17 @@ int cbm_pipeline_run(cbm_pipeline_t *p) {
46124707
pipeline_release_test_config(p);
46134708
return rc;
46144709
}
4615-
if (check_cancel(p)) {
4710+
/* A cancelled run and a live no-op both end here without publishing:
4711+
* the stage placeholder is discarded, and only the no-op goes on to
4712+
* finish on the live generation. */
4713+
bool cancelled = check_cancel(p);
4714+
if (cancelled || live_noop) {
46164715
cleanup_staging_db(staging_path);
46174716
free(staging_path);
4717+
rc = cancelled ? CBM_PIPELINE_ABORT_PRESERVE_DB : finish_live_noop(p, final_path);
46184718
free(final_path);
46194719
pipeline_release_test_config(p);
4620-
return CBM_PIPELINE_ABORT_PRESERVE_DB;
4720+
return rc;
46214721
}
46224722
if (seal_staging_db(staging_path) != 0) {
46234723
cleanup_staging_db(staging_path);

‎src/pipeline/pipeline_incremental.c‎

Lines changed: 61 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -3557,6 +3557,62 @@ static bool incr_doc_links_current(cbm_store_t *store, const char *project) {
35573557
return current;
35583558
}
35593559

3560+
/* The exact no-op test: the stored generation's metadata is current for this
3561+
* mode, its doc-link layer is usable, and its recorded semantic manifest
3562+
* equals the baseline byte for byte. Read-only. The two out flags report the
3563+
* halves the staged route also needs for its own routing. */
3564+
static bool incr_generation_exact(cbm_pipeline_t *p, cbm_store_t *store, const char *project,
3565+
const cbm_file_hash_t *stored, int stored_count,
3566+
const cbm_file_hash_t *baseline_manifest, int baseline_count,
3567+
bool *metadata_current_out, bool *doc_links_current_out) {
3568+
cbm_coverage_meta_t meta = {0};
3569+
int meta_rc = cbm_store_coverage_meta_get(store, project, &meta);
3570+
const char *mode_name = incr_mode_name(cbm_pipeline_get_mode(p));
3571+
bool metadata_current =
3572+
meta_rc == CBM_STORE_OK && meta.coverage_version == CBM_SEMANTIC_INDEX_VERSION &&
3573+
meta.hash_records_complete && meta.index_mode && strcmp(meta.index_mode, mode_name) == 0;
3574+
/* A generation whose doc-link layer is missing or failed is not
3575+
* current even with identical inputs: index_status tells the user to
3576+
* re-run, and the re-run has to rebuild. */
3577+
bool doc_links_current = metadata_current && incr_doc_links_current(store, project);
3578+
bool exact = doc_links_current && cbm_pipeline_semantic_manifests_equal(
3579+
stored, stored_count, baseline_manifest, baseline_count);
3580+
cbm_store_coverage_meta_clear(&meta);
3581+
*metadata_current_out = metadata_current;
3582+
*doc_links_current_out = doc_links_current;
3583+
return exact;
3584+
}
3585+
3586+
bool cbm_pipeline_incremental_generation_exact(cbm_pipeline_t *p, cbm_store_t *store,
3587+
const cbm_file_hash_t *baseline_manifest,
3588+
int baseline_count) {
3589+
#if defined(CBM_INCREMENTAL_TEST_API) && CBM_INCREMENTAL_TEST_API
3590+
/* A forced legacy-partial run must reach the staged route that takes
3591+
* the one-shot flag; answering here would consume the run without it. */
3592+
if (atomic_load(&g_incr_test_force_legacy_partial)) {
3593+
return false;
3594+
}
3595+
#endif
3596+
const char *project = cbm_pipeline_project_name(p);
3597+
cbm_file_hash_t *stored = NULL;
3598+
int stored_count = 0;
3599+
if (!store ||
3600+
cbm_store_get_file_hashes(store, project, &stored, &stored_count) != CBM_STORE_OK) {
3601+
return false;
3602+
}
3603+
bool metadata_current = false;
3604+
bool doc_links_current = false;
3605+
bool exact = incr_generation_exact(p, store, project, stored, stored_count, baseline_manifest,
3606+
baseline_count, &metadata_current, &doc_links_current);
3607+
cbm_store_free_file_hashes(stored, stored_count);
3608+
#if defined(CBM_INCREMENTAL_TEST_API) && CBM_INCREMENTAL_TEST_API
3609+
if (exact) {
3610+
incr_test_set_last_route(CBM_INCREMENTAL_ROUTE_NOOP);
3611+
}
3612+
#endif
3613+
return exact;
3614+
}
3615+
35603616
int cbm_pipeline_run_incremental(cbm_pipeline_t *p, const char *db_path, cbm_file_info_t *files,
35613617
int file_count, const cbm_file_hash_t *baseline_manifest,
35623618
int baseline_count, bool force_full_on_mismatch) {
@@ -3602,21 +3658,11 @@ int cbm_pipeline_run_incremental(cbm_pipeline_t *p, const char *db_path, cbm_fil
36023658
* rebuilds the complete graph. This avoids partial LSP/package/config
36033659
* convergence gaps while retaining the common unchanged-repo fast path. */
36043660
if (!force_legacy_partial) {
3605-
cbm_coverage_meta_t meta = {0};
3606-
int meta_rc = cbm_store_coverage_meta_get(store, project, &meta);
3607-
const char *mode_name = incr_mode_name(cbm_pipeline_get_mode(p));
3608-
bool metadata_current = meta_rc == CBM_STORE_OK &&
3609-
meta.coverage_version == CBM_SEMANTIC_INDEX_VERSION &&
3610-
meta.hash_records_complete && meta.index_mode &&
3611-
strcmp(meta.index_mode, mode_name) == 0;
3612-
/* A generation whose doc-link layer is missing or failed is not
3613-
* current even with identical inputs: index_status tells the user to
3614-
* re-run, and the re-run has to rebuild. */
3615-
bool doc_links_current = metadata_current && incr_doc_links_current(store, project);
3616-
bool exact = doc_links_current &&
3617-
cbm_pipeline_semantic_manifests_equal(stored, stored_count, baseline_manifest,
3618-
baseline_count);
3619-
cbm_store_coverage_meta_clear(&meta);
3661+
bool metadata_current = false;
3662+
bool doc_links_current = false;
3663+
bool exact =
3664+
incr_generation_exact(p, store, project, stored, stored_count, baseline_manifest,
3665+
baseline_count, &metadata_current, &doc_links_current);
36203666
if (exact) {
36213667
cbm_store_free_file_hashes(stored, stored_count);
36223668
cbm_store_close(store);

‎src/pipeline/pipeline_internal.h‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1382,6 +1382,13 @@ int cbm_pipeline_run_incremental(cbm_pipeline_t *p, const char *db_path, cbm_fil
13821382
int file_count, const cbm_file_hash_t *baseline_manifest,
13831383
int baseline_count, bool force_full_on_mismatch);
13841384

1385+
/* The exact no-op test cbm_pipeline_run_incremental routes by, read-only on
1386+
* an already opened generation: true when that generation is current for the
1387+
* pipeline's mode and records exactly the baseline manifest. */
1388+
bool cbm_pipeline_incremental_generation_exact(cbm_pipeline_t *p, cbm_store_t *store,
1389+
const cbm_file_hash_t *baseline_manifest,
1390+
int baseline_count);
1391+
13851392
/* Exact semantic inputs for no-op/forced-full routing. The manifest contains
13861393
* every discovered source plus repository controls actually consumed by
13871394
* discovery, package mapping, path aliases, and extension overrides. */

0 commit comments

Comments
 (0)