Reuse remote repository sources

Author: j178

crates/prek/src/cli/cache_gc.rs

@@ -25,6 +25,7 @@ use crate::store::{CacheBucket, REPO_MARKER, Store, ToolBucket};
 #[derive(Debug, Copy, Clone, Eq, PartialEq)]
 enum RemovalKind {
     Repos,
+    RepoSources,
     HookEnvs,
     Tools,
     CacheEntries,
@@ -36,6 +37,7 @@ impl RemovalKind {
         if count > 1 {
             match self {
                 RemovalKind::Repos => "repos",
+                RemovalKind::RepoSources => "repo sources",
                 RemovalKind::HookEnvs => "hook envs",
                 RemovalKind::Tools => "tools",
                 RemovalKind::CacheEntries => "cache entries",
@@ -44,6 +46,7 @@ impl RemovalKind {
         } else {
             match self {
                 RemovalKind::Repos => "repo",
+                RemovalKind::RepoSources => "repo source",
                 RemovalKind::HookEnvs => "hook env",
                 RemovalKind::Tools => "tool",
                 RemovalKind::CacheEntries => "cache entry",
@@ -159,6 +162,7 @@ pub(crate) async fn cache_gc(
 
     let mut kept_configs: FxHashSet<&Path> = FxHashSet::default();
     let mut used_repo_keys: FxHashSet<String> = FxHashSet::default();
+    let mut used_repo_source_keys: FxHashSet<String> = FxHashSet::default();
     let mut used_hook_env_dirs: FxHashSet<String> = FxHashSet::default();
     let mut used_tools: FxHashSet<ToolBucket> = FxHashSet::default();
     let mut used_tool_versions: FxHashMap<ToolBucket, FxHashSet<String>> = FxHashMap::default();
@@ -171,7 +175,7 @@ pub(crate) async fn cache_gc(
     let install_cache = InstallCache::new();
 
     for config_path in &tracked_configs {
-        let config = match load_config(config_path) {
+        let mut config = match load_config(config_path) {
             Ok(config) => {
                 trace!(path = %config_path.display(), "Found tracked config");
                 config
@@ -188,6 +192,11 @@ pub(crate) async fn cache_gc(
                 }
             },
         };
+        if let Err(err) = config::resolve_relative_repo_paths(&mut config, config_path) {
+            warn!(path = %config_path.display(), %err, "Failed to resolve config repo paths, skipping for GC");
+            kept_configs.insert(config_path);
+            continue;
+        }
         kept_configs.insert(config_path);
 
         used_env_requirements.extend(hook_env_requirements_from_config(store, &config));
@@ -204,6 +213,7 @@ pub(crate) async fn cache_gc(
                 if let Some(key) = key {
                     used_repo_keys.insert(key);
                 }
+                used_repo_source_keys.insert(Store::repo_source_key(&remote.repo));
             }
         }
     }
@@ -251,6 +261,15 @@ pub(crate) async fn cache_gc(
         verbose,
     )?;
 
+    // Sweep repo-sources/<hash(repo)>; the hidden lock directory is ignored by the sweeper.
+    let removed_repo_sources = sweep_dir_by_name(
+        RemovalKind::RepoSources,
+        &store.repo_sources_dir(),
+        &used_repo_source_keys,
+        dry_run,
+        verbose,
+    )?;
+
     // Sweep hooks/<hash>
     let removed_hooks = sweep_dir_by_name(
         RemovalKind::HookEnvs,
@@ -288,15 +307,20 @@ pub(crate) async fn cache_gc(
         verbose,
     )?;
 
-    // Seep scratch/, as it is only temporary data.
+    // Sweep scratch/, as it is only temporary data.
     if !dry_run {
         let _ = fs_err::remove_dir_all(store.scratch_path());
+        // `Store::init` establishes this directory as an invariant. Recreate it while the global
+        // store lock is still held so a command that was waiting for GC can immediately create
+        // clone/download temporaries with its already-initialized `Store`.
+        fs_err::create_dir_all(store.scratch_path())?;
     }
     // Keep recent recovery patches, but clear out stale ones that are unlikely to be useful.
     let removed_patches = sweep_stale_patch_files(&store.patches_dir(), dry_run, verbose)?;
 
     let mut removed = RemovalSummary::default();
     removed += &removed_repos;
+    removed += &removed_repo_sources;
     removed += &removed_hooks;
     removed += &removed_tools;
     removed += &removed_cache;
@@ -318,6 +342,7 @@ pub(crate) async fn cache_gc(
 
         if verbose {
             print_removed_details(printer, verb, &removed_repos)?;
+            print_removed_details(printer, verb, &removed_repo_sources)?;
             print_removed_details(printer, verb, &removed_hooks)?;
             print_removed_details(printer, verb, &removed_tools)?;
             print_removed_details(printer, verb, &removed_cache)?;

crates/prek/src/cli/try_repo.rs

@@ -29,77 +29,16 @@ async fn get_head_rev(repo: &Path) -> Result<String> {
     Ok(head_rev)
 }
 
-async fn clone_and_commit(repo_path: &Path, head_rev: &str, tmp_dir: &Path) -> Result<PathBuf> {
-    let shadow = tmp_dir.join("shadow-repo");
-    git::git_cmd()?
-        .arg("clone")
-        .arg(repo_path)
-        .arg(&shadow)
-        .output()
-        .await?;
-    git::git_cmd()?
-        .arg("checkout")
-        .arg(head_rev)
-        .arg("-b")
-        .arg("_prek_tmp")
-        .current_dir(&shadow)
-        .output()
-        .await?;
-
-    let index_path = shadow.join(".git/index");
-    let objects_path = shadow.join(".git/objects");
-
-    let staged_files = git::get_staged_files(repo_path).await?;
-    if !staged_files.is_empty() {
-        git::git_cmd()?
-            .arg("add")
-            .arg("--")
-            .file_args(&staged_files)
-            .current_dir(repo_path)
-            .env("GIT_INDEX_FILE", &index_path)
-            .env("GIT_OBJECT_DIRECTORY", &objects_path)
-            .output()
-            .await?;
-    }
-
-    let mut add_u_cmd = git::git_cmd()?;
-    add_u_cmd
-        .arg("add")
-        .arg("--update") // Update tracked files
-        .current_dir(repo_path)
-        .env("GIT_INDEX_FILE", &index_path)
-        .env("GIT_OBJECT_DIRECTORY", &objects_path)
-        .output()
-        .await?;
-
-    git::git_cmd()?
-        .arg("commit")
-        .arg("-m")
-        .arg("Temporary commit by prek try-repo")
-        .arg("--no-gpg-sign")
-        .arg("--no-edit")
-        .arg("--no-verify")
-        .current_dir(&shadow)
-        .env("GIT_AUTHOR_NAME", "prek test")
-        .env("GIT_AUTHOR_EMAIL", "test@example.com")
-        .env("GIT_COMMITTER_NAME", "prek test")
-        .env("GIT_COMMITTER_EMAIL", "test@example.com")
-        .output()
-        .await?;
-
-    Ok(shadow)
-}
-
 struct PreparedRepo<'a> {
     runtime_source: Cow<'a, str>,
     display_source: Option<&'a str>,
     rev: String,
 }
 
 async fn prepare_repo<'a>(
+    store: &Store,
     repo: &'a str,
     rev: Option<&str>,
-    tmp_dir: &Path,
 ) -> Result<PreparedRepo<'a>> {
     let repo_path = Path::new(repo);
     let is_local = repo_path.is_dir();
@@ -143,15 +82,23 @@ async fn prepare_repo<'a>(
             .to_string()
     };
 
-    // If repo is a local repo with uncommitted changes, create a shadow repo to commit the changes.
+    // Persist a deterministic synthetic commit in the shared source. The logical source remains
+    // the canonical local path, so identical dirty trees get the same repo and environment keys.
     if is_local && git::has_diff("HEAD", repo_path).await? {
         warn_user!("Creating temporary repo with uncommitted changes...");
-        let shadow = clone_and_commit(repo_path, &head_rev, tmp_dir).await?;
-        let head_rev = get_head_rev(&shadow).await?;
+        let source = store.repo_source_path(runtime_source.as_ref());
+        let _source_lock = store.repo_source_lock(runtime_source.as_ref()).await?;
+        git::ensure_bare_repo(runtime_source.as_ref(), &source).await?;
+        let head_rev =
+            git::fetch_repo_source_revision(&source, &head_rev, git::TerminalPrompt::Disabled)
+                .await?;
+        let snapshot =
+            git::create_repo_snapshot(&source, repo_path, head_rev.commit(), &store.scratch_path())
+                .await?;
         Ok(PreparedRepo {
-            runtime_source: Cow::Owned(shadow.to_string_lossy().into_owned()),
-            display_source: None,
-            rev: head_rev,
+            runtime_source,
+            display_source: Some(repo),
+            rev: snapshot,
         })
     } else {
         Ok(PreparedRepo {
@@ -202,38 +149,45 @@ pub(crate) async fn try_repo(
     }
 
     let store = Store::from_settings()?;
-    let tmp_dir = TempDir::with_prefix_in("try-repo-", store.scratch_path())?;
-
-    let prepared = prepare_repo(&repo, rev.as_deref(), tmp_dir.path())
-        .await
-        .context("Failed to determine repository and revision")?;
-
-    let store = Store::from_path(tmp_dir.path()).init()?;
-    let repo_config = config::RemoteRepo::new(
-        prepared.runtime_source.to_string(),
-        prepared.rev.clone(),
-        vec![],
-    );
-    let repo_clone_path = store.clone_repo(&repo_config, None).await?;
-
     let selectors = Selectors::load(&run_args.includes, &run_args.skips, GIT_ROOT.as_ref()?)?;
 
-    let manifest =
-        config::read_manifest(&repo_clone_path.join(prek_consts::PRE_COMMIT_HOOKS_YAML))?;
-
-    let hooks = manifest
-        .hooks
-        .into_iter()
-        .filter(|hook| selectors.matches_hook_id(&hook.id))
-        .map(|hook| hook.id)
-        .collect::<Vec<_>>();
+    let (_tmp_dir, prepared, hooks, config_str, config_file) = {
+        let _lock = store.lock_async().await?;
+        // `cache gc` clears the store scratch directory after taking the store lock. Keep the
+        // active generated config in the system temporary directory so it survives between this
+        // preparation lock and the lock acquired by `run`.
+        let tmp_dir = TempDir::with_prefix("try-repo-")?;
+        let prepared = prepare_repo(&store, &repo, rev.as_deref())
+            .await
+            .context("Failed to determine repository and revision")?;
+        let repo_config = config::RemoteRepo::new(
+            prepared.runtime_source.to_string(),
+            prepared.rev.clone(),
+            vec![],
+        );
+        let repo_clone_path = store.clone_repo(&repo_config, None).await?;
+        let manifest =
+            config::read_manifest(&repo_clone_path.join(prek_consts::PRE_COMMIT_HOOKS_YAML))?;
+        let hooks = manifest
+            .hooks
+            .into_iter()
+            .filter(|hook| selectors.matches_hook_id(&hook.id))
+            .map(|hook| hook.id)
+            .collect::<Vec<_>>();
+
+        let config_str = render_repo_config_toml(&prepared.runtime_source, &prepared.rev, &hooks);
+        let config_file = tmp_dir.path().join(PREK_TOML);
+        fs_err::tokio::write(&config_file, &config_str).await?;
+        // Make the new source/checkout visible to GC before releasing the store lock. `run` also
+        // tracks this path, but a concurrent `cache gc` must not sweep a dirty synthetic commit in
+        // the interval between preparation and hook initialization.
+        store.track_configs(std::iter::once(config_file.as_path()))?;
+
+        (tmp_dir, prepared, hooks, config_str, config_file)
+    };
 
     // The scratch config needs the resolved source, while the displayed config should preserve
     // the user's path so it remains meaningful when copied into their project.
-    let config_str = render_repo_config_toml(&prepared.runtime_source, &prepared.rev, &hooks);
-    let config_file = tmp_dir.path().join(PREK_TOML);
-    fs_err::tokio::write(&config_file, &config_str).await?;
-
     let display_config_str = match prepared.display_source {
         Some(source) => Cow::Owned(render_repo_config_toml(source, &prepared.rev, &hooks)),
         None => Cow::Borrowed(config_str.as_str()),

crates/prek/src/cli/update/mod.rs

@@ -386,10 +386,20 @@ pub(crate) async fn update(
     filesystem: Option<FilesystemOptions>,
     printer: Printer,
 ) -> Result<ExitStatus> {
+    // Repository sources are shared with normal hook initialization and try-repo. Keep the
+    // command-level store invariant while per-source locks below serialize individual fetches.
+    let _store_lock = store.lock_async().await?;
+
     let workspace_root = Workspace::find_root(config.as_deref(), &CWD)?;
     // TODO: support selectors?
     let selectors = Selectors::default();
     let workspace = Workspace::discover(store, workspace_root, config, Some(&selectors), true)?;
+    store.track_configs(
+        workspace
+            .projects()
+            .iter()
+            .map(|project| project.config_file()),
+    )?;
 
     let tag_filters =
         TagFilters::new(include_tag, exclude_tag, repo_include_tag, repo_exclude_tag)?;
@@ -409,7 +419,7 @@ pub(crate) async fn update(
         .map(async |repo_source| {
             let progress = reporter.on_update_start(repo_source.repo);
             let result =
-                evaluate_repo_source(repo_source, bleeding_edge, freeze, &tag_filters).await;
+                evaluate_repo_source(store, repo_source, bleeding_edge, freeze, &tag_filters).await;
             reporter.on_update_complete(progress);
             result
         })