limit number of file descriptors opened by FileSystemStorage

Problem: when loading a large number of documents it was possible for
the file system storage backend to exhaust the available file
descriptors.

Solution: limit the number of files opened at once to (somewhat
arbitrarily) 500.

Author: LilithSilver

samod/src/storage/filesystem.rs

@@ -20,8 +20,11 @@ pub mod tokio {
     use std::{
         collections::HashMap,
         path::{Path, PathBuf},
+        sync::Arc,
     };
 
+    use tokio::sync::Semaphore;
+
     use crate::storage::Storage;
 
     use super::key_to_path;
@@ -32,12 +35,14 @@ pub mod tokio {
     #[derive(Clone, Debug)]
     pub struct FilesystemStorage {
         data_dir: PathBuf,
+        read_limit: Arc<Semaphore>,
     }
 
     impl FilesystemStorage {
         pub fn new<P: AsRef<Path>>(data_dir: P) -> Self {
             Self {
                 data_dir: data_dir.as_ref().to_path_buf(),
+                read_limit: Arc::new(Semaphore::new(500)),
             }
         }
 
@@ -80,6 +85,7 @@ pub mod tokio {
                 // Now walk the directory, recursively
                 let mut to_visit = vec![(path, prefix)];
                 while let Some((next_path, key_prefix)) = to_visit.pop() {
+                    let permit = self.read_limit.acquire().await;
                     let Ok(mut entries) = tokio::fs::read_dir(&next_path).await else {
                         continue;
                     };
@@ -109,6 +115,7 @@ pub mod tokio {
                             result.insert(next_key_prefix, data);
                         }
                     }
+                    drop(permit);
                 }
                 result
             }