feat: use WAL mode for the library database (#1525)

Author: CyanVoxel

src/tagstudio/core/library/alchemy/library.py

@@ -4,10 +4,12 @@
 
 import re
 import shutil
+import sqlite3
 import sys
 import time
 import unicodedata
 from collections.abc import Iterable, Iterator, Sequence
+from contextlib import closing
 from dataclasses import dataclass
 from datetime import UTC, datetime
 from os import makedirs
@@ -20,7 +22,6 @@
     URL,
     ColumnExpressionArgument,
     Engine,
-    NullPool,
     ScalarResult,
     Update,
     and_,
@@ -393,29 +394,21 @@ def open_library(self, library_dir: Path, in_memory: bool = False) -> OpenLibrar
 
     @staticmethod
     def __get_engine(library_dir: Path, in_memory: bool, sql_filename: str):
+        db_path = library_dir / TS_FOLDER_NAME / sql_filename
         connection_string = URL.create(
             drivername="sqlite",
-            database=(
-                ":memory:" if in_memory else str(library_dir / TS_FOLDER_NAME / sql_filename)
-            ),
+            database=":memory:" if in_memory else str(db_path),
         )
-        # NOTE: File-based databases should use NullPool to create new DB connection in order to
-        # keep connections on separate threads, which prevents the DB files from being locked
-        # even after a connection has been closed.
-        # SingletonThreadPool (the default for :memory:) should still be used for in-memory DBs.
-        # More info can be found on the SQLAlchemy docs:
-        # https://docs.sqlalchemy.org/en/20/changelog/migration_07.html
-        # Under -> sqlite-the-sqlite-dialect-now-uses-nullpool-for-file-based-databases
-        poolclass = None if in_memory else NullPool
 
-        logger.info(
-            "[Library] Creating SQLAlchemy Engine",
-            connection_string=connection_string,
-            poolclass=poolclass,
-        )
-        return create_engine(
-            connection_string, poolclass=poolclass, connect_args={"autocommit": False}
-        )
+        if not in_memory:
+            try:
+                with closing(sqlite3.connect(db_path, autocommit=True)) as conn:
+                    conn.execute("PRAGMA journal_mode=WAL")
+            except sqlite3.OperationalError as e:
+                logger.warning("[Library] Could not set journal mode", error=e)
+
+        logger.info("[Library] Creating SQLAlchemy Engine", connection_string=connection_string)
+        return create_engine(connection_string, connect_args={"autocommit": False})
 
     def create_sqlite_library(
         self, library_dir: Path, in_memory: bool, sql_filename: str = SQL_FILENAME
@@ -1619,17 +1612,18 @@ def save_library_backup_to_disk(library_dir: Path) -> Path:
         makedirs(str(library_dir / TS_FOLDER_NAME / BACKUP_FOLDER_NAME), exist_ok=True)
 
         filename = f"ts_library_backup_{datetime.now(UTC).strftime('%Y_%m_%d_%H%M%S')}.sqlite"
+        library_backup_path = library_dir / TS_FOLDER_NAME / BACKUP_FOLDER_NAME / filename
 
-        target_path = library_dir / TS_FOLDER_NAME / BACKUP_FOLDER_NAME / filename
-
-        shutil.copyfile(
-            library_dir / TS_FOLDER_NAME / SQL_FILENAME,
-            target_path,
-        )
+        with (
+            closing(sqlite3.connect(library_dir / TS_FOLDER_NAME / SQL_FILENAME)) as library,
+            closing(sqlite3.connect(library_backup_path)) as library_backup,
+        ):
+            library.backup(library_backup)
+            library_backup.execute("PRAGMA journal_mode=DELETE")  # Don't use WAL for backups
 
-        logger.info("Library backup saved to disk.", path=target_path)
+        logger.info("[Library] Library backup saved to disk", path=library_backup_path)
 
-        return target_path
+        return library_backup_path
 
     def get_tag(self, tag_id: int) -> Tag | None:
         with Session(self.engine) as session:

src/tagstudio/qt/mixed/migration_modal.py

@@ -438,7 +438,6 @@ def update_parity_ui(self):
         self.update_parity_value(self.aliases_row, self.alias_parity)
         self.update_parity_value(self.colors_row, self.color_parity)
         self.update_parity_value(self.ext_row, self.ext_parity)
-        self.sql_lib.close()
 
     def update_sql_value_ui(self, show_msg_box: bool = True):
         """Update the SQL value count UI."""
@@ -469,6 +468,7 @@ def update_sql_value_ui(self, show_msg_box: bool = True):
 
     def finish_migration(self):
         """Finish the migration upon user approval."""
+        self.sql_lib.close()
         final_name = self.json_lib.library_dir / TS_FOLDER_NAME / SQL_FILENAME
         if self.temp_path.exists():
             self.temp_path.rename(final_name)

tests/conftest.py

@@ -5,6 +5,7 @@
 # pyright: reportUnusedFunction=false
 
 
+import shutil
 import sys
 from collections.abc import Callable, Generator
 from pathlib import Path
@@ -22,7 +23,7 @@
 # this needs to be above `src` imports
 sys.path.insert(0, str(CWD.parent))
 
-from tagstudio.core.constants import THUMB_CACHE_NAME, TS_FOLDER_NAME
+from tagstudio.core.constants import BACKUP_FOLDER_NAME, THUMB_CACHE_NAME, TS_FOLDER_NAME
 from tagstudio.core.library.alchemy.library import Library
 from tagstudio.core.library.alchemy.models import Entry, Tag
 from tagstudio.qt.qt_driver import QtDriver
@@ -134,11 +135,17 @@ def library(request, library_dir: Path):  # pyright: ignore
 
 
 @pytest.fixture
-def search_library() -> Library:
+def search_library(tmp_path: Path) -> Generator[Library]:
+    shutil.copytree(
+        CWD / "fixtures" / "search_library" / TS_FOLDER_NAME,
+        tmp_path / TS_FOLDER_NAME,
+        ignore=shutil.ignore_patterns(BACKUP_FOLDER_NAME),
+    )
     lib = Library()
-    status = lib.open_library(Path(CWD / "fixtures" / "search_library"))
+    status = lib.open_library(tmp_path)
     assert status.success
-    return lib
+    yield lib
+    lib.close()
 
 
 @pytest.fixture