Merge branch 'main' into checksum-split-awareness

Author: crocodilestick

CONTRIBUTORS

@@ -313,8 +313,7 @@ Copyright (C) 2024-2026 Calibre-Web Automated contributors
 - zelazna (1 commits)
 - zhiyue (1 commits)
 # Fork Contributors (crocodilestick/calibre-web-automated)
-
-- crocodilestick (930 commits)
+- crocodilestick (931 commits)
 - jmarmstrong1207 (73 commits)
 - demitrix (30 commits)
 - sirwolfgang (29 commits)
@@ -401,6 +400,7 @@ Copyright (C) 2024-2026 Calibre-Web Automated contributors
 - spezzino (1 commits)
 - stadler-pascal (1 commits)
 - stefanop1 (1 commits)
+- thetorminal (1 commits)
 - tmacphail (1 commits)
 - tomried (1 commits)
 - Turmaxx (1 commits)

README.md

@@ -46,11 +46,14 @@ Calibre-Web Automated aims to be an all-in-one solution, combining the modern li
 
 ## _Affiliated Projects_ 👬
 
-### Calibre-Web Automated Book Downloader
+### Shelfmark: Book Downloader
 
 - An intuitive web interface for searching and requesting book downloads, designed to work seamlessly with Calibre-Web-Automated. This project streamlines the process of downloading books and preparing them for integration into your Calibre library
 
-[<img src="https://raw.githubusercontent.com/vadret/android/master/assets/get-github.png" alt="Get it on GitHub" height="80">](https://github.com/calibrain/calibre-web-automated-book-downloader)
+> [!IMPORTANT]  
+> CWA does not approve of or support piracy of copyrighted materials and is not responsible for user behaviour
+
+[<img src="https://raw.githubusercontent.com/vadret/android/master/assets/get-github.png" alt="Get it on GitHub" height="80">](https://github.com/calibrain/shelfmark)
 
 ___
 
@@ -142,7 +145,7 @@ This tells CWA to avoid enabling WAL on the Calibre `metadata.db` and the `app.d
   - Using the information provided in the Calibre eBook-converter documentation on which formats convert best into epubs, CWA is able to determine from downloads containing multiple eBook formats, which format will convert most optimally, ignoring the other formats to ensure the **best possible quality** and no **duplicate imports** -->
 
 #### **Automatic Conversion Service** 🔃
-- On by default though can be toggled of in the CWA Settings page, with EPUB as the default target format
+- On by default though can be toggled off in the CWA Settings page, with EPUB as the default target format
   - _Available target formats include:_ **EPUB**, **MOBI**, **AZW3**, **KEPUB** & **PDF**
 - Upon detecting new files in the Ingest Directory, if any of the files are in formats the user has configured CWA to auto-convert to the current target format,
 - The following **28 file types are currently supported:**

cps/__init__.py

@@ -244,10 +244,28 @@ def create_app():
     # Ensure a valid calibre_db session exists before handling each request
     @app.before_request
     def _cwa_ensure_db_session():
+        from flask import g, request
         from .cw_login import current_user
         from sqlalchemy import or_
         import time
 
+        if config.config_allow_reverse_proxy_header_login:
+            """
+            Load user from reverse proxy authentication header if configured.
+            Sets g.flask_httpauth_user early so that current_user proxy resolves correctly
+            for user-specific settings like theme preferences.
+
+            This must run before any blueprint before_request handlers that access current_user.
+            """
+
+            from . import usermanagement
+            user = usermanagement.load_user_from_reverse_proxy_header(request)
+            if user:
+                g.flask_httpauth_user = user
+            else:
+                # Explicitly set to None to indicate we checked but found nothing
+                g.flask_httpauth_user = None
+
         if current_user.is_authenticated:
             try:
                 # Verify required tables exist before querying
@@ -358,28 +376,6 @@ def shutdown_session(exception=None):
         if calibre_db.session_factory:
             calibre_db.session_factory.remove()
 
-    # Load user from reverse proxy header early in request lifecycle
-    # This ensures current_user resolves correctly before any code accesses user settings
-    @app.before_request
-    def _load_reverse_proxy_user():
-        """
-        Load user from reverse proxy authentication header if configured.
-        Sets g.flask_httpauth_user early so that current_user proxy resolves correctly
-        for user-specific settings like theme preferences.
-
-        This must run before any blueprint before_request handlers that access current_user.
-        """
-        from flask import g, request
-
-        if config.config_allow_reverse_proxy_header_login:
-            from . import usermanagement
-            user = usermanagement.load_user_from_reverse_proxy_header(request)
-            if user:
-                g.flask_httpauth_user = user
-            else:
-                # Explicitly set to None to indicate we checked but found nothing
-                g.flask_httpauth_user = None
-
     from .schedule import register_scheduled_tasks, register_startup_tasks
     register_scheduled_tasks(config.schedule_reconnect)
     register_startup_tasks()

cps/cwa_functions.py

@@ -19,7 +19,7 @@
 from time import sleep
 
 import json
-from threading import Thread
+from threading import Thread, Lock, Timer
 import queue
 import os
 import tempfile
@@ -59,6 +59,10 @@
 LOG_ARCHIVE = "/config/log_archive"
 DIRS_JSON = "/app/calibre-web-automated/dirs.json"
 
+# Debounced duplicate scan timer (web process)
+_duplicate_scan_timer = None
+_duplicate_scan_lock = Lock()
+
 ##———————————————————————————END OF GLOBAL VARIABLES——————————————————————————##
 
 ##————————————————————————————————————————————————————————————————————————————##
@@ -346,6 +350,57 @@ def _enqueue_autosend():
         log.error(f"Internal auto-send schedule failed: {e}")
         return jsonify({"error": str(e)}), 400
 
+
+@csrf.exempt
+@cwa_internal.route('/cwa-internal/queue-duplicate-scan', methods=["POST"])
+def cwa_internal_queue_duplicate_scan():
+    """Debounce and queue an incremental duplicate scan in the web process.
+
+    Security: Limited to localhost callers (within container/host).
+    Payload JSON: {delay_seconds:int}
+    """
+    try:
+        remote = request.headers.get('X-Forwarded-For', request.remote_addr)
+        if remote not in (None, '127.0.0.1', '::1'):
+            abort(403)
+
+        db = CWA_DB()
+        enabled = bool(db.cwa_settings.get('duplicate_scan_enabled', 0))
+        frequency = db.cwa_settings.get('duplicate_scan_frequency', 'manual')
+
+        data = request.get_json(force=True, silent=True) or {}
+        default_delay = db.cwa_settings.get('duplicate_scan_debounce_seconds', 30)
+        delay_seconds = int(data.get('delay_seconds', default_delay))
+        delay_seconds = max(5, min(600, delay_seconds))
+
+        if not enabled or frequency != 'after_import':
+            return jsonify({"success": True, "skipped": True, "reason": "disabled_or_manual"}), 200
+
+        global _duplicate_scan_timer
+        with _duplicate_scan_lock:
+            if _duplicate_scan_timer is not None:
+                try:
+                    _duplicate_scan_timer.cancel()
+                except Exception:
+                    pass
+
+            def _enqueue_scan():
+                try:
+                    from .tasks.duplicate_scan import TaskDuplicateScan
+                    WorkerThread.add('System', TaskDuplicateScan(full_scan=False, trigger_type='after_import'), hidden=False)
+                    log.info("[cwa-duplicates] Debounced duplicate scan queued (after_import)")
+                except Exception as e:
+                    log.error("[cwa-duplicates] Failed to queue debounced duplicate scan: %s", str(e))
+
+            _duplicate_scan_timer = Timer(delay_seconds, _enqueue_scan)
+            _duplicate_scan_timer.daemon = True
+            _duplicate_scan_timer.start()
+
+        return jsonify({"success": True, "queued": True, "delay_seconds": delay_seconds}), 200
+    except Exception as e:
+        log.error("[cwa-duplicates] Failed to schedule debounced duplicate scan: %s", str(e))
+        return jsonify({"success": False, "error": str(e)}), 500
+
 @csrf.exempt
 @cwa_internal.route('/cwa-internal/schedule-convert-library', methods=["POST"])
 def cwa_internal_schedule_convert_library():
@@ -554,8 +609,8 @@ def set_cwa_settings():
     boolean_settings = []
     string_settings = []
     list_settings = []
-    integer_settings = ['ingest_timeout_minutes', 'auto_send_delay_minutes']  # Special handling for integer settings
-    json_settings = ['metadata_provider_hierarchy', 'metadata_providers_enabled']  # Special handling for JSON settings
+    integer_settings = ['ingest_timeout_minutes', 'auto_send_delay_minutes', 'duplicate_scan_hour', 'duplicate_scan_chunk_size', 'duplicate_scan_debounce_seconds']  # Special handling for integer settings
+    json_settings = ['metadata_provider_hierarchy', 'metadata_providers_enabled', 'duplicate_format_priority']  # Special handling for JSON settings
 
     for setting in cwa_default_settings:
         if setting in integer_settings or setting in json_settings:
@@ -567,6 +622,10 @@ def set_cwa_settings():
         else:
             list_settings.append(setting)
 
+    # Ensure cron expression is treated as a string even if default is empty
+    if 'duplicate_scan_cron' not in string_settings:
+        string_settings.append('duplicate_scan_cron')
+
     for format in ignorable_formats:
         string_settings.append(f"ignore_ingest_{format}")
         string_settings.append(f"ignore_convert_{format}")
@@ -631,18 +690,28 @@ def set_cwa_settings():
                             int_value = max(5, min(120, int_value))  # Clamp between 5 and 120 minutes
                         elif setting == 'auto_send_delay_minutes':
                             int_value = max(1, min(60, int_value))  # Clamp between 1 and 60 minutes
+                        elif setting == 'duplicate_scan_hour':
+                            int_value = max(0, min(23, int_value))
+                        elif setting == 'duplicate_scan_chunk_size':
+                            int_value = max(500, min(50000, int_value))
+                        elif setting == 'duplicate_scan_debounce_seconds':
+                            int_value = max(5, min(600, int_value))
                         result[setting] = int_value
                     except (ValueError, TypeError):
                         # Use current value if conversion fails
                         if setting == 'ingest_timeout_minutes':
                             result[setting] = cwa_db.cwa_settings.get(setting, 15)  # Default to 15 minutes
                         elif setting == 'auto_send_delay_minutes':
                             result[setting] = cwa_db.cwa_settings.get(setting, 5)  # Default to 5 minutes
+                        elif setting == 'duplicate_scan_debounce_seconds':
+                            result[setting] = cwa_db.cwa_settings.get(setting, 30)
                 else:
                     if setting == 'ingest_timeout_minutes':
                         result[setting] = cwa_db.cwa_settings.get(setting, 15)  # Default to 15 minutes
                     elif setting == 'auto_send_delay_minutes':
                         result[setting] = cwa_db.cwa_settings.get(setting, 5)  # Default to 5 minutes
+                    elif setting == 'duplicate_scan_debounce_seconds':
+                        result[setting] = cwa_db.cwa_settings.get(setting, 30)
 
             # Handle JSON settings
             for setting in json_settings:
@@ -689,6 +758,17 @@ def set_cwa_settings():
                     else:
                         result[setting] = cwa_db.cwa_settings.get(setting, '[]')
 
+            # Validate cron expression if provided
+            cron_expr = result.get('duplicate_scan_cron', '')
+            if cron_expr:
+                try:
+                    from apscheduler.triggers.cron import CronTrigger
+                    CronTrigger.from_crontab(cron_expr)
+                except Exception:
+                    # Revert to previous value and notify user
+                    result['duplicate_scan_cron'] = cwa_db.cwa_settings.get('duplicate_scan_cron', '')
+                    flash(_("Invalid cron expression for duplicate scans. Changes were not saved."), category="error")
+
             # DEBUGGING
             # with open("/config/post_request" ,"w") as f:
             #     for key in result.keys():
@@ -713,9 +793,33 @@ def set_cwa_settings():
         cwa_db = CWA_DB()
         cwa_settings = cwa_db.get_cwa_settings()
 
+    next_scan_run = get_next_duplicate_scan_run(cwa_settings)
+
     return render_title_template("cwa_settings.html", title=_("Calibre-Web Automated User Settings"), page="cwa-settings",
                                     cwa_settings=cwa_settings, ignorable_formats=ignorable_formats, target_formats=target_formats,
-                                    automerge_options=automerge_options, autoingest_options=autoingest_options)
+                                    automerge_options=automerge_options, autoingest_options=autoingest_options,
+                                    next_duplicate_scan_run=next_scan_run)
+
+
+def get_next_duplicate_scan_run(settings):
+    """Compute next scheduled duplicate scan run time based on settings."""
+    try:
+        enabled = bool(settings.get('duplicate_scan_enabled', 0))
+        cron_expr = (settings.get('duplicate_scan_cron') or '').strip()
+
+        if not enabled:
+            return None
+
+        if not cron_expr:
+            return None
+
+        from apscheduler.triggers.cron import CronTrigger
+        now = datetime.now().astimezone()
+        trigger = CronTrigger.from_crontab(cron_expr, timezone=now.tzinfo)
+        next_run = trigger.get_next_fire_time(None, now)
+        return next_run.isoformat() if next_run else None
+    except Exception:
+        return None
 
 ##————————————————————————————————————————————————————————————————————————————##
 ##                                                                            ##