Merge origin/main: Integrate Hardcover auto-fetch feature (PR #877) with json.txt fix for issue #925

Author: crocodilestick

cps/admin.py

@@ -190,6 +190,193 @@ def queue_metadata_backup():
     return json.dumps(show_text)
 
 
+@admi.route("/hardcover_auto_fetch", methods=["POST"])
+@user_login_required
+@admin_required
+def trigger_hardcover_auto_fetch():
+    """Manually trigger Hardcover auto-fetch task"""
+    show_text = {}
+    
+    try:
+        # Check if token is available
+        from os import getenv
+        token_available = bool(
+            getattr(config, "config_hardcover_token", None) or 
+            getenv("HARDCOVER_TOKEN")
+        )
+        
+        if not token_available:
+            show_text['text'] = _('Error: No Hardcover token available. Set HARDCOVER_TOKEN environment variable or configure in Basic Configuration.')
+            return json.dumps(show_text), 400
+        
+        # Get settings
+        import sys as _sys
+        if '/app/calibre-web-automated/scripts/' not in _sys.path:
+            _sys.path.insert(1, '/app/calibre-web-automated/scripts/')
+        from cwa_db import CWA_DB
+        from cps.tasks.auto_hardcover_id import TaskAutoHardcoverID
+        from cps.services.worker import WorkerThread
+        
+        cwa_db = CWA_DB()
+        cwa_settings = cwa_db.get_cwa_settings()
+        
+        min_confidence = float(cwa_settings.get('hardcover_auto_fetch_min_confidence', 0.85))
+        batch_size = int(cwa_settings.get('hardcover_auto_fetch_batch_size', 50))
+        rate_limit = float(cwa_settings.get('hardcover_auto_fetch_rate_limit', 5.0))
+        
+        # Create and enqueue task
+        task = TaskAutoHardcoverID(
+            min_confidence=min_confidence,
+            batch_size=batch_size,
+            rate_limit_delay=rate_limit
+        )
+        
+        WorkerThread.add(current_user.name, task, hidden=False)
+        
+        log.info(f"Hardcover auto-fetch task manually triggered by {current_user.name}")
+        show_text['text'] = _('Success! Hardcover auto-fetch task started. Check Tasks panel for progress.')
+        return json.dumps(show_text)
+        
+    except Exception as e:
+        log.error(f"Error triggering Hardcover auto-fetch: {e}")
+        show_text['text'] = _('Error starting Hardcover auto-fetch task: %(error)s', error=str(e))
+        return json.dumps(show_text), 500
+
+
+@admi.route("/admin/hardcover/review-matches")
+@user_login_required
+@admin_required
+def hardcover_review_matches():
+    """Display queue of Hardcover matches needing manual review"""
+    try:
+        # Get pending matches from database
+        pending_matches = ub.session.query(ub.HardcoverMatchQueue).filter(
+            ub.HardcoverMatchQueue.reviewed == 0
+        ).order_by(ub.HardcoverMatchQueue.created_at.desc()).all()
+        
+        # Parse JSON data for each match
+        matches_data = []
+        for match in pending_matches:
+            import json
+            try:
+                results = json.loads(match.hardcover_results)
+                scores = json.loads(match.confidence_scores)
+                
+                matches_data.append({
+                    'id': match.id,
+                    'book_id': match.book_id,
+                    'book_title': match.book_title,
+                    'book_authors': match.book_authors,
+                    'search_query': match.search_query,
+                    'results': results,
+                    'scores': scores,
+                    'created_at': match.created_at
+                })
+            except Exception as e:
+                log.error(f"Error parsing match queue entry {match.id}: {e}")
+                continue
+        
+        return render_title_template(
+            "hardcover_review_matches.html",
+            title=_("Review Hardcover Matches"),
+            page="hardcover-review",
+            matches=matches_data
+        )
+        
+    except Exception as e:
+        log.error(f"Error loading Hardcover review queue: {e}")
+        flash(_("Error loading review queue: %(error)s", error=str(e)), category="error")
+        return redirect(url_for('admin.admin'))
+
+
+@admi.route("/admin/hardcover/review-action", methods=["POST"])
+@user_login_required
+@admin_required
+def hardcover_review_action():
+    """Process review action (accept/reject/skip) for a queued match"""
+    try:
+        data = request.get_json()
+        queue_id = int(data.get('queue_id'))
+        action = data.get('action')  # 'accept', 'reject', 'skip'
+        selected_result_id = data.get('selected_result_id')
+        
+        # Get queue entry
+        match = ub.session.query(ub.HardcoverMatchQueue).filter(
+            ub.HardcoverMatchQueue.id == queue_id
+        ).first()
+        
+        if not match:
+            return json.dumps({'success': False, 'error': 'Match not found'}), 404
+        
+        if action == 'accept' and selected_result_id:
+            # Apply the selected Hardcover ID to the book
+            import json
+            results = json.loads(match.hardcover_results)
+            selected_result = next((r for r in results if str(r['id']) == str(selected_result_id)), None)
+            
+            if not selected_result:
+                return json.dumps({'success': False, 'error': 'Selected result not found'}), 400
+            
+            # Get the book
+            book = calibre_db.session.query(db.Books).filter(
+                db.Books.id == match.book_id
+            ).first()
+            
+            if not book:
+                return json.dumps({'success': False, 'error': 'Book not found'}), 404
+            
+            # Add identifiers
+            try:
+                identifiers_to_add = selected_result.get('identifiers', {})
+                for id_type, id_value in identifiers_to_add.items():
+                    # Check if identifier already exists
+                    existing = calibre_db.session.query(db.Identifiers).filter(
+                        db.Identifiers.book == match.book_id,
+                        db.Identifiers.type == id_type
+                    ).first()
+                    
+                    if not existing:
+                        new_identifier = db.Identifiers(str(id_value), id_type, match.book_id)
+                        calibre_db.session.add(new_identifier)
+                
+                calibre_db.session.commit()
+                
+                # Mark as reviewed
+                match.reviewed = 1
+                match.selected_result_id = str(selected_result_id)
+                match.review_action = 'accept'
+                match.reviewed_at = datetime.datetime.utcnow().isoformat()
+                match.reviewed_by = current_user.name
+                ub.session.commit()
+                
+                log.info(f"User {current_user.name} accepted Hardcover match for book {match.book_id}")
+                return json.dumps({'success': True, 'message': _('Hardcover ID applied successfully')})
+                
+            except Exception as e:
+                calibre_db.session.rollback()
+                ub.session.rollback()
+                log.error(f"Error applying Hardcover ID: {e}")
+                return json.dumps({'success': False, 'error': str(e)}), 500
+        
+        elif action in ['reject', 'skip']:
+            # Mark as reviewed with appropriate action
+            match.reviewed = 1
+            match.review_action = action
+            match.reviewed_at = datetime.datetime.utcnow().isoformat()
+            match.reviewed_by = current_user.name
+            ub.session.commit()
+            
+            log.info(f"User {current_user.name} {action}ed Hardcover match for book {match.book_id}")
+            return json.dumps({'success': True, 'message': _('Match %(action)s', action=action)})
+        
+        else:
+            return json.dumps({'success': False, 'error': 'Invalid action'}), 400
+            
+    except Exception as e:
+        log.error(f"Error processing review action: {e}")
+        return json.dumps({'success': False, 'error': str(e)}), 500
+
+
 # method is available without login and not protected by CSRF to make it easy reachable, is per default switched off
 # needed for docker applications, as changes on metadata.db from host are not visible to application
 @admi.route("/reconnect", methods=['GET'])

cps/cwa_functions.py

@@ -609,11 +609,12 @@ def set_cwa_settings():
     boolean_settings = []
     string_settings = []
     list_settings = []
-    integer_settings = ['ingest_timeout_minutes', 'auto_send_delay_minutes', 'duplicate_scan_hour', 'duplicate_scan_chunk_size', 'duplicate_scan_debounce_seconds']  # Special handling for integer settings
+    integer_settings = ['ingest_timeout_minutes', 'auto_send_delay_minutes', 'hardcover_auto_fetch_batch_size', 'hardcover_auto_fetch_schedule_hour', 'duplicate_scan_hour', 'duplicate_scan_chunk_size', 'duplicate_scan_debounce_seconds']  # Special handling for integer settings
+    float_settings = ['hardcover_auto_fetch_min_confidence', 'hardcover_auto_fetch_rate_limit']  # Special handling for float settings
     json_settings = ['metadata_provider_hierarchy', 'metadata_providers_enabled', 'duplicate_format_priority']  # Special handling for JSON settings
 
     for setting in cwa_default_settings:
-        if setting in integer_settings or setting in json_settings:
+        if setting in integer_settings or setting in float_settings or setting in json_settings:
             continue  # Handle separately
         elif isinstance(cwa_default_settings[setting], int):
             boolean_settings.append(setting)
@@ -684,11 +685,15 @@ def set_cwa_settings():
                 if value is not None:
                     try:
                         int_value = int(value)
-                        # Validate timeout range
+                        # Validate range
                         if setting == 'ingest_timeout_minutes':
                             int_value = max(5, min(120, int_value))  # Clamp between 5 and 120 minutes
                         elif setting == 'auto_send_delay_minutes':
                             int_value = max(1, min(60, int_value))  # Clamp between 1 and 60 minutes
+                        elif setting == 'hardcover_auto_fetch_batch_size':
+                            int_value = max(10, min(200, int_value))  # Clamp between 10 and 200
+                        elif setting == 'hardcover_auto_fetch_schedule_hour':
+                            int_value = max(0, min(23, int_value))  # Clamp between 0 and 23 hours
                         elif setting == 'duplicate_scan_hour':
                             int_value = max(0, min(23, int_value))
                         elif setting == 'duplicate_scan_chunk_size':
@@ -702,16 +707,49 @@ def set_cwa_settings():
                             result[setting] = cwa_db.cwa_settings.get(setting, 15)  # Default to 15 minutes
                         elif setting == 'auto_send_delay_minutes':
                             result[setting] = cwa_db.cwa_settings.get(setting, 5)  # Default to 5 minutes
+                        elif setting == 'hardcover_auto_fetch_batch_size':
+                            result[setting] = cwa_db.cwa_settings.get(setting, 50)  # Default to 50
+                        elif setting == 'hardcover_auto_fetch_schedule_hour':
+                            result[setting] = cwa_db.cwa_settings.get(setting, 2)  # Default to 2 AM
                         elif setting == 'duplicate_scan_debounce_seconds':
                             result[setting] = cwa_db.cwa_settings.get(setting, 30)
                 else:
                     if setting == 'ingest_timeout_minutes':
                         result[setting] = cwa_db.cwa_settings.get(setting, 15)  # Default to 15 minutes
                     elif setting == 'auto_send_delay_minutes':
                         result[setting] = cwa_db.cwa_settings.get(setting, 5)  # Default to 5 minutes
+                    elif setting == 'hardcover_auto_fetch_batch_size':
+                        result[setting] = cwa_db.cwa_settings.get(setting, 50)  # Default to 50
+                    elif setting == 'hardcover_auto_fetch_schedule_hour':
+                        result[setting] = cwa_db.cwa_settings.get(setting, 2)  # Default to 2 AM
                     elif setting == 'duplicate_scan_debounce_seconds':
                         result[setting] = cwa_db.cwa_settings.get(setting, 30)
 
+            # Handle float settings
+            for setting in float_settings:
+                value = request.form.get(setting)
+                if value is not None:
+                    try:
+                        float_value = float(value)
+                        # Validate range
+                        if setting == 'hardcover_auto_fetch_min_confidence':
+                            float_value = max(0.5, min(1.0, float_value))  # Clamp between 0.5 and 1.0
+                        elif setting == 'hardcover_auto_fetch_rate_limit':
+                            float_value = max(0.0, min(60.0, float_value))  # Clamp between 0 and 60 seconds
+                        result[setting] = float_value
+                    except (ValueError, TypeError):
+                        # Use current value if conversion fails
+                        if setting == 'hardcover_auto_fetch_min_confidence':
+                            result[setting] = cwa_db.cwa_settings.get(setting, 0.85)  # Default to 0.85
+                        elif setting == 'hardcover_auto_fetch_rate_limit':
+                            result[setting] = cwa_db.cwa_settings.get(setting, 5.0)  # Default to 5.0 seconds
+                else:
+                    if setting == 'hardcover_auto_fetch_min_confidence':
+                        result[setting] = cwa_db.cwa_settings.get(setting, 0.85)  # Default to 0.85
+                    elif setting == 'hardcover_auto_fetch_rate_limit':
+                        result[setting] = cwa_db.cwa_settings.get(setting, 5.0)  # Default to 5.0 seconds
+
+
             # Handle JSON settings
             for setting in json_settings:
                 value = request.form.get(setting)
@@ -792,11 +830,19 @@ def set_cwa_settings():
         cwa_db = CWA_DB()
         cwa_settings = cwa_db.get_cwa_settings()
 
+    # Check if Hardcover token is available
+    from os import getenv
+    hardcover_token_available = bool(
+        getattr(config, "config_hardcover_token", None) or 
+        getenv("HARDCOVER_TOKEN")
+    )
+
     next_scan_run = get_next_duplicate_scan_run(cwa_settings)
 
     return render_title_template("cwa_settings.html", title=_("Calibre-Web Automated User Settings"), page="cwa-settings",
                                     cwa_settings=cwa_settings, ignorable_formats=ignorable_formats, target_formats=target_formats,
                                     automerge_options=automerge_options, autoingest_options=autoingest_options,
+                                    hardcover_token_available=hardcover_token_available,
                                     next_duplicate_scan_run=next_scan_run, config=config)
 
 
@@ -820,6 +866,7 @@ def get_next_duplicate_scan_run(settings):
     except Exception:
         return None
 
+
 ##————————————————————————————————————————————————————————————————————————————##
 ##                                                                            ##
 ##                               CWA SHOW HISTORY                             ##
@@ -988,6 +1035,38 @@ def cwa_stats_show():
     data_epub_fixer = cwa_db.get_epub_fixer_history(fixes=False, verbose=False)
     data_epub_fixer_with_fixes = cwa_db.get_epub_fixer_history(fixes=True, verbose=False)
 
+    # Get Hardcover auto-fetch stats
+    hardcover_stats = None
+    try:
+        # Get total stats from hardcover_auto_fetch_stats table
+        total_processed = cwa_db.execute_read(
+            "SELECT SUM(books_processed) FROM hardcover_auto_fetch_stats"
+        )
+        total_auto_matched = cwa_db.execute_read(
+            "SELECT SUM(auto_matched) FROM hardcover_auto_fetch_stats"
+        )
+        
+        # Get pending review count
+        pending_review = ub.session.query(ub.HardcoverMatchQueue).filter(
+            ub.HardcoverMatchQueue.reviewed == 0
+        ).count()
+        
+        # Get manually reviewed count
+        manually_reviewed = ub.session.query(ub.HardcoverMatchQueue).filter(
+            ub.HardcoverMatchQueue.reviewed == 1,
+            ub.HardcoverMatchQueue.review_action == 'accept'
+        ).count()
+        
+        hardcover_stats = {
+            'total_processed': total_processed[0][0] if total_processed and total_processed[0][0] else 0,
+            'total_auto_matched': total_auto_matched[0][0] if total_auto_matched and total_auto_matched[0][0] else 0,
+            'pending_review': pending_review,
+            'manually_reviewed': manually_reviewed
+        }
+    except Exception as e:
+        log.debug(f"Error fetching Hardcover stats: {e}")
+        hardcover_stats = None
+
     return render_title_template("cwa_stats_tabs.html", title=_("Calibre-Web Automated Stats & Activity"),
                                 page="cwa-stats",
                                 active_tab=active_tab,
@@ -1024,14 +1103,14 @@ def cwa_stats_show():
                                 active_users=active_users,
                                 selected_user_id=user_id,
                                 cwa_stats=get_cwa_stats(),
+                                hardcover_stats=hardcover_stats,
                                 data_enforcement=data_enforcement, headers_enforcement=headers["enforcement"]["no_paths"], 
                                 data_enforcement_with_paths=data_enforcement_with_paths, headers_enforcement_with_paths=headers["enforcement"]["with_paths"], 
                                 data_imports=data_imports, headers_import=headers["imports"],
                                 data_conversions=data_conversions, headers_conversion=headers["conversions"],
                                 data_epub_fixer=data_epub_fixer, headers_epub_fixer=headers["epub_fixer"]["no_fixes"],
                                 data_epub_fixer_with_fixes=data_epub_fixer_with_fixes, headers_epub_fixer_with_fixes=headers["epub_fixer"]["with_fixes"])
 
-
 @cwa_stats.route("/cwa-stats-export-csv/<tab_name>", methods=["GET"])
 @login_required_if_no_ano
 @admin_required