config.py

"""
Configuration settings for the topic modeling and burst detection system.
"""
import os

# ==================== DATA PATHS ====================
# Update this path to point to your data file
DATA_FILE_PATH = 'train_preprocessed.csv'  # Main training dataset

# ==================== OUTPUT PATHS ====================
MODEL_SAVE_PATH = "my_dynamic_bertopic_model"

RESULTS_DIR                = "results"
RQ1_TOPIC_MODELING_DIR     = "results/rq1_topic_modeling"
RQ1_BASELINES_DIR          = "results/rq1_baselines"
RQ2_TRENDS_DIR             = "results/rq2_trends"
OUTPUT_DIR                 = "results/rq2_visualizations"
RQ3_OUTPUT_DIR             = "results/rq3_information_loss"
ARTICLE_EVAL_DIR           = "results/novel_article_evaluation"

# ==================== TOPIC MODELING PARAMETERS ====================
# Sentence transformer model for embeddings
EMBEDDING_MODEL_NAME = "all-MiniLM-L6-v2"

# BERTopic parameters
BERTOPIC_LANGUAGE = "english"
BERTOPIC_CALCULATE_PROBS = False  # Disabled to reduce memory usage (50%+ reduction)
BERTOPIC_VERBOSE = True

# UMAP parameters for dimensionality reduction
# Optimized for CPU mode to reduce memory usage
UMAP_N_NEIGHBORS = 10  # Reduced from 15 (less RAM)
UMAP_N_COMPONENTS = 3  # Reduced from 5 (less RAM, faster)
UMAP_MIN_DIST = 0.0
UMAP_METRIC = 'cosine'

# HDBSCAN parameters for clustering
# Optimized for CPU mode to reduce memory usage
HDBSCAN_MIN_CLUSTER_SIZE = 20  # Increased from 15 (fewer clusters, less RAM)
HDBSCAN_MIN_SAMPLES = 10
HDBSCAN_METRIC = 'euclidean'
HDBSCAN_CLUSTER_SELECTION_METHOD = 'eom'
HDBSCAN_N_JOBS = -1  # Use all CPU cores (OpenMP conflicts fixed in main.py)

# Stop words for vectorizer
STOP_WORDS = "english"

# Number of time bins for temporal analysis
# NR_BINS = 20  # Fixed bins (old approach)
# For weekly bins, this will be calculated dynamically in the scripts
# based on actual data time range
NR_BINS = None  # Set to None for dynamic weekly calculation

# ==================== EMERGENCE DETECTION PARAMETERS ====================
EMERGENCE_SIGNIFICANCE_LEVEL = 0.05
EMERGENCE_MIN_PERSISTENCE = 3
EMERGENCE_TOP_N = 10

# ==================== BURST DETECTION PARAMETERS ====================
# Kleinberg's algorithm cost parameter (higher = fewer bursts)
BURST_GAMMA = 1.0

# Sensitivity for threshold-based detection (in standard deviations)
BURST_SENSITIVITY = 2.0

# Minimum consecutive time points to consider as a burst
BURST_MIN_LENGTH = 2

# EWMA alpha parameter
BURST_EWMA_ALPHA = 0.3

# Moving average window size
BURST_MA_WINDOW = 5

# Minimum number of methods for consensus burst detection
BURST_CONSENSUS_MIN_METHODS = 2

# Number of top topics to analyze for bursts
BURST_TOP_N_TOPICS = 20

# ==================== VISUALIZATION PARAMETERS ====================
# Number of top emerging topics to visualize in detail
VIZ_TOP_EMERGING_TOPICS = 3

# Number of topics to show in BERTopic visualizations
VIZ_BERTOPIC_TOP_N = 10

# Number of topics to show in distance heatmaps
VIZ_DISTANCE_HEATMAP_LIMIT = 50

# Figure sizes
VIZ_BURST_DETECTION_FIGSIZE = (15, 12)
VIZ_EMERGING_TRENDS_FIGSIZE = (18, 10)
VIZ_DISTANCE_HEATMAP_FIGSIZE = (12, 10)
VIZ_HIERARCHICAL_FIGSIZE = (15, 8)
VIZ_TSNE_PCA_FIGSIZE = (14, 10)

# DPI for saved figures
VIZ_DPI = 300

# ==================== EVALUATION PARAMETERS (OBJECTIVE 3) ====================
# Coherence metrics to calculate
EVAL_COHERENCE_METRICS = ['c_v', 'c_npmi', 'c_uci']

# Baseline methods to compare against
EVAL_BASELINE_METHODS = ['lda', 'nmf']

# Number of topics for baseline methods
EVAL_BASELINE_N_TOPICS = [5, 10, 15, 20, 25]

# Performance metrics
EVAL_MEASURE_PERFORMANCE = True

# ==================== SUMMARIZATION PARAMETERS (OBJECTIVE 4 / RQ3) ====================
# Summarization methods to test
# Using only best-performing graph-based methods for better results
SUMMARIZATION_METHODS = ['textrank', 'lexrank']

# Summary length ratios (% of original text)
# Increased ratios to reduce information loss (0.5, 0.65, 0.8 = keep 50%, 65%, 80% of text)
SUMMARY_LENGTH_RATIOS = [0.5, 0.65, 0.8]

# Minimum summary length (in tokens)
MIN_SUMMARY_LENGTH = 50

# Maximum summary length (in tokens)
MAX_SUMMARY_LENGTH = 512

# Number of sentences for extractive methods
# Increased from 3 to 5 to preserve more context
EXTRACTIVE_N_SENTENCES = 5

# Topic similarity threshold for matching topics between full/summary
# Relaxed to 0.5 for better matching on shorter texts
TOPIC_SIMILARITY_THRESHOLD = 0.5

# Comparison metrics to calculate
COMPARISON_METRICS = ['topic_overlap', 'coherence', 'diversity', 'temporal', 'emergence', 'burst']

# Output paths for RQ3 (defined in OUTPUT PATHS section above)
RQ3_COMPARISON_REPORT = "information_loss_report.md"

# ==================== GENERATIVE AI PARAMETERS (OBJECTIVE 5) ====================
# LLM Provider
LLM_PROVIDER = "ollama"  # Options: 'gemini', 'openai', 'claude', 'ollama'
                         # Gemini: cloud API, aggressive safety filters blocked 100% of topics
                         # Ollama: local model on server GPU, no filters, no rate limits

# Gemini API Key (set as environment variable or here)
# os.environ["GEMINI_API_KEY"] = "your-api-key-here"
GEMINI_API_KEY = "AIzaSyAl0JtU22w7SbEqhjFBZXX17rT2Oq1H6e8"

# Gemini model to use
GEMINI_MODEL = "models/gemini-2.5-flash"  # Fast, stable model for topic descriptions

# Temperature for generation (0.0 = deterministic, 1.0 = creative)
LLM_TEMPERATURE = 0.3

# Maximum tokens for generation
LLM_MAX_TOKENS = 500

# Number of sample documents to include in prompts
LLM_SAMPLE_DOCS = 3

# Number of keywords to include in prompts
LLM_N_KEYWORDS = 10

# Enable caching of LLM responses (to save API calls)
LLM_ENABLE_CACHE = True
LLM_CACHE_DIR = "llm_cache"

# Delay between API requests in seconds (free tier: 10 requests/min = 6+ seconds delay)
LLM_REQUEST_DELAY = 7.0  # Seconds between requests (ignored for Ollama — no rate limit)

# ==================== OLLAMA PARAMETERS ====================
# Ollama runs locally on your server — no API key, no rate limits, no safety filters
# Install: curl -fsSL https://ollama.com/install.sh | sh && ollama pull llama3.2
OLLAMA_MODEL = "llama3.2"          # Model to use (must be pulled first via: ollama pull llama3.2)
OLLAMA_HOST  = "http://localhost:11434"  # Default Ollama host

# Output format options
GENAI_OUTPUT_FORMATS = ['markdown', 'html', 'json']

# Generate descriptions for how many topics
# Gemini: 49 topics × 2 requests = 98 API calls; ALL blocked by safety filters (Civil Comments is political)
# Ollama: no rate limits, no filters — all 49 topics complete in ~5 minutes on server GPU
# Set to 5 for testing, change to 49 for full run
GENAI_N_TOPICS = 49

# Output paths for Objective 5
GENAI_OUTPUT_DIR = "genai_presentations"
GENAI_DASHBOARD_PATH = "topic_dashboard.html"
GENAI_REPORT_PATH = "topic_analysis_report.md"

# ==================== GPU SETTINGS ====================
# GPU/Device Configuration
# Set to True to force CPU usage (useful for debugging)
FORCE_CPU = False

# Use GPU-accelerated UMAP/HDBSCAN (requires cuML)
# Set to False for CUDA 13+ compatibility (cuML not yet supported)
USE_GPU_UMAP = False

# Skip UMAP for very large datasets (1M+ documents).
# BERTopic works directly on high-dim embeddings; saves significant RAM.
SKIP_UMAP = True

# ---- Multi-GPU (RTX 5090 x2) -------------------------------------------
# Set to None to let gpu_utils.py auto-detect from VRAM.
# RTX 5090 has 32 GB VRAM each → 64 GB combined.
# Per-GPU sweet spot for all-MiniLM-L6-v2 (384-dim): ~1024-1536.
# The multi-process pool in main.py doubles effective throughput automatically.
EMBEDDING_BATCH_SIZE = None   # Auto-detected; override with e.g. 1024 if needed

# Use SentenceTransformer's built-in multi-process pool to spread encoding
# across all available CUDA GPUs (each GPU gets its own process + model copy).
# Ignored when only 1 GPU is available.
USE_MULTI_GPU_ENCODING = True

# Precision for embedding model:
#   'auto'   → bf16 on Ampere/Ada/Blackwell, fp16 on Turing, fp32 on older
#   'bf16'   → force BF16 (RTX 5090 recommended — wider range than fp16)
#   'fp16'   → force FP16
#   'fp32'   → no mixed precision
EMBEDDING_PRECISION = "auto"

# ---- Data-loading (CPU → GPU pipeline) ----------------------------------
# Workers for pandas/CSV reading (more = faster for large CSVs)
NUM_WORKERS = 8  # Use ~(num_CPU_cores / 2); vast.ai instances often have 16+ cores

# Enable CUDA pinned (page-locked) memory for host tensors.
# Pinned memory allows async DMA transfers CPU→GPU, removing CPU bottleneck.
USE_PINNED_MEMORY = True

# Prefetch factor: how many extra batches to prepare ahead of time.
# Higher = more RAM used but fewer GPU idle gaps.
PREFETCH_FACTOR = 2

# Clear GPU cache between major operations (embedding → clustering → viz)
AUTO_CLEAR_CACHE = True

# ==================== OTHER SETTINGS ====================
# Disable tokenizers parallelism warning
os.environ["TOKENIZERS_PARALLELISM"] = "false"