emergence_detector.py

"""
Emergence detection for identifying rapidly growing topics.
"""
import numpy as np
import pandas as pd
from typing import List
from scipy import stats
from scipy.signal import find_peaks
from sklearn.preprocessing import StandardScaler


class EmergenceDetector:
    """
    Detect emerging topics using statistical methods including:
    - CUSUM change point detection
    - Growth rate analysis
    - Trend strength calculation
    - Persistence measurement
    """

    def __init__(self, significance_level: float = 0.05, min_persistence: int = 3):
        """
        Initialize the emergence detector.

        Parameters:
        -----------
        significance_level : float
            Statistical significance level for tests
        min_persistence : int
            Minimum number of time points a topic should persist to be considered emerging
        """
        self.significance_level = significance_level
        self.min_persistence = min_persistence
        self.scaler = StandardScaler()

    def detect_change_points(self, topic_frequencies: np.ndarray) -> List[int]:
        """
        Detect change points in topic frequency using CUSUM algorithm.

        Parameters:
        -----------
        topic_frequencies : np.ndarray
            Array of topic frequencies over time

        Returns:
        --------
        List[int]
            Indices of detected change points
        """
        mean_freq = np.mean(topic_frequencies)
        cusum_pos = np.maximum.accumulate(np.maximum(0,
            np.cumsum(topic_frequencies - mean_freq)))
        cusum_neg = np.maximum.accumulate(np.maximum(0,
            np.cumsum(mean_freq - topic_frequencies)))

        threshold = 3 * np.std(topic_frequencies)
        changes_pos = find_peaks(cusum_pos, height=threshold)[0]
        changes_neg = find_peaks(cusum_neg, height=threshold)[0]

        return sorted(list(changes_pos) + list(changes_neg))

    def calculate_emergence_score(self,
                                topic_frequencies: np.ndarray,
                                time_window: int = 5) -> float:
        """
        Calculate emergence score for a topic based on growth rate, acceleration,
        and persistence.

        Parameters:
        -----------
        topic_frequencies : np.ndarray
            Array of topic frequencies over time
        time_window : int
            Number of recent time points to consider

        Returns:
        --------
        float
            Emergence score (higher = more emerging)
        """
        if len(topic_frequencies) < time_window:
            return 0.0

        recent = topic_frequencies[-time_window:]
        historical = topic_frequencies[:-time_window] if len(topic_frequencies) > time_window else np.array([0])

        growth_rate = (np.mean(recent) - np.mean(historical)) / (np.mean(historical) + 1e-6)

        if len(topic_frequencies) >= 3:
            acceleration = np.mean(np.diff(np.diff(topic_frequencies)))
        else:
            acceleration = 0

        persistence = len([x for x in recent if x > np.mean(historical)]) / len(recent)

        emergence_score = (0.4 * growth_rate +
                         0.3 * acceleration +
                         0.3 * persistence)

        return max(0, emergence_score)

    def detect_emerging_topics(self, topics_over_time_df: pd.DataFrame,
                              top_n: int = 10) -> pd.DataFrame:
        """
        Detect emerging topics from topics over time data.

        Parameters:
        -----------
        topics_over_time_df : pd.DataFrame
            DataFrame with columns: Topic, Timestamp, Frequency
        top_n : int
            Number of top emerging topics to return

        Returns:
        --------
        pd.DataFrame
            DataFrame of emerging topics sorted by emergence score
        """
        emerging_topics = []

        unique_topics = [t for t in topics_over_time_df['Topic'].unique() if t != -1]

        for topic in unique_topics:
            topic_data = topics_over_time_df[topics_over_time_df['Topic'] == topic].sort_values('Timestamp')
            frequencies = topic_data['Frequency'].values

            if len(frequencies) < 3:
                continue

            emergence_score = self.calculate_emergence_score(frequencies)
            change_points = self.detect_change_points(frequencies)

            if len(frequencies) > 1:
                slope, _, r_value, _, _ = stats.linregress(range(len(frequencies)), frequencies)
                trend_strength = r_value ** 2
            else:
                slope = 0
                trend_strength = 0

            recent_avg = np.mean(frequencies[-3:]) if len(frequencies) >= 3 else frequencies[-1]
            historical_avg = np.mean(frequencies[:-3]) if len(frequencies) > 3 else frequencies[0]
            growth_ratio = recent_avg / (historical_avg + 1e-6)

            emerging_topics.append({
                'Topic': topic,
                'Emergence_Score': emergence_score,
                'Growth_Ratio': growth_ratio,
                'Trend_Strength': trend_strength,
                'Slope': slope,
                'Change_Points': len(change_points),
                'Recent_Avg_Freq': recent_avg,
                'Historical_Avg_Freq': historical_avg
            })

        emerging_df = pd.DataFrame(emerging_topics)
        emerging_df = emerging_df.sort_values('Emergence_Score', ascending=False).head(top_n)

        return emerging_df