article_based_evaluator.py

"""
Article-Based Evaluation Framework.

Novel evaluation approach using article IDs to validate topic models and
emergence detection without manual labeling.

Key Ideas:
1. Article Topic Coherence: Comments on same article should cluster together
2. Emergence Validation: Real emerging topics correlate with article timing
3. Organic Trends: Topics trending without article correlation = news leads
"""
import pandas as pd
from typing import List, Dict, Tuple
from bertopic import BERTopic
from collections import Counter, defaultdict
from datetime import datetime, timedelta
from scipy.stats import spearmanr


class ArticleBasedEvaluator:
    """
    Evaluate topic models using article structure as ground truth.
    """

    def __init__(self, topic_model: BERTopic, docs: List[str],
                 article_ids: List[str], timestamps: List[str]):
        """
        Initialize evaluator.

        Parameters:
        -----------
        topic_model : BERTopic
            Trained topic model
        docs : List[str]
            Document texts
        article_ids : List[str]
            Article IDs for each document
        timestamps : List[str]
            ISO timestamps for each document
        """
        self.topic_model = topic_model
        self.docs = docs
        self.article_ids = article_ids
        self.timestamps = timestamps

        # Get topic assignments
        self.topics = topic_model.topics_

        # Build article-level data
        self._build_article_index()

    def _build_article_index(self):
        """Build index of articles and their comments."""
        self.article_data = defaultdict(lambda: {
            'comment_indices': [],
            'topics': [],
            'timestamps': []
        })

        for i, article_id in enumerate(self.article_ids):
            self.article_data[article_id]['comment_indices'].append(i)
            self.article_data[article_id]['topics'].append(self.topics[i])
            self.article_data[article_id]['timestamps'].append(self.timestamps[i])

    def calculate_article_topic_coherence(self, min_comments: int = 5) -> Dict:
        """
        Calculate topic coherence within articles.

        Hypothesis: Comments on same article should mostly belong to same topic(s).
        High coherence = good topic model quality.

        Parameters:
        -----------
        min_comments : int
            Minimum comments per article to include

        Returns:
        --------
        Dict
            Coherence metrics
        """
        coherence_scores = []
        article_stats = []

        for article_id, data in self.article_data.items():
            n_comments = len(data['topics'])

            if n_comments < min_comments:
                continue

            # Get topic distribution for this article
            topic_counts = Counter(data['topics'])

            # Remove outlier topic (-1) and catch-all topic 0
            if -1 in topic_counts:
                del topic_counts[-1]
            if 0 in topic_counts:
                del topic_counts[0]

            if not topic_counts:
                continue

            # Calculate concentration (Gini coefficient variant)
            # High value = concentrated in few topics (good)
            total = sum(topic_counts.values())

            # Dominant topic
            dominant_topic = max(topic_counts, key=topic_counts.get)
            dominant_ratio = topic_counts[dominant_topic] / total

            coherence_scores.append({
                'article_id': article_id,
                'n_comments': n_comments,
                'n_topics': len(topic_counts),
                'dominant_topic': dominant_topic,
                'dominant_topic_count': topic_counts[dominant_topic],
                'dominant_topic_ratio': round(dominant_ratio, 4),
            })

            # Full topic breakdown for this article
            for topic_id, count in sorted(topic_counts.items()):
                article_stats.append({
                    'article_id': article_id,
                    'topic_id': topic_id,
                    'comment_count': count,
                    'ratio': round(count / total, 4),
                    'is_dominant': topic_id == dominant_topic
                })

        summary_df = pd.DataFrame(coherence_scores)
        distribution_df = pd.DataFrame(article_stats)

        return {
            'article_coherence_scores': summary_df,
            'article_topic_distribution': distribution_df,
            'mean_coherence': summary_df['dominant_topic_ratio'].mean(),
            'median_coherence': summary_df['dominant_topic_ratio'].median(),
            'mean_topics_per_article': summary_df['n_topics'].mean(),
            'articles_evaluated': len(summary_df)
        }

    def validate_emerging_topics(self, emerging_topics: pd.DataFrame,
                                 time_window_days: int = 7) -> Dict:
        """
        Validate emerging topics by checking article correlation.

        Hypothesis: Real emerging topics should correlate with articles
        published around the emergence timepoint.

        Parameters:
        -----------
        emerging_topics : pd.DataFrame
            Emerging topics from EmergenceDetector
        time_window_days : int
            Days before/after emergence to look for articles

        Returns:
        --------
        Dict
            Validation results for each emerging topic
        """
        if emerging_topics.empty:
            return {'validated_topics': [], 'validation_scores': []}

        validation_results = []

        for _, row in emerging_topics.iterrows():
            topic_id = row['Topic']

            # Find emergence time from topic data
            # Use the time when topic had highest recent activity
            topic_comment_indices = [i for i, t in enumerate(self.topics) if t == topic_id]

            if not topic_comment_indices:
                continue

            topic_timestamps = [pd.to_datetime(self.timestamps[i]) for i in topic_comment_indices]

            # Find emergence time as the median time of recent comments
            # (approximation: when topic was most active)
            topic_timestamps_sorted = sorted(topic_timestamps)

            if len(topic_timestamps_sorted) < 5:
                # Use median for small topics
                emergence_time = topic_timestamps_sorted[len(topic_timestamps_sorted)//2]
            else:
                # Use the median of the most recent 30% of comments as emergence time
                recent_cutoff = int(len(topic_timestamps_sorted) * 0.7)
                recent_timestamps = topic_timestamps_sorted[recent_cutoff:]
                emergence_time = recent_timestamps[len(recent_timestamps)//2]

            # Get article IDs for this topic's comments
            topic_articles = [self.article_ids[i] for i in topic_comment_indices]

            # Find articles published around emergence time
            window_start = emergence_time - timedelta(days=time_window_days)
            window_end = emergence_time + timedelta(days=time_window_days)

            # Count comments in window vs outside
            comments_in_window = sum(1 for t in topic_timestamps
                                   if window_start <= t <= window_end)
            comments_outside_window = len(topic_timestamps) - comments_in_window

            # Calculate article publication times
            article_first_comment = {}
            for i in topic_comment_indices:
                art_id = self.article_ids[i]
                timestamp = pd.to_datetime(self.timestamps[i])
                if art_id not in article_first_comment:
                    article_first_comment[art_id] = timestamp
                else:
                    article_first_comment[art_id] = min(article_first_comment[art_id], timestamp)

            # Count articles published in window
            articles_in_window = sum(1 for t in article_first_comment.values()
                                   if window_start <= t <= window_end)

            # Validation score: ratio of comments in window
            validation_score = comments_in_window / len(topic_timestamps) if topic_timestamps else 0

            validation_results.append({
                'topic_id': topic_id,
                'emergence_time': emergence_time,
                'total_comments': len(topic_timestamps),
                'comments_in_window': comments_in_window,
                'comments_outside_window': comments_outside_window,
                'articles_in_topic': len(set(topic_articles)),
                'articles_in_window': articles_in_window,
                'validation_score': validation_score,
                'is_validated': validation_score > 0.5  # >50% in window
            })

        df = pd.DataFrame(validation_results)

        return {
            'validation_results': df,
            'mean_validation_score': df['validation_score'].mean(),
            'validated_count': df['is_validated'].sum(),
            'total_emerging': len(df),
            'validation_rate': df['is_validated'].sum() / len(df) if len(df) > 0 else 0
        }

    def detect_organic_trends(self, topics_over_time: pd.DataFrame,
                             min_comments: int = 20,
                             article_correlation_threshold: float = 0.3) -> pd.DataFrame:
        """
        Find topics trending WITHOUT recent article correlation.

        These are "organic" trends - topics people discuss without
        the publication covering them. Potential news leads.

        Parameters:
        -----------
        topics_over_time : pd.DataFrame
            Topics over time from BERTopic
        min_comments : int
            Minimum comments to consider
        article_correlation_threshold : float
            Max article correlation to be considered "organic"

        Returns:
        --------
        pd.DataFrame
            Organic trending topics with article gaps
        """
        organic_trends = []

        # Get unique topics (exclude outliers and catch-all topic 0)
        unique_topics = [t for t in set(self.topics) if t not in (-1, 0)]

        for topic_id in unique_topics:
            # Get all comments in this topic
            topic_indices = [i for i, t in enumerate(self.topics) if t == topic_id]

            if len(topic_indices) < min_comments:
                continue

            # Get temporal distribution of comments
            topic_timestamps = [pd.to_datetime(self.timestamps[i]) for i in topic_indices]
            topic_timestamps_sorted = sorted(topic_timestamps)

            # Find peak activity period (last 30 days with most comments)
            if len(topic_timestamps_sorted) < 10:
                continue

            # Calculate rolling 30-day activity
            df_temp = pd.DataFrame({'timestamp': topic_timestamps_sorted})
            df_temp['count'] = 1
            df_temp = df_temp.set_index('timestamp')
            rolling = df_temp['count'].rolling('30D').sum()

            if rolling.empty or rolling.max() < min_comments:
                continue

            peak_time = rolling.idxmax()

            # Find articles published in peak period
            peak_start = peak_time - timedelta(days=15)
            peak_end = peak_time + timedelta(days=15)

            # Get unique articles in this topic
            topic_articles = list(set([self.article_ids[i] for i in topic_indices]))

            # Find article publication times (using first comment as proxy)
            article_times = []
            for art_id in topic_articles:
                art_comments = self.article_data[art_id]['timestamps']
                if art_comments:
                    article_times.append(pd.to_datetime(min(art_comments)))

            # Count articles published during peak
            articles_in_peak = sum(1 for t in article_times
                                 if peak_start <= t <= peak_end)

            # Calculate article correlation
            article_correlation = articles_in_peak / len(topic_articles) if topic_articles else 0

            # Organic = low article correlation during peak
            if article_correlation < article_correlation_threshold:
                # Get topic keywords
                topic_words = self.topic_model.get_topic(topic_id)
                keywords = [word for word, _ in topic_words[:10]] if topic_words else []

                organic_trends.append({
                    'topic_id': topic_id,
                    'peak_time': peak_time,
                    'total_comments': len(topic_indices),
                    'peak_period_comments': int(rolling.max()),
                    'total_articles': len(topic_articles),
                    'articles_in_peak': articles_in_peak,
                    'article_correlation': article_correlation,
                    'keywords': ', '.join(keywords),
                    'organic_score': 1 - article_correlation  # Higher = more organic
                })

        df = pd.DataFrame(organic_trends)

        if not df.empty:
            df = df.sort_values('organic_score', ascending=False)

        return df

    def generate_comprehensive_report(self, emerging_topics: pd.DataFrame = None,
                                     topics_over_time: pd.DataFrame = None) -> Dict:
        """
        Generate comprehensive evaluation report.

        Returns:
        --------
        Dict
            Complete evaluation results
        """
        report = {}

        # 1. Article Topic Coherence
        print("Calculating article topic coherence...")
        coherence = self.calculate_article_topic_coherence()
        report['article_coherence'] = coherence

        # 2. Emerging Topics Validation (if provided)
        if emerging_topics is not None and not emerging_topics.empty:
            print("Validating emerging topics...")
            validation = self.validate_emerging_topics(emerging_topics)
            report['emergence_validation'] = validation

        # 3. Organic Trends Detection (if topics_over_time provided)
        if topics_over_time is not None and not topics_over_time.empty:
            print("Detecting organic trends...")
            organic = self.detect_organic_trends(topics_over_time)
            report['organic_trends'] = organic

        return report

    def print_report(self, report: Dict):
        """Print formatted report."""
        print("\n" + "="*80)
        print("ARTICLE-BASED EVALUATION REPORT")
        print("="*80)

        # Article Coherence
        if 'article_coherence' in report:
            coh = report['article_coherence']
            print("\n📰 ARTICLE TOPIC COHERENCE")
            print("-" * 80)
            print(f"Articles Evaluated: {coh['articles_evaluated']}")
            print(f"Mean Coherence Score: {coh['mean_coherence']:.3f}")
            print(f"Median Coherence: {coh['median_coherence']:.3f}")
            print(f"Avg Topics per Article: {coh['mean_topics_per_article']:.1f}")
            print(f"\nInterpretation:")
            if coh['mean_coherence'] > 0.6:
                print("  ✓ HIGH coherence - Comments cluster well by article")
            elif coh['mean_coherence'] > 0.4:
                print("  ~ MODERATE coherence - Some cross-article discussion")
            else:
                print("  ✗ LOW coherence - Topics may be too broad/narrow")

        # Emergence Validation
        if 'emergence_validation' in report:
            val = report['emergence_validation']
            print("\n🔥 EMERGING TOPICS VALIDATION")
            print("-" * 80)
            print(f"Emerging Topics Analyzed: {val['total_emerging']}")
            print(f"Validated (>50% in window): {val['validated_count']}")
            print(f"Validation Rate: {val['validation_rate']*100:.1f}%")
            print(f"Mean Validation Score: {val['mean_validation_score']:.3f}")
            print(f"\nInterpretation:")
            if val['validation_rate'] > 0.7:
                print("  ✓ HIGH validation - Emerging topics correlate with articles")
            elif val['validation_rate'] > 0.4:
                print("  ~ MODERATE validation - Mix of article-driven and organic")
            else:
                print("  ⚠ LOW validation - Many topics emerge organically")

        # Organic Trends
        if 'organic_trends' in report:
            org = report['organic_trends']
            print("\n💡 ORGANIC TRENDING TOPICS (News Leads)")
            print("-" * 80)
            if len(org) > 0:
                print(f"Found {len(org)} organic trends\n")
                print("Top 5 Organic Trends:")
                for i, row in org.head(5).iterrows():
                    print(f"\n  Topic {row['topic_id']}: {row['keywords']}")
                    print(f"    Peak: {row['peak_time']}")
                    print(f"    Comments: {row['total_comments']} (peak: {row['peak_period_comments']})")
                    print(f"    Article correlation: {row['article_correlation']:.2f} (LOW = organic)")
                    print(f"    Organic score: {row['organic_score']:.3f}")
            else:
                print("No significant organic trends detected")

        print("\n" + "="*80)