eda_features.py

import pandas as pd
import matplotlib.pyplot as plt
import numpy as np
from datetime import datetime

class EDAFeatureBuilder:
    def __init__(self):
        self.load_data()
        
    def load_data(self):
        """Load all created tables"""
        print("Loading data tables...")
        
        self.trades = pd.read_csv("data/trades.csv", parse_dates=["ts"])
        self.orders = pd.read_csv("data/orders.csv", parse_dates=["ts"])
        self.scans = pd.read_csv("data/scans.csv", parse_dates=["ts"])
        self.schedule = pd.read_csv("data/schedule.csv", parse_dates=["start_ts","end_ts"])
        
        print(f"Loaded: {self.trades.shape[0]} trades, {self.orders.shape[0]} orders, {self.scans.shape[0]} scans")
        print(f"Time range: {self.orders.ts.min()} to {self.orders.ts.max()}")
        
    def quick_eda(self):
        """Quick EDA checks"""
        print("\n=== Quick EDA ===")
        print(f"Trades head:\n{self.trades.head(3)}")
        print(f"\nOrders by hour:")
        print(self.orders.groupby(self.orders.ts.dt.hour)['order_id'].count())
        
        # Create simple plots
        fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(12, 4))
        
        # Orders per hour
        hourly_orders = self.orders.groupby(self.orders.ts.dt.hour)['order_id'].count()
        ax1.bar(hourly_orders.index, hourly_orders.values, color='steelblue')
        ax1.set_title('Orders by Hour')
        ax1.set_xlabel('Hour')
        ax1.set_ylabel('Number of Orders')
        
        # Token delta distribution
        token_deltas = self.trades[self.trades.trade_type == 'redemption']['tokens_delta']
        ax2.hist(token_deltas, bins=20, color='orange', alpha=0.7)
        ax2.set_title('Token Usage Distribution')
        ax2.set_xlabel('Tokens Delta')
        ax2.set_ylabel('Frequency')
        
        plt.tight_layout()
        plt.savefig('eda_plots.png', dpi=300, bbox_inches='tight')
        plt.show()
        
    def build_features(self):
        """Build minute-level features as specified"""
        print("\n=== Building Features ===")
        
        # Aggregate per minute per outlet
        trades_min = (self.trades.set_index("ts")
                     .groupby([pd.Grouper(freq="1min"), "outlet_id"])["tokens_delta"]
                     .sum().reset_index()
                     .rename(columns={"tokens_delta":"tokens_per_min"}))
        
        orders_min = (self.orders.set_index("ts")
                     .groupby([pd.Grouper(freq="1min"), "outlet_id"])["order_id"]
                     .count().reset_index()
                     .rename(columns={"order_id":"orders_per_min"}))
        
        scans_min = (self.scans.set_index("ts")
                    .groupby([pd.Grouper(freq="1min"), "outlet_id"])["count_since_last"]
                    .sum().reset_index()
                    .rename(columns={"count_since_last":"scans_per_min"}))
        
        # Join into feature table
        features = (trades_min.merge(orders_min, on=["ts","outlet_id"], how="outer")
                   .merge(scans_min, on=["ts","outlet_id"], how="outer")
                   .fillna(0))
        
        # Add time features
        features["minute_of_day"] = features.ts.dt.hour*60 + features.ts.dt.minute
        features["day_of_week"] = features.ts.dt.weekday
        
        # Rolling features
        features = features.sort_values(["outlet_id", "ts"])
        features["orders_5min"] = (features.groupby("outlet_id")["orders_per_min"]
                                  .rolling(5, min_periods=1).mean()
                                  .reset_index(0, drop=True))
        
        # Add class schedule features - classes ending in next 10 minutes
        features["classes_ending_10min"] = 0
        for idx, row in features.iterrows():
            current_time = row['ts']
            future_time = current_time + pd.Timedelta(minutes=10)
            
            # Count classes ending in this window
            ending_classes = self.schedule[
                (self.schedule['end_ts'] >= current_time) & 
                (self.schedule['end_ts'] <= future_time)
            ]
            features.loc[idx, 'classes_ending_10min'] = len(ending_classes)
        
        self.features = features
        print(f"Built features table: {len(features)} rows")
        print(f"Feature columns: {list(features.columns)}")
        
        # Save for model training
        self.features.to_csv('data/features.csv', index=False)
        return features
    
    def create_eda_heatmap(self):
        """Create heatmap by minute_of_day and day_of_week"""
        if not hasattr(self, 'features'):
            self.build_features()
            
        # Create pivot for heatmap
        heatmap_data = (self.features.groupby(['minute_of_day', 'day_of_week'])['orders_per_min']
                       .mean().unstack(fill_value=0))
        
        # Convert minute_of_day to hour labels for readability
        hour_labels = [f"{i//60:02d}:{i%60:02d}" for i in range(0, 1440, 60)]
        
        plt.figure(figsize=(10, 8))
        plt.imshow(heatmap_data.T, cmap='Blues', aspect='auto', interpolation='nearest')
        plt.colorbar(label='Average Orders per Minute')
        plt.title('Order Demand Heatmap')
        plt.xlabel('Hour of Day')
        plt.ylabel('Day of Week')
        plt.xticks(range(0, 1440, 60), [f"{i//60:02d}:00" for i in range(0, 1440, 60)], rotation=45)
        plt.yticks(range(7), ['Mon', 'Tue', 'Wed', 'Thu', 'Fri', 'Sat', 'Sun'])
        plt.tight_layout()
        plt.savefig('demand_heatmap.png', dpi=300, bbox_inches='tight')
        plt.show()
        
        print("Created demand heatmap")

def main():
    builder = EDAFeatureBuilder()
    builder.quick_eda()
    features = builder.build_features()
    builder.create_eda_heatmap()
    
    print(f"\nFeature summary:")
    print(features[['orders_per_min', 'tokens_per_min', 'scans_per_min', 'classes_ending_10min']].describe())

if __name__ == "__main__":
    main()