Create_Reverse_Archive_v2.py

"""
OutlookMiner to NotebookLM: Chronological Text Refinery (Public Release)
Repository: https://github.com/sjcoker/OutlookMiner
"""
text_block ="""
Description:
This script is a post-processing companion tool for the OutlookMiner VBA macro. 
It ingests the fragmented, folder-based text extracts generated by OutlookMiner, 
cleans the data of LLM-breaking noise (like URLs and zero-width characters), 
sorts all emails into a single, continuous Reverse Chronological timeline, 
and chunks the output into specific file sizes optimized for AI ingestion 
(e.g., Google NotebookLM, Claude, ChatGPT).

Features:
- Strips all HTTP/HTTPS links and tracking URLs to prevent AI hallucination.
- Resolves Windows encoding artifacts and standardizes line breaks.
- Enforces a strict UTF-8 with BOM signature for seamless AI uploads.
- Allows the user to dynamically set the MB chunk size limit.
- Lets the user select their input and output directories via visual dialogs.
- Includes an easily accessible 'Custom Filters' section in the code for users to strip out their own specific boilerplate, legal footers, or repetitive signatures.

Requirements:
- Python 3.x
- Standard libraries only (os, re, datetime, tkinter)
"""
print(text_block)

import os
import re
from datetime import datetime
import tkinter as tk
from tkinter import filedialog, messagebox, simpledialog

def clean_content(content):
    # 1. Remove Soft Hyphens and Zero-Width characters FIRST
    # This heals "fragmented" URLs that have invisible line breaks in them
    content = re.sub(r'[\u00AD\u200B\u200C\u200D\u2060\uFEFF]', '', content)

    # 2. Convert exotic spaces to standard spaces
    content = re.sub(r'[\xA0\u2002\u2003\u2007\u2009\u200A]', ' ', content)

    # 3. Strip URLs entirely (now that they are solid, continuous strings)
    content = re.sub(r'<https?://[^>]+>', '', content)
    content = re.sub(r'https?://[^\s<>"]+|www\.[^\s<>"]+', '', content)

    # 4. Purge any residual [URL] placeholder tags
    content = re.sub(r'\[\s*URL\s*\]', '', content)
    
    # Deep clean null bytes and illegal control characters
    content = content.replace('\x00', '')
    content = re.sub(r'[\x01-\x08\x0B\x0C\x0E-\x1F\x7F]', '', content)

    # Remove mailto
    content = re.sub(r'mailto:[^\s<>"]+', '', content)

    # Clean up empty angle brackets left behind by URL/mailto stripping
    content = re.sub(r'<\s*>', '', content)

    # Remove Quoted-Printable Line Breaks
    content = re.sub(r'=\n', '\n', content)
    
    # Remove Missing Image Artifacts
    content = content.replace('Error! Filename not specified.', '')

    # ---------------------------------------------------------
    # OPTIONAL CUSTOM FILTERS (Uncomment and edit as needed)
    # ---------------------------------------------------------
    # Example 1: Purge massive instructional blocks & footers (e.g., DNA tests)
    # content = re.sub(r'Understanding your matches.*?please click here \.', '', content, flags=re.DOTALL)
    # content = re.sub(r'Explore\. Discover\. Connect\..*?address book\.', '', content, flags=re.DOTALL)
    
    # Example 2: Purge commercial legal footers (e.g., RetailMeNot)
    # content = re.sub(r'Need Help\?.*?USA\.', '', content, flags=re.DOTALL)
    # content = re.sub(r'This is a marketing email sent to.*?Unsubscribe \.', '', content, flags=re.DOTALL)
    # ---------------------------------------------------------
    
    # Neutralize Windows formatting
    content = content.replace('\r\n', '\n').replace('\r', '\n')
    
    # Strip trailing whitespace from each line
    content = re.sub(r'[ \t]+\n', '\n', content)
    
    # Consolidate multiple standard spaces into a single space
    content = re.sub(r' {2,}', ' ', content)
    
    # Collapse forwarding artifacts (e.g., >>> to > )
    content = re.sub(r'>{2,}', '> ', content)
    
    # Purge double newlines inside the header block
    content = content.replace('\n\nFrom:', '\nFrom:')
    content = content.replace('\n\nSent:', '\nSent:')
    content = content.replace('\n\nSubject:', '\nSubject:')
    
    # Unify and collapse all long dividers (-, =, *, _) to standard '---'
    content = re.sub(r'[-=*_]{10,}', '---', content)
    
    # Remove double dividers utilizing the new '---' standard
    content = content.replace('---\n\n---', '---')
    content = content.replace('---\n---', '---')
    
    # Collapse massive blocks of blank space
    content = re.sub(r'\n{3,}', '\n\n', content)
    content = re.sub(r'Subject: (.*?)\n\n---', r'Subject: \1\n\n---', content)
    
    return content
    
def main():
    root = tk.Tk()
    root.withdraw() # Hide the main window
    
    # --- Welcome & Instructions Screen ---
    welcome_msg = (
        "Welcome to the OutlookMiner Text Refinery!\n\n"
        "This tool prepares your extracted emails for AI analysis (like NotebookLM) by doing the following:\n\n"
        "1. Cleans LLM-breaking noise (strips URLs and tracking links).\n"
        "2. Sorts emails into a single Reverse-Chronological timeline.\n"
        "3. Chunks the output into specified sizes for use with AI.\n\n"
        "Click 'OK' to configure your extraction settings."
    )
    messagebox.showinfo("Instructions & Overview", welcome_msg)

    # Prompt user for the target MB chunk size
    chunk_mb = simpledialog.askfloat(
        "Set Chunk Size", 
        "Enter the target chunk size in MB (e.g., 3.0) for AI compatibility:", 
        initialvalue=3.0, 
        minvalue=0.1
    )
    
    # Gracefully exit if the user clicks 'Cancel'
    if chunk_mb is None:
        print("Operation cancelled by user at Chunk Size prompt.")
        return

    # Ask user to select the INPUT folder
    default_dir = os.path.expanduser("~/Documents")
    if not os.path.exists(default_dir):
        default_dir = "C:\\"
        
    input_dir = filedialog.askdirectory(
        initialdir=default_dir,
        title="Select the INPUT folder (OutlookMiner Extraction Folder)"
    )
    
    if not input_dir:
        print("Operation cancelled by user at Input Directory selection.")
        return
        
    # Ask user to select the OUTPUT parent folder
    output_parent_dir = filedialog.askdirectory(
        initialdir=input_dir,
        title="Select the OUTPUT destination for the refined chunks"
    )
    
    if not output_parent_dir:
        print("Operation cancelled by user at Output Directory selection.")
        return
    
    # Setup dynamic output folder based on today's date
    timestamp = datetime.now().strftime("%Y%m%d_%H%M")
    output_dir = os.path.join(output_parent_dir, f"NotebookLM_MasterArchive_NoURLs_{timestamp}")
    
    # --- Pre-Flight Check Screen ---
    flight_check_msg = (
        "PRE-FLIGHT SUMMARY\n\n"
        f"Target Chunk Size: {chunk_mb} MB\n\n"
        f"Input Source:\n{input_dir}\n\n"
        f"Output Destination:\n{output_dir}\n\n"
        "Click 'OK' to begin reading and processing the files. "
        "This may take a few minutes depending on the size of your archive."
    )
    
    if not messagebox.askokcancel("Ready to Process?", flight_check_msg):
        print("Operation cancelled by user at Pre-Flight Check.")
        return

    # Create the output directory only after the user confirms the flight check
    os.makedirs(output_dir, exist_ok=True)
    print(f"Selected Input: {input_dir}")
    print(f"Output Directory: {output_dir}")

    date_patterns = [
        '%m/%d/%Y %I:%M:%S %p',
        '%A, %B %d, %Y %I:%M %p',
        '%m/%d/%Y %I:%M %p',
        '%d %B %Y %H:%M'
    ]
    
    emails = []
    print("Reading and parsing all text files. This will take a few minutes...")
    
    total_files = 0
    for root_dir, dirs, files in os.walk(input_dir):
        for file in files:
            if file.endswith('.txt'):
                total_files += 1
                filepath = os.path.join(root_dir, file)
                try:
                    with open(filepath, 'rb') as f:
                        raw_bytes = f.read()
                        
                    if raw_bytes.startswith(b'\xef\xbb\xbf'):
                        raw_bytes = raw_bytes[3:]
                        
                    try:
                        content = raw_bytes.decode('utf-8')
                    except UnicodeDecodeError:
                        content = raw_bytes.decode('utf-8', errors='replace')
                        
                    parts = content.split('\nFrom: ')
                    for part in parts[1:]:
                        part = 'From: ' + part
                        
                        sent_match = re.search(r'^Sent:\s*(.*)$', part, re.MULTILINE)
                        if sent_match:
                            date_str = sent_match.group(1).strip()
                            date_str = date_str.replace('\u200e', '').strip()
                            
                            parsed_date = None
                            for fmt in date_patterns:
                                try:
                                    parsed_date = datetime.strptime(date_str, fmt)
                                    break
                                except ValueError:
                                    continue
                                    
                            if parsed_date:
                                emails.append({
                                    'date': parsed_date,
                                    'content': part
                                })
                except Exception as e:
                    print(f"Error processing {filepath}: {e}")
                    
    print(f"Finished reading {total_files} files.")
    print(f"Successfully parsed {len(emails)} individual emails.")
    
    if len(emails) == 0:
        print("No valid emails found to process.")
        messagebox.showwarning("No Data", "No valid text chunks were found in the selected input directory.")
        return

    print("Sorting globally by date (Newest to Oldest)...")
    emails.sort(key=lambda x: x['date'], reverse=True)

    print(f"Applying deep formatting clean and writing to {chunk_mb} MB chunks...")
    
    MAX_BYTES = int(chunk_mb * 1024 * 1024)
    file_index = 1
    current_size = 0
    
    out_file = open(os.path.join(output_dir, f"Reverse_Master_NoURLs_Part{file_index:03d}.txt"), 'wb')

    for email in emails:
        cleaned_content = clean_content(email['content'])
        
        # Prevent stacking dividers between emails
        cleaned_content = cleaned_content.strip() 
        if cleaned_content.endswith('---'):
            block = cleaned_content.replace('\n', '\r\n') + "\r\n"
        else:
            block = cleaned_content.replace('\n', '\r\n') + "\r\n---\r\n"
        
        if current_size == 0:
            out_file.write(b'\xef\xbb\xbf') # Write the exact Windows BOM
            
        encoded_block = block.encode('utf-8')
        
        if current_size + len(encoded_block) > MAX_BYTES:
            out_file.close()
            file_index += 1
            out_file = open(os.path.join(output_dir, f"Reverse_Master_NoURLs_Part{file_index:03d}.txt"), 'wb')
            out_file.write(b'\xef\xbb\xbf')
            current_size = 0
            
        out_file.write(encoded_block)
        current_size += len(encoded_block)
        
    out_file.close()
    print(f"\nSUCCESS! Created {file_index} files in:")
    print(output_dir)
    
    # --- Force the final popup to stay on top ---
    root.attributes('-topmost', True) 
    root.lift() 
    
    messagebox.showinfo("Extraction Complete", f"Successfully extracted, sorted, and stripped URLs from {len(emails)} emails into {file_index} chunks.\n\nSaved to:\n{output_dir}")

if __name__ == "__main__":
    main()