ekfiddle_scanner.py

import os
import subprocess
import re
import sys
import argparse
import hashlib
import base64
from html.parser import HTMLParser
from urllib.parse import urlparse, urljoin
from dataclasses import dataclass, field
from typing import List, Dict, Tuple, Optional, Set

def install_package(package_name):
    try:
        print(f"[*] Installing required package: {package_name}")
        subprocess.check_call([sys.executable, "-m", "pip", "install", package_name], 
                            stdout=subprocess.PIPE, stderr=subprocess.PIPE)
        print(f"[+] Successfully installed {package_name}")
        return True
    except subprocess.CalledProcessError:
        print(f"[!] Failed to install {package_name}. Please install manually: pip install {package_name}")
        return False

try:
    import requests
except ImportError:
    print("[!] 'requests' module not found. Attempting to install...")
    if not install_package("requests"):
        sys.exit(1)
    import requests

try:
    import urllib3
    urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
except ImportError:
    print("[!] 'urllib3' module not found. Attempting to install...")
    if not install_package("urllib3"):
        sys.exit(1)
    import urllib3
    urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)

# Flask is optional - only needed for API server mode
# Import is deferred to avoid requiring Flask for CLI-only usage
FLASK_AVAILABLE = False
try:
    from flask import Flask, request as flask_request, jsonify
    FLASK_AVAILABLE = True
except ImportError:
    pass  # Flask will be installed on-demand when --server is used

import threading
import uuid
import time
import json


# Default request headers for browser emulation
DEFAULT_HEADERS = {
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
    'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
    'Accept-Language': 'en-US,en;q=0.5',
    'Accept-Encoding': 'gzip, deflate',
    'Connection': 'keep-alive',
    'Upgrade-Insecure-Requests': '1',
}


@dataclass
class EKFiddleRule:
    """Represents a single EKFiddle detection rule"""
    rule_type: str  # SourceCode, URI, Headers, IP, Hash
    name: str
    pattern: str
    description: str = ""
    severity: str = "med"  # high, med, low
    has_and_condition: bool = False
    and_patterns: List[str] = field(default_factory=list)
    
    def __post_init__(self):
        # Extract severity from name if present
        name_lower = self.name.lower()
        if name_lower.startswith("high:"):
            self.severity = "high"
            self.name = self.name[5:].strip()
        elif name_lower.startswith("med:"):
            self.severity = "med"
            self.name = self.name[4:].strip()
        elif name_lower.startswith("low:"):
            self.severity = "low"
            self.name = self.name[4:].strip()
        
        # Check for AND conditions
        if " *AND* " in self.pattern:
            self.has_and_condition = True
            self.and_patterns = [p.strip() for p in self.pattern.split(" *AND* ")]


@dataclass
class Detection:
    """Represents a detection result"""
    rule: EKFiddleRule
    matches: List[str] = field(default_factory=list)
    match_contexts: List[str] = field(default_factory=list)
    source_url: str = ""  # Which resource this detection came from
    source_type: str = ""  # main_page, script, iframe, css, redirect, data_uri


@dataclass
class RedirectHop:
    """Represents a single redirect hop"""
    url: str
    status_code: int
    headers: Dict[str, str]
    content: Optional[str] = None


@dataclass
class ScannableContent:
    """Represents content that can be scanned"""
    url: str
    content_type: str  # main_page, script, iframe, css, data_uri, redirect, inline_script
    content: str
    headers: Dict[str, str] = field(default_factory=dict)
    source: str = ""  # Where this was referenced from


@dataclass
class ExtractedResources:
    """Resources extracted from HTML"""
    scripts: List[str] = field(default_factory=list)  # External script URLs
    iframes: List[str] = field(default_factory=list)  # Iframe URLs
    css: List[str] = field(default_factory=list)  # CSS URLs
    inline_scripts: List[str] = field(default_factory=list)  # Inline script content
    inline_styles: List[str] = field(default_factory=list)  # Inline style content
    data_uris: List[Tuple[str, str]] = field(default_factory=list)  # (type, decoded_content)
    imports: List[str] = field(default_factory=list)  # @import URLs


@dataclass
class ExtractedIOCs:
    """Indicators of Compromise extracted from content"""
    ipv4_addresses: Set[str] = field(default_factory=set)
    ipv6_addresses: Set[str] = field(default_factory=set)
    domains: Set[str] = field(default_factory=set)
    urls: Set[str] = field(default_factory=set)
    email_addresses: Set[str] = field(default_factory=set)
    bitcoin_addresses: Set[str] = field(default_factory=set)
    ethereum_addresses: Set[str] = field(default_factory=set)
    file_hashes_md5: Set[str] = field(default_factory=set)
    file_hashes_sha1: Set[str] = field(default_factory=set)
    file_hashes_sha256: Set[str] = field(default_factory=set)
    
    def is_empty(self) -> bool:
        """Check if no IOCs were found"""
        return not any([
            self.ipv4_addresses, self.ipv6_addresses, self.domains,
            self.urls, self.email_addresses, self.bitcoin_addresses,
            self.ethereum_addresses, self.file_hashes_md5,
            self.file_hashes_sha1, self.file_hashes_sha256
        ])
    
    def total_count(self) -> int:
        """Get total count of all IOCs"""
        return sum([
            len(self.ipv4_addresses), len(self.ipv6_addresses), len(self.domains),
            len(self.urls), len(self.email_addresses), len(self.bitcoin_addresses),
            len(self.ethereum_addresses), len(self.file_hashes_md5),
            len(self.file_hashes_sha1), len(self.file_hashes_sha256)
        ])


@dataclass  
class ExtractedPayloads:
    """Extracted malicious payloads and commands"""
    powershell_commands: List[str] = field(default_factory=list)
    powershell_encoded: List[Tuple[str, str]] = field(default_factory=list)  # (encoded, decoded)
    msiexec_commands: List[str] = field(default_factory=list)
    mshta_commands: List[str] = field(default_factory=list)
    curl_wget_commands: List[str] = field(default_factory=list)
    eval_content: List[str] = field(default_factory=list)
    download_urls: List[str] = field(default_factory=list)
    
    def is_empty(self) -> bool:
        """Check if no payloads were found"""
        return not any([
            self.powershell_commands, self.powershell_encoded,
            self.msiexec_commands, self.mshta_commands,
            self.curl_wget_commands, self.eval_content, self.download_urls
        ])


class ResourceExtractor(HTMLParser):
    """HTML Parser to extract resources from HTML content"""
    
    def __init__(self, base_url: str):
        super().__init__()
        self.base_url = base_url
        self.resources = ExtractedResources()
        self._in_script = False
        self._in_style = False
        self._current_script = []
        self._current_style = []
    
    def handle_starttag(self, tag, attrs):
        attrs_dict = dict(attrs)
        
        if tag == 'script':
            src = attrs_dict.get('src', '')
            if src:
                resolved = self._resolve_url(src)
                if resolved:
                    self.resources.scripts.append(resolved)
            else:
                # Inline script - capture content
                self._in_script = True
                self._current_script = []
        
        elif tag == 'iframe':
            src = attrs_dict.get('src', '')
            if src:
                resolved = self._resolve_url(src)
                if resolved:
                    self.resources.iframes.append(resolved)
        
        elif tag == 'link':
            rel = attrs_dict.get('rel', '').lower()
            href = attrs_dict.get('href', '')
            if 'stylesheet' in rel and href:
                resolved = self._resolve_url(href)
                if resolved:
                    self.resources.css.append(resolved)
        
        elif tag == 'style':
            self._in_style = True
            self._current_style = []
    
    def handle_endtag(self, tag):
        if tag == 'script' and self._in_script:
            self._in_script = False
            content = ''.join(self._current_script).strip()
            if content:
                self.resources.inline_scripts.append(content)
                # Check for data URIs in script content
                self._extract_data_uris(content)
            self._current_script = []
        
        elif tag == 'style' and self._in_style:
            self._in_style = False
            content = ''.join(self._current_style).strip()
            if content:
                self.resources.inline_styles.append(content)
                # Extract @import rules
                self._extract_imports(content)
            self._current_style = []
    
    def handle_data(self, data):
        if self._in_script:
            self._current_script.append(data)
        elif self._in_style:
            self._current_style.append(data)
    
    def _resolve_url(self, url: str) -> Optional[str]:
        """Resolve relative URL to absolute"""
        if not url:
            return None
        
        # Skip data URIs, javascript:, etc
        if url.startswith(('data:', 'javascript:', 'about:', '#')):
            if url.startswith('data:'):
                self._extract_single_data_uri(url)
            return None
        
        # Handle protocol-relative URLs
        if url.startswith('//'):
            parsed_base = urlparse(self.base_url)
            return f"{parsed_base.scheme}:{url}"
        
        # Already absolute
        if url.startswith(('http://', 'https://')):
            return url
        
        # Relative URL
        return urljoin(self.base_url, url)
    
    def _extract_single_data_uri(self, data_uri: str):
        """Extract and decode a single data URI"""
        decoded = decode_data_uri(data_uri)
        if decoded:
            content_type, content = decoded
            self.resources.data_uris.append((content_type, content))
    
    def _extract_data_uris(self, content: str):
        """Extract data URIs from content"""
        pattern = r'data:([^;,]+)(?:;base64)?,([^\s\"\'\)]+)'
        matches = re.findall(pattern, content)
        for mime_type, data in matches:
            try:
                if 'base64' in mime_type or len(data) > 100:
                    decoded = base64.b64decode(data).decode('utf-8', errors='ignore')
                    self.resources.data_uris.append((mime_type, decoded))
            except:
                pass
    
    def _extract_imports(self, css_content: str):
        """Extract @import URLs from CSS"""
        patterns = [
            r'@import\s+[\'\"](https?://[^\'\"]+)[\'\"]',
            r'@import\s+url\([\'\"](https?://[^\'\"]+)[\'\"]\)',
            r'@import\s+url\((https?://[^\)]+)\)',
        ]
        for pattern in patterns:
            matches = re.findall(pattern, css_content)
            for url in matches:
                resolved = self._resolve_url(url)
                if resolved:
                    self.resources.imports.append(resolved)


@dataclass
class FormInfo:
    """Information about a form found in HTML"""
    action_url: str = ""
    method: str = "GET"
    has_password_field: bool = False
    has_username_field: bool = False
    has_email_field: bool = False
    has_credit_card_field: bool = False
    hidden_fields: List[Tuple[str, str]] = field(default_factory=list)
    is_cross_origin: bool = False
    is_suspicious: bool = False
    suspicion_reasons: List[str] = field(default_factory=list)


class FormExtractor(HTMLParser):
    """HTML Parser to extract and analyze forms"""
    
    def __init__(self, base_url: str):
        super().__init__()
        self.base_url = base_url
        self.forms: List[FormInfo] = []
        self._current_form: Optional[FormInfo] = None
        
        # Parse base domain for cross-origin check
        parsed = urlparse(base_url)
        self._base_domain = parsed.netloc.lower()
    
    def handle_starttag(self, tag, attrs):
        attrs_dict = dict(attrs)
        
        if tag == 'form':
            self._current_form = FormInfo()
            action = attrs_dict.get('action', '')
            self._current_form.method = attrs_dict.get('method', 'GET').upper()
            
            # Resolve action URL
            if action:
                if action.startswith(('http://', 'https://')):
                    self._current_form.action_url = action
                elif action.startswith('//'):
                    parsed = urlparse(self.base_url)
                    self._current_form.action_url = f"{parsed.scheme}:{action}"
                else:
                    self._current_form.action_url = urljoin(self.base_url, action)
            else:
                self._current_form.action_url = self.base_url
            
            # Check cross-origin
            try:
                action_parsed = urlparse(self._current_form.action_url)
                action_domain = action_parsed.netloc.lower()
                if action_domain and action_domain != self._base_domain:
                    self._current_form.is_cross_origin = True
                    self._current_form.suspicion_reasons.append(f"Form submits to different domain: {action_domain}")
            except:
                pass
        
        elif tag == 'input' and self._current_form:
            input_type = attrs_dict.get('type', 'text').lower()
            input_name = attrs_dict.get('name', '').lower()
            input_id = attrs_dict.get('id', '').lower()
            input_value = attrs_dict.get('value', '')
            
            # Password fields
            if input_type == 'password':
                self._current_form.has_password_field = True
            
            # Username/email fields
            if input_type == 'email' or 'email' in input_name or 'email' in input_id:
                self._current_form.has_email_field = True
            
            if any(x in input_name or x in input_id for x in ['user', 'login', 'account', 'uname']):
                self._current_form.has_username_field = True
            
            # Credit card detection
            if any(x in input_name or x in input_id for x in ['card', 'cc', 'cvv', 'ccnum', 'cardnum', 'expir']):
                self._current_form.has_credit_card_field = True
            
            # Hidden fields
            if input_type == 'hidden' and input_name:
                self._current_form.hidden_fields.append((input_name, input_value))
    
    def handle_endtag(self, tag):
        if tag == 'form' and self._current_form:
            # Analyze form for suspicious patterns
            form = self._current_form
            
            if form.has_password_field:
                if form.is_cross_origin:
                    form.is_suspicious = True
                    form.suspicion_reasons.append("Login form submits credentials to different domain")
                
                # Check for suspicious action URLs
                action_lower = form.action_url.lower()
                suspicious_patterns = [
                    '.php', '/login', '/signin', '/auth', 
                    'verify', 'secure', 'update', 'confirm'
                ]
                if any(p in action_lower for p in suspicious_patterns):
                    # Not inherently suspicious, but note it
                    pass
            
            if form.has_credit_card_field:
                form.is_suspicious = True
                form.suspicion_reasons.append("Form collects credit card information")
                if form.is_cross_origin:
                    form.suspicion_reasons.append("Credit card form submits to different domain")
            
            if len(form.hidden_fields) > 3:
                form.suspicion_reasons.append(f"Form has {len(form.hidden_fields)} hidden fields")
            
            self.forms.append(form)
            self._current_form = None


def analyze_forms(html_content: str, base_url: str) -> List[FormInfo]:
    """Extract and analyze forms from HTML content"""
    parser = FormExtractor(base_url)
    try:
        parser.feed(html_content)
    except:
        pass
    return parser.forms


@dataclass
class JSRedirect:
    """JavaScript-based redirect detected in content"""
    redirect_type: str  # location, meta_refresh, document_write, etc.
    target_url: str
    code_snippet: str
    is_obfuscated: bool = False


def detect_js_redirects(content: str) -> List[JSRedirect]:
    """
    Detect JavaScript-based redirects in content.
    These are often used by malware to evade static analysis.
    """
    redirects = []
    
    if not content:
        return redirects
    
    # 1. window.location / document.location assignments
    location_patterns = [
        (r'(?:window|document)\.location\s*=\s*["\']([^"\']+)["\']', 'location_assign'),
        (r'(?:window|document)\.location\.href\s*=\s*["\']([^"\']+)["\']', 'location_href'),
        (r'location\.replace\s*\(\s*["\']([^"\']+)["\']\s*\)', 'location_replace'),
        (r'location\.assign\s*\(\s*["\']([^"\']+)["\']\s*\)', 'location_assign_method'),
        (r'window\.open\s*\(\s*["\']([^"\']+)["\']\s*\)', 'window_open'),
    ]
    
    for pattern, redirect_type in location_patterns:
        for match in re.finditer(pattern, content, re.IGNORECASE):
            url = match.group(1)
            if url.startswith(('http://', 'https://', '//')):
                redirects.append(JSRedirect(
                    redirect_type=redirect_type,
                    target_url=url,
                    code_snippet=match.group(0)[:100]
                ))
    
    # 2. Meta refresh detection
    meta_pattern = r'<meta[^>]+http-equiv\s*=\s*["\']?refresh["\']?[^>]+content\s*=\s*["\']?\d+\s*;\s*url\s*=\s*([^"\'>\s]+)'
    for match in re.finditer(meta_pattern, content, re.IGNORECASE):
        url = match.group(1)
        redirects.append(JSRedirect(
            redirect_type='meta_refresh',
            target_url=url,
            code_snippet=match.group(0)[:100]
        ))
    
    # 3. document.write with redirects
    doc_write_pattern = r'document\.write\s*\([^)]*(?:location|redirect|window\.open)[^)]*\)'
    for match in re.finditer(doc_write_pattern, content, re.IGNORECASE):
        redirects.append(JSRedirect(
            redirect_type='document_write',
            target_url='(dynamic)',
            code_snippet=match.group(0)[:100],
            is_obfuscated=True
        ))
    
    # 4. Obfuscated redirects (eval with location patterns)
    eval_redirect_pattern = r'eval\s*\([^)]*(?:location|href|window\.open)[^)]*\)'
    for match in re.finditer(eval_redirect_pattern, content, re.IGNORECASE):
        redirects.append(JSRedirect(
            redirect_type='eval_redirect',
            target_url='(obfuscated)',
            code_snippet=match.group(0)[:100],
            is_obfuscated=True
        ))
    
    # 5. setTimeout/setInterval with redirects
    timeout_pattern = r'set(?:Timeout|Interval)\s*\([^,)]*(?:location|window\.open)[^,)]*,'
    for match in re.finditer(timeout_pattern, content, re.IGNORECASE):
        redirects.append(JSRedirect(
            redirect_type='delayed_redirect',
            target_url='(delayed)',
            code_snippet=match.group(0)[:100]
        ))
    
    # 6. Form auto-submit (often used in phishing)
    auto_submit_pattern = r'(?:document\.forms\[0\]|form)\.submit\s*\(\s*\)'
    if re.search(auto_submit_pattern, content, re.IGNORECASE):
        # Check for onload trigger
        if re.search(r'(?:onload|DOMContentLoaded)[^{]*{[^}]*submit', content, re.IGNORECASE):
            redirects.append(JSRedirect(
                redirect_type='form_auto_submit',
                target_url='(form action)',
                code_snippet='Auto-submitting form on page load'
            ))
    
    return redirects


def decode_data_uri(data_uri: str) -> Optional[Tuple[str, str]]:
    """
    Decode a data URI and return (content_type, decoded_content).
    Handles base64 and plain text encoding.
    """
    if not data_uri.startswith('data:'):
        return None
    
    try:
        # Remove 'data:' prefix
        data_part = data_uri[5:]
        
        # Split on comma to separate metadata from data
        if ',' not in data_part:
            return None
        
        metadata, encoded_data = data_part.split(',', 1)
        
        # Parse metadata (e.g., "text/javascript;base64")
        is_base64 = ';base64' in metadata.lower()
        content_type = metadata.split(';')[0] if metadata else 'text/plain'
        
        if is_base64:
            # Decode base64
            decoded = base64.b64decode(encoded_data).decode('utf-8', errors='ignore')
        else:
            # URL decode
            from urllib.parse import unquote
            decoded = unquote(encoded_data)
        
        return (content_type, decoded)
    
    except Exception:
        return None


def extract_resources(html_content: str, base_url: str) -> ExtractedResources:
    """
    Extract all resources from HTML content.
    Returns ExtractedResources containing scripts, iframes, CSS, etc.
    """
    parser = ResourceExtractor(base_url)
    try:
        parser.feed(html_content)
    except Exception:
        pass  # Continue even if parsing fails
    
    # Also extract data URIs from the raw HTML using regex
    # (catches cases the parser might miss)
    data_uri_pattern = r'(data:(?:text/javascript|text/html|application/javascript)[^\"\'>\s]+)'
    additional_data_uris = re.findall(data_uri_pattern, html_content)
    for uri in additional_data_uris:
        decoded = decode_data_uri(uri)
        if decoded and decoded not in parser.resources.data_uris:
            parser.resources.data_uris.append(decoded)
    
    return parser.resources


def fetch_with_redirect_tracking(
    url: str,
    timeout: int = 30,
    max_redirects: int = 10,
    proxy: Optional[str] = None
) -> Tuple[Optional[str], Dict[str, str], List[RedirectHop], str]:
    """
    Fetch URL while tracking redirect chain.
    Returns: (final_content, final_headers, redirect_chain, final_url)
    """
    redirect_chain = []
    session = requests.Session()
    
    # Configure proxy - explicitly clear system proxies if none provided
    if proxy:
        proxies = {'http': proxy, 'https': proxy}
        session.proxies.update(proxies)
    else:
        # Clear any system/environment proxy settings
        session.proxies = {'http': None, 'https': None}
        session.trust_env = False
    
    current_url = url
    hop_count = 0
    
    while hop_count < max_redirects:
        try:
            response = session.get(
                current_url,
                headers=DEFAULT_HEADERS,
                timeout=timeout,
                verify=False,
                allow_redirects=False
            )
            
            hop = RedirectHop(
                url=current_url,
                status_code=response.status_code,
                headers=dict(response.headers),
                content=response.text if response.status_code == 200 else None
            )
            redirect_chain.append(hop)
            
            # Check for redirect
            if response.status_code in (301, 302, 303, 307, 308):
                location = response.headers.get('Location', '')
                if not location:
                    break
                
                # Handle relative redirects
                if not location.startswith(('http://', 'https://')):
                    location = urljoin(current_url, location)
                
                current_url = location
                hop_count += 1
            else:
                # Not a redirect, we're done
                break
                
        except requests.exceptions.SSLError:
            # Try HTTP if HTTPS fails
            if current_url.startswith('https://'):
                current_url = current_url.replace('https://', 'http://', 1)
                continue
            else:
                break
        except requests.exceptions.RequestException as e:
            print(f"[!] Error fetching {current_url}: {e}")
            break
        except Exception as e:
            print(f"[!] Unexpected error fetching {current_url}: {e}")
            break
    
    # Get final content
    if redirect_chain:
        last_hop = redirect_chain[-1]
        if last_hop.status_code == 200 and last_hop.content:
            return last_hop.content, last_hop.headers, redirect_chain, last_hop.url
    
    return None, {}, redirect_chain, url


def fetch_single_resource(url: str, timeout: int = 10, proxy: Optional[str] = None) -> Tuple[Optional[str], str]:
    """
    Fetch a single resource (script, CSS, etc.).
    Returns (content, status) where status is 'ok', 'timeout', 'error', or error message.
    Safe: only fetches text content, no execution.
    """
    # Configure proxy - explicitly bypass system proxies if none provided
    if proxy:
        proxies = {'http': proxy, 'https': proxy}
    else:
        proxies = {'http': None, 'https': None}
    
    try:
        session = requests.Session()
        session.trust_env = False if not proxy else True
        response = session.get(
            url,
            headers=DEFAULT_HEADERS,
            timeout=timeout,
            verify=False,
            allow_redirects=True,
            proxies=proxies
        )
        response.raise_for_status()
        return response.text, 'ok'
    except requests.exceptions.Timeout:
        return None, 'timeout'
    except requests.exceptions.ConnectionError:
        return None, 'connection_error'
    except requests.exceptions.HTTPError as e:
        return None, f'http_{e.response.status_code}'
    except Exception as e:
        return None, 'error'


def fetch_resources(
    resources: ExtractedResources,
    base_url: str,
    resource_timeout: int = 10,
    max_resources: int = 50,
    verbose: bool = False,
    proxy: Optional[str] = None
) -> Tuple[Dict[str, ScannableContent], Dict[str, str]]:
    """
    Fetch all external resources.
    Returns (fetched_dict, failed_dict) where:
        - fetched_dict: URL -> ScannableContent for successful fetches
        - failed_dict: URL -> reason for failed fetches
    Safe: Only fetches text content, no JavaScript/HTML execution.
    """
    fetched = {}
    failed = {}
    all_urls = []
    
    # Collect all resource URLs with their types (skip Google domains)
    skipped_count = 0
    for url in resources.scripts[:max_resources]:
        if is_skipped_domain(url):
            skipped_count += 1
            continue
        all_urls.append((url, 'script'))
    for url in resources.iframes[:max_resources]:
        if is_skipped_domain(url):
            skipped_count += 1
            continue
        all_urls.append((url, 'iframe'))
    for url in resources.css[:max_resources]:
        if is_skipped_domain(url):
            skipped_count += 1
            continue
        all_urls.append((url, 'css'))
    for url in resources.imports[:max_resources]:
        if is_skipped_domain(url):
            skipped_count += 1
            continue
        all_urls.append((url, 'css_import'))
    
    if skipped_count > 0 and verbose:
        print(f"[*] Skipped {skipped_count} Google/trusted CDN resource(s)")
    
    # Deduplicate while preserving order
    seen = set()
    unique_urls = []
    for url, rtype in all_urls:
        if url not in seen:
            seen.add(url)
            unique_urls.append((url, rtype))
    
    total = len(unique_urls[:max_resources])
    if total == 0:
        return fetched, failed
    
    # Fetch each resource with progress indicator
    success_count = 0
    fail_count = 0
    
    for i, (url, resource_type) in enumerate(unique_urls[:max_resources], 1):
        # Progress bar
        progress = f"[{i}/{total}]"
        url_short = url[:50] + "..." if len(url) > 50 else url
        
        # Show progress (overwrite line)
        sys.stdout.write(f"\r    {progress} Fetching {resource_type}: {url_short:<55}")
        sys.stdout.flush()
        
        content, status = fetch_single_resource(url, resource_timeout, proxy)
        
        if content and status == 'ok':
            fetched[url] = ScannableContent(
                url=url,
                content_type=resource_type,
                content=content,
                source=base_url
            )
            success_count += 1
            if verbose:
                print(f" -> {len(content)} bytes")
        else:
            failed[url] = status
            fail_count += 1
            if verbose:
                print(f" -> FAILED ({status})")
    
    # Clear the progress line and show summary
    sys.stdout.write("\r" + " " * 80 + "\r")
    sys.stdout.flush()
    
    if fail_count > 0:
        print(f"    [+] Fetched: {success_count}/{total} resources ({fail_count} failed)")
    else:
        print(f"    [+] Fetched: {success_count}/{total} resources")
    
    return fetched, failed


def build_content_buffer(
    main_content: str,
    main_url: str,
    main_headers: Dict[str, str],
    resources: ExtractedResources,
    fetched_resources: Dict[str, ScannableContent],
    redirect_chain: List[RedirectHop]
) -> List[ScannableContent]:
    """
    Build unified buffer of all scannable content.
    """
    buffer = []
    
    # Add redirect chain pages (excluding final page which is main_content)
    for i, hop in enumerate(redirect_chain[:-1] if len(redirect_chain) > 1 else []):
        if hop.content:
            buffer.append(ScannableContent(
                url=hop.url,
                content_type='redirect',
                content=hop.content,
                headers=hop.headers,
                source=f'redirect_hop_{i+1}'
            ))
    
    # Add main page content
    buffer.append(ScannableContent(
        url=main_url,
        content_type='main_page',
        content=main_content,
        headers=main_headers,
        source='main'
    ))
    
    # Add inline scripts
    for i, script_content in enumerate(resources.inline_scripts):
        buffer.append(ScannableContent(
            url=f"{main_url}#inline_script_{i+1}",
            content_type='inline_script',
            content=script_content,
            source=main_url
        ))
    
    # Add inline styles
    for i, style_content in enumerate(resources.inline_styles):
        buffer.append(ScannableContent(
            url=f"{main_url}#inline_style_{i+1}",
            content_type='inline_style',
            content=style_content,
            source=main_url
        ))
    
    # Add decoded data URIs
    for i, (content_type, content) in enumerate(resources.data_uris):
        buffer.append(ScannableContent(
            url=f"{main_url}#data_uri_{i+1}",
            content_type='data_uri',
            content=content,
            source=main_url
        ))
    
    # Add fetched external resources
    for url, scannable in fetched_resources.items():
        buffer.append(scannable)
    
    return buffer


def parse_ekfiddle_rules(filepath: str) -> List[EKFiddleRule]:
    """
    Parse EKFiddle format rules file.
    Format: Type TAB RuleName TAB Regex TAB OptionalDescription
    """
    rules = []
    
    if not os.path.exists(filepath):
        print(f"[!] Error: Rules file '{filepath}' not found")
        return rules
    
    with open(filepath, 'r', encoding='utf-8') as f:
        for line_num, line in enumerate(f, 1):
            line = line.rstrip('\n\r')
            
            # Skip empty lines
            if not line.strip():
                continue
            
            # Skip comments (lines starting with # or ##)
            if line.strip().startswith('#'):
                continue
            
            # Parse tab-separated fields
            parts = line.split('\t')
            
            if len(parts) < 3:
                continue
            
            rule_type = parts[0].strip()
            name = parts[1].strip()
            pattern = parts[2].strip()
            description = parts[3].strip() if len(parts) > 3 else ""
            
            # Validate rule type
            valid_types = ['SourceCode', 'URI', 'Headers', 'IP', 'Hash']
            if rule_type not in valid_types:
                continue
            
            # Skip empty patterns
            if not pattern:
                continue
            
            try:
                rule = EKFiddleRule(
                    rule_type=rule_type,
                    name=name,
                    pattern=pattern,
                    description=description
                )
                rules.append(rule)
            except Exception as e:
                print(f"[!] Warning: Failed to parse rule at line {line_num}: {e}")
                continue
    
    return rules


def check_and_pattern(rule: EKFiddleRule, content: str) -> Tuple[bool, List[str]]:
    """
    Check if all AND conditions in a rule match the content.
    Returns (matched, list_of_matched_strings)
    """
    if not rule.has_and_condition:
        # Single pattern
        try:
            matches = re.findall(rule.pattern, content, re.IGNORECASE | re.MULTILINE)
            if matches:
                # Flatten if tuples
                flat_matches = []
                for m in matches:
                    if isinstance(m, tuple):
                        flat_matches.append(m[0] if m[0] else str(m))
                    else:
                        flat_matches.append(str(m))
                return True, flat_matches
        except re.error:
            # Invalid regex, skip silently
            pass
        return False, []
    
    # AND condition - all patterns must match
    all_matches = []
    for pattern in rule.and_patterns:
        try:
            matches = re.findall(pattern, content, re.IGNORECASE | re.MULTILINE)
            if not matches:
                return False, []
            # Collect first match from each pattern
            if isinstance(matches[0], tuple):
                all_matches.append(matches[0][0] if matches[0][0] else str(matches[0]))
            else:
                all_matches.append(str(matches[0]))
        except re.error:
            return False, []
    
    return True, all_matches


def extract_urls_from_content(content: str) -> Set[str]:
    """Extract all URLs from HTML/JS content"""
    url_pattern = r'https?://[^\s\"\'\<\>\)\]\}]+'
    urls = set(re.findall(url_pattern, content, re.IGNORECASE))
    
    # Also extract from common JS patterns
    js_patterns = [
        r'src=[\'\"](https?://[^\'\"]+)[\'\"]',
        r'href=[\'\"](https?://[^\'\"]+)[\'\"]',
        r'url\([\'\"](https?://[^\'\"]+)[\'\"]\)',
    ]
    
    for pattern in js_patterns:
        matches = re.findall(pattern, content, re.IGNORECASE)
        urls.update(matches)
    
    return urls


def normalize_url(target: str) -> Optional[str]:
    """Normalize a domain or URL to a proper URL format"""
    target = target.strip()
    if not target:
        return None
    if not target.startswith('http://') and not target.startswith('https://'):
        target = 'https://' + target
    return target


def decode_content(content: str) -> str:
    """Decode common HTML entities and escape sequences"""
    if not content:
        return ""
    
    content_decoded = content
    content_decoded = content_decoded.replace('\\/', '/')
    content_decoded = content_decoded.replace('&amp;', '&')
    content_decoded = content_decoded.replace('&quot;', '"')
    content_decoded = content_decoded.replace('&#39;', "'")
    content_decoded = content_decoded.replace('&lt;', '<')
    content_decoded = content_decoded.replace('&gt;', '>')
    
    # Decode \xNN hex sequences manually to avoid deprecation warnings
    # This handles sequences like \x3c -> <, \x3e -> >, etc.
    def decode_hex_escape(match):
        try:
            return chr(int(match.group(1), 16))
        except (ValueError, OverflowError):
            return match.group(0)
    
    def decode_unicode_escape(match):
        try:
            return chr(int(match.group(1), 16))
        except (ValueError, OverflowError):
            return match.group(0)
    
    # Decode \xNN patterns (2 hex digits)
    content_decoded = re.sub(r'\\x([0-9a-fA-F]{2})', decode_hex_escape, content_decoded)
    
    # Decode \uNNNN patterns (4 hex digits)
    content_decoded = re.sub(r'\\u([0-9a-fA-F]{4})', decode_unicode_escape, content_decoded)
    
    return content_decoded


def deobfuscate_javascript(content: str) -> str:
    """
    Perform basic JavaScript deobfuscation.
    Handles common obfuscation techniques used by malware.
    """
    if not content:
        return ""
    
    result = content
    
    # 1. Decode hex escapes (\x41 -> A)
    def decode_hex(match):
        try:
            return chr(int(match.group(1), 16))
        except:
            return match.group(0)
    result = re.sub(r'\\x([0-9a-fA-F]{2})', decode_hex, result)
    
    # 2. Decode unicode escapes (\u0041 -> A)
    def decode_unicode(match):
        try:
            return chr(int(match.group(1), 16))
        except:
            return match.group(0)
    result = re.sub(r'\\u([0-9a-fA-F]{4})', decode_unicode, result)
    
    # 3. Decode String.fromCharCode patterns
    # Matches: String.fromCharCode(72,101,108,108,111) -> Hello
    def decode_fromcharcode(match):
        try:
            codes = match.group(1)
            chars = [chr(int(c.strip())) for c in codes.split(',') if c.strip().isdigit()]
            return '"' + ''.join(chars) + '"'
        except:
            return match.group(0)
    result = re.sub(r'String\.fromCharCode\s*\(\s*([0-9,\s]+)\s*\)', decode_fromcharcode, result)
    
    # 4. Resolve simple string concatenation ("a"+"b"+"c" -> "abc")
    # Only for adjacent string literals
    def resolve_concat(match):
        try:
            full = match.group(0)
            # Extract all quoted strings
            strings = re.findall(r'["\']([^"\']*)["\']', full)
            return '"' + ''.join(strings) + '"'
        except:
            return match.group(0)
    # Match patterns like "a"+"b" or 'a'+'b'
    result = re.sub(r'(["\'][^"\']*["\'])\s*\+\s*(["\'][^"\']*["\'])(?:\s*\+\s*["\'][^"\']*["\'])*', resolve_concat, result)
    
    # 5. Decode base64 strings (look for atob() calls)
    def decode_atob(match):
        try:
            b64 = match.group(1)
            decoded = base64.b64decode(b64).decode('utf-8', errors='ignore')
            return f'atob("{b64}") /* DECODED: {decoded[:100]} */'
        except:
            return match.group(0)
    result = re.sub(r'atob\s*\(\s*["\']([A-Za-z0-9+/=]+)["\']\s*\)', decode_atob, result)
    
    # 6. Detect and decode standalone base64 strings that look like commands/URLs
    def decode_suspicious_b64(match):
        try:
            b64 = match.group(1)
            if len(b64) < 20:  # Skip short strings
                return match.group(0)
            decoded = base64.b64decode(b64).decode('utf-8', errors='ignore')
            # Only add comment if it looks like a command or URL
            if any(x in decoded.lower() for x in ['http', 'powershell', 'cmd', 'script', '://']):
                return f'"{b64}" /* B64_DECODED: {decoded[:150]} */'
            return match.group(0)
        except:
            return match.group(0)
    # Look for base64 strings in quotes that are >= 20 chars
    result = re.sub(r'["\']([A-Za-z0-9+/]{20,}={0,2})["\']', decode_suspicious_b64, result)
    
    # 7. Decode HTML numeric entities (&#72;&#101;&#108;&#108;&#111; -> Hello)
    def decode_html_entity(match):
        try:
            code = match.group(1)
            return chr(int(code))
        except:
            return match.group(0)
    result = re.sub(r'&#(\d+);', decode_html_entity, result)
    
    # 8. Decode hex HTML entities (&#x48;&#x65;&#x6c;&#x6c;&#x6f; -> Hello)
    def decode_hex_entity(match):
        try:
            code = match.group(1)
            return chr(int(code, 16))
        except:
            return match.group(0)
    result = re.sub(r'&#x([0-9a-fA-F]+);', decode_hex_entity, result)
    
    return result


def compute_content_hash(content: str) -> str:
    """Compute SHA-256 hash of content"""
    return hashlib.sha256(content.encode('utf-8', errors='ignore')).hexdigest()


# IOC Extraction Patterns
IOC_PATTERNS = {
    'ipv4': r'\b(?:(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.){3}(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\b',
    'ipv6': r'\b(?:[0-9a-fA-F]{1,4}:){7}[0-9a-fA-F]{1,4}\b|\b(?:[0-9a-fA-F]{1,4}:){1,7}:\b|\b(?:[0-9a-fA-F]{1,4}:){1,6}:[0-9a-fA-F]{1,4}\b',
    'domain': r'\b(?:[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?\.)+(?:com|net|org|info|biz|xyz|top|online|site|club|ru|cn|tk|ml|ga|cf|gq|pw|cc|io|co|me|tv|ws|in|uk|de|fr|br|au|ca|nl|eu|ch|at|pl|se|no|fi|dk|be|it|es|pt|cz|hu|ro|bg|gr|tr|ua|kz|by|su|asia|pro|mobi|name|aero|jobs|museum|travel|xxx|edu|gov|mil)\b',
    'url': r'https?://[^\s<>\"\'\)]+',
    'email': r'\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Z|a-z]{2,}\b',
    'bitcoin': r'\b[13][a-km-zA-HJ-NP-Z1-9]{25,34}\b|\bbc1[a-zA-HJ-NP-Z0-9]{39,59}\b',
    'ethereum': r'\b0x[a-fA-F0-9]{40}\b',
    'md5': r'\b[a-fA-F0-9]{32}\b',
    'sha1': r'\b[a-fA-F0-9]{40}\b',
    'sha256': r'\b[a-fA-F0-9]{64}\b',
}

# Common private/reserved IP ranges to exclude
PRIVATE_IP_RANGES = [
    r'^10\.',
    r'^172\.(1[6-9]|2[0-9]|3[01])\.',
    r'^192\.168\.',
    r'^127\.',
    r'^0\.',
    r'^169\.254\.',
    r'^224\.',
    r'^255\.',
]

# Common non-malicious domains to exclude from IOC extraction
BENIGN_DOMAINS = {
    'google.com', 'googleapis.com', 'gstatic.com', 'googleusercontent.com',
    'facebook.com', 'fbcdn.net', 'twitter.com', 'twimg.com',
    'microsoft.com', 'windows.net', 'azure.com', 'live.com', 'office.com',
    'apple.com', 'icloud.com', 'amazon.com', 'amazonaws.com', 'cloudfront.net',
    'jquery.com', 'jsdelivr.net', 'cloudflare.com', 'cdnjs.cloudflare.com',
    'bootstrapcdn.com', 'unpkg.com', 'npmjs.com',
    'youtube.com', 'ytimg.com', 'vimeo.com',
    'linkedin.com', 'instagram.com', 'pinterest.com',
    'wordpress.org', 'wordpress.com', 'wp.com', 'gravatar.com',
    'w3.org', 'schema.org', 'example.com', 'localhost',
}


def is_private_ip(ip: str) -> bool:
    """Check if IP is in private/reserved range"""
    for pattern in PRIVATE_IP_RANGES:
        if re.match(pattern, ip):
            return True
    return False


def is_benign_domain(domain: str) -> bool:
    """Check if domain is in benign list"""
    domain_lower = domain.lower()
    for benign in BENIGN_DOMAINS:
        if domain_lower == benign or domain_lower.endswith('.' + benign):
            return True
    return False


def extract_iocs(content: str, include_benign: bool = False) -> ExtractedIOCs:
    """
    Extract all IOCs from content.
    
    Args:
        content: Text content to extract IOCs from
        include_benign: If True, include known benign domains/IPs
    
    Returns:
        ExtractedIOCs dataclass with all found indicators
    """
    iocs = ExtractedIOCs()
    
    if not content:
        return iocs
    
    # Extract IPv4 addresses
    for match in re.finditer(IOC_PATTERNS['ipv4'], content):
        ip = match.group(0)
        if include_benign or not is_private_ip(ip):
            iocs.ipv4_addresses.add(ip)
    
    # Extract IPv6 addresses
    for match in re.finditer(IOC_PATTERNS['ipv6'], content):
        iocs.ipv6_addresses.add(match.group(0))
    
    # Extract URLs
    for match in re.finditer(IOC_PATTERNS['url'], content):
        url = match.group(0).rstrip('.,;:\'\")')
        try:
            parsed = urlparse(url)
            if parsed.netloc:
                if include_benign or not is_benign_domain(parsed.netloc):
                    iocs.urls.add(url)
        except:
            pass
    
    # Extract domains
    for match in re.finditer(IOC_PATTERNS['domain'], content):
        domain = match.group(0).lower()
        if include_benign or not is_benign_domain(domain):
            iocs.domains.add(domain)
    
    # Extract email addresses
    for match in re.finditer(IOC_PATTERNS['email'], content):
        email = match.group(0).lower()
        domain = email.split('@')[1] if '@' in email else ''
        if include_benign or not is_benign_domain(domain):
            iocs.email_addresses.add(email)
    
    # Extract Bitcoin addresses
    for match in re.finditer(IOC_PATTERNS['bitcoin'], content):
        iocs.bitcoin_addresses.add(match.group(0))
    
    # Extract Ethereum addresses
    for match in re.finditer(IOC_PATTERNS['ethereum'], content):
        iocs.ethereum_addresses.add(match.group(0))
    
    # Extract file hashes (be careful - these patterns can match other hex strings)
    # Only extract if they look like standalone hashes (not part of URLs, etc.)
    hash_context_pattern = r'(?:^|[\s\"\'\=\:\,])([a-fA-F0-9]{32,64})(?:$|[\s\"\'\,])'
    for match in re.finditer(hash_context_pattern, content):
        hex_str = match.group(1)
        if len(hex_str) == 32:
            iocs.file_hashes_md5.add(hex_str.lower())
        elif len(hex_str) == 40:
            iocs.file_hashes_sha1.add(hex_str.lower())
        elif len(hex_str) == 64:
            iocs.file_hashes_sha256.add(hex_str.lower())
    
    return iocs


def merge_iocs(iocs_list: List[ExtractedIOCs]) -> ExtractedIOCs:
    """Merge multiple ExtractedIOCs into one"""
    merged = ExtractedIOCs()
    for iocs in iocs_list:
        merged.ipv4_addresses.update(iocs.ipv4_addresses)
        merged.ipv6_addresses.update(iocs.ipv6_addresses)
        merged.domains.update(iocs.domains)
        merged.urls.update(iocs.urls)
        merged.email_addresses.update(iocs.email_addresses)
        merged.bitcoin_addresses.update(iocs.bitcoin_addresses)
        merged.ethereum_addresses.update(iocs.ethereum_addresses)
        merged.file_hashes_md5.update(iocs.file_hashes_md5)
        merged.file_hashes_sha1.update(iocs.file_hashes_sha1)
        merged.file_hashes_sha256.update(iocs.file_hashes_sha256)
    return merged


def extract_payloads(content: str) -> ExtractedPayloads:
    """
    Extract malicious payloads and commands from content.
    Focuses on ClickFix-style attacks and common malware delivery methods.
    """
    payloads = ExtractedPayloads()
    
    if not content:
        return payloads
    
    # PowerShell commands
    ps_patterns = [
        r'powershell(?:\.exe)?\s+[^\n\r]{10,500}',
        r'pwsh(?:\.exe)?\s+[^\n\r]{10,500}',
    ]
    for pattern in ps_patterns:
        for match in re.finditer(pattern, content, re.IGNORECASE):
            cmd = match.group(0).strip()
            if cmd not in payloads.powershell_commands:
                payloads.powershell_commands.append(cmd)
    
    # PowerShell encoded commands (-EncodedCommand, -enc, -e)
    encoded_ps_pattern = r'powershell[^\n]*?(?:-enc(?:odedcommand)?|-e)\s+([A-Za-z0-9+/=]{20,})'
    for match in re.finditer(encoded_ps_pattern, content, re.IGNORECASE):
        encoded = match.group(1)
        try:
            # PowerShell uses UTF-16LE encoding
            decoded = base64.b64decode(encoded).decode('utf-16-le', errors='ignore')
            decoded = decoded.strip().replace('\x00', '')
            if decoded:
                payloads.powershell_encoded.append((encoded[:50] + '...', decoded))
        except:
            pass
    
    # MSIEXEC commands
    msiexec_pattern = r'msiexec(?:\.exe)?(?:\s+/[a-z]+)*\s+(?:/i|/package)\s+[^\s\n\r\"\']+(?:https?://[^\s\n\r\"\']+|[^\s\n\r\"\']+\.msi)'
    for match in re.finditer(msiexec_pattern, content, re.IGNORECASE):
        cmd = match.group(0).strip()
        if cmd not in payloads.msiexec_commands:
            payloads.msiexec_commands.append(cmd)
    
    # Alternative MSIEXEC pattern
    msiexec_alt_pattern = r'msiexec\s+/i\s+https?://[^\s\"\'\)]+\.msi'
    for match in re.finditer(msiexec_alt_pattern, content, re.IGNORECASE):
        cmd = match.group(0).strip()
        if cmd not in payloads.msiexec_commands:
            payloads.msiexec_commands.append(cmd)
    
    # MSHTA commands
    mshta_pattern = r'mshta(?:\.exe)?\s+[^\n\r]{10,300}'
    for match in re.finditer(mshta_pattern, content, re.IGNORECASE):
        cmd = match.group(0).strip()
        if cmd not in payloads.mshta_commands:
            payloads.mshta_commands.append(cmd)
    
    # Curl/wget commands
    curl_wget_pattern = r'(?:curl|wget)(?:\.exe)?\s+[^\n\r]{10,500}'
    for match in re.finditer(curl_wget_pattern, content, re.IGNORECASE):
        cmd = match.group(0).strip()
        if cmd not in payloads.curl_wget_commands:
            payloads.curl_wget_commands.append(cmd)
    
    # JavaScript eval() content
    eval_pattern = r'eval\s*\(\s*[\"\'`]([^\"\'`]{10,500})[\"\'`]\s*\)'
    for match in re.finditer(eval_pattern, content):
        eval_str = match.group(1)
        if eval_str not in payloads.eval_content:
            payloads.eval_content.append(eval_str)
    
    # Download URLs (suspicious file extensions)
    download_pattern = r'https?://[^\s\"\'\)<>]+\.(?:exe|msi|dll|bat|cmd|ps1|vbs|js|hta|scr|pif|jar|zip|rar|7z)\b'
    for match in re.finditer(download_pattern, content, re.IGNORECASE):
        url = match.group(0).rstrip('.,;:')
        if url not in payloads.download_urls:
            payloads.download_urls.append(url)
    
    return payloads


def merge_payloads(payloads_list: List[ExtractedPayloads]) -> ExtractedPayloads:
    """Merge multiple ExtractedPayloads into one"""
    merged = ExtractedPayloads()
    for payloads in payloads_list:
        for cmd in payloads.powershell_commands:
            if cmd not in merged.powershell_commands:
                merged.powershell_commands.append(cmd)
        for item in payloads.powershell_encoded:
            if item not in merged.powershell_encoded:
                merged.powershell_encoded.append(item)
        for cmd in payloads.msiexec_commands:
            if cmd not in merged.msiexec_commands:
                merged.msiexec_commands.append(cmd)
        for cmd in payloads.mshta_commands:
            if cmd not in merged.mshta_commands:
                merged.mshta_commands.append(cmd)
        for cmd in payloads.curl_wget_commands:
            if cmd not in merged.curl_wget_commands:
                merged.curl_wget_commands.append(cmd)
        for content in payloads.eval_content:
            if content not in merged.eval_content:
                merged.eval_content.append(content)
        for url in payloads.download_urls:
            if url not in merged.download_urls:
                merged.download_urls.append(url)
    return merged


def export_iocs_csv(iocs: ExtractedIOCs, filepath: str):
    """Export IOCs to CSV format"""
    import csv
    with open(filepath, 'w', newline='', encoding='utf-8') as f:
        writer = csv.writer(f)
        writer.writerow(['Type', 'Value'])
        for ip in sorted(iocs.ipv4_addresses):
            writer.writerow(['IPv4', ip])
        for ip in sorted(iocs.ipv6_addresses):
            writer.writerow(['IPv6', ip])
        for domain in sorted(iocs.domains):
            writer.writerow(['Domain', domain])
        for url in sorted(iocs.urls):
            writer.writerow(['URL', url])
        for email in sorted(iocs.email_addresses):
            writer.writerow(['Email', email])
        for addr in sorted(iocs.bitcoin_addresses):
            writer.writerow(['Bitcoin', addr])
        for addr in sorted(iocs.ethereum_addresses):
            writer.writerow(['Ethereum', addr])
        for h in sorted(iocs.file_hashes_md5):
            writer.writerow(['MD5', h])
        for h in sorted(iocs.file_hashes_sha1):
            writer.writerow(['SHA1', h])
        for h in sorted(iocs.file_hashes_sha256):
            writer.writerow(['SHA256', h])


def export_iocs_json(iocs: ExtractedIOCs) -> dict:
    """Export IOCs to JSON-compatible dict"""
    return {
        'ipv4_addresses': sorted(list(iocs.ipv4_addresses)),
        'ipv6_addresses': sorted(list(iocs.ipv6_addresses)),
        'domains': sorted(list(iocs.domains)),
        'urls': sorted(list(iocs.urls)),
        'email_addresses': sorted(list(iocs.email_addresses)),
        'bitcoin_addresses': sorted(list(iocs.bitcoin_addresses)),
        'ethereum_addresses': sorted(list(iocs.ethereum_addresses)),
        'file_hashes': {
            'md5': sorted(list(iocs.file_hashes_md5)),
            'sha1': sorted(list(iocs.file_hashes_sha1)),
            'sha256': sorted(list(iocs.file_hashes_sha256)),
        }
    }


def export_iocs_stix(iocs: ExtractedIOCs, source_url: str) -> dict:
    """Export IOCs to STIX 2.1 format"""
    import json
    from datetime import datetime
    
    stix_bundle = {
        'type': 'bundle',
        'id': f'bundle--{hashlib.md5(source_url.encode()).hexdigest()}',
        'objects': []
    }
    
    timestamp = datetime.utcnow().strftime('%Y-%m-%dT%H:%M:%S.000Z')
    
    # Add indicator for each IOC
    for ip in iocs.ipv4_addresses:
        stix_bundle['objects'].append({
            'type': 'indicator',
            'id': f'indicator--{hashlib.md5(ip.encode()).hexdigest()}',
            'created': timestamp,
            'modified': timestamp,
            'pattern': f"[ipv4-addr:value = '{ip}']",
            'pattern_type': 'stix',
            'valid_from': timestamp,
            'labels': ['malicious-activity']
        })
    
    for domain in iocs.domains:
        stix_bundle['objects'].append({
            'type': 'indicator',
            'id': f'indicator--{hashlib.md5(domain.encode()).hexdigest()}',
            'created': timestamp,
            'modified': timestamp,
            'pattern': f"[domain-name:value = '{domain}']",
            'pattern_type': 'stix',
            'valid_from': timestamp,
            'labels': ['malicious-activity']
        })
    
    for url in iocs.urls:
        stix_bundle['objects'].append({
            'type': 'indicator',
            'id': f'indicator--{hashlib.md5(url.encode()).hexdigest()}',
            'created': timestamp,
            'modified': timestamp,
            'pattern': f"[url:value = '{url}']",
            'pattern_type': 'stix',
            'valid_from': timestamp,
            'labels': ['malicious-activity']
        })
    
    for h in iocs.file_hashes_sha256:
        stix_bundle['objects'].append({
            'type': 'indicator',
            'id': f'indicator--{hashlib.md5(h.encode()).hexdigest()}',
            'created': timestamp,
            'modified': timestamp,
            'pattern': f"[file:hashes.'SHA-256' = '{h}']",
            'pattern_type': 'stix',
            'valid_from': timestamp,
            'labels': ['malicious-activity']
        })
    
    return stix_bundle


@dataclass
class DomainEnrichment:
    """Domain enrichment data from DNS and WHOIS lookups"""
    domain: str
    ip_addresses: List[str] = field(default_factory=list)
    dns_records: Dict[str, List[str]] = field(default_factory=dict)
    whois_registrar: str = ""
    whois_creation_date: str = ""
    whois_expiration_date: str = ""
    domain_age_days: int = -1
    is_newly_registered: bool = False  # < 30 days
    error: str = ""


def enrich_domain(domain: str) -> DomainEnrichment:
    """
    Enrich domain with DNS and basic WHOIS information.
    Uses built-in modules to avoid external dependencies.
    """
    import socket
    from datetime import datetime
    
    result = DomainEnrichment(domain=domain)
    
    # Clean domain (remove protocol, path, port)
    try:
        parsed = urlparse(domain if '://' in domain else f'http://{domain}')
        clean_domain = parsed.netloc or parsed.path
        clean_domain = clean_domain.split(':')[0]  # Remove port
        result.domain = clean_domain
    except:
        clean_domain = domain
    
    # DNS A record lookup
    try:
        ips = socket.gethostbyname_ex(clean_domain)
        result.ip_addresses = ips[2] if ips[2] else []
        result.dns_records['A'] = result.ip_addresses
    except socket.gaierror as e:
        result.error = f"DNS lookup failed: {e}"
    except Exception as e:
        result.error = f"DNS error: {e}"
    
    # Try WHOIS lookup using socket connection
    try:
        whois_data = _simple_whois_lookup(clean_domain)
        if whois_data:
            # Parse basic WHOIS fields
            for line in whois_data.split('\n'):
                line_lower = line.lower()
                if 'registrar:' in line_lower:
                    result.whois_registrar = line.split(':', 1)[1].strip()
                elif 'creation date:' in line_lower or 'created:' in line_lower:
                    date_str = line.split(':', 1)[1].strip()
                    result.whois_creation_date = date_str
                    # Try to parse date and calculate age
                    try:
                        for fmt in ['%Y-%m-%dT%H:%M:%S', '%Y-%m-%d', '%d-%b-%Y']:
                            try:
                                created = datetime.strptime(date_str[:10], fmt[:len(date_str[:10])])
                                age = (datetime.now() - created).days
                                result.domain_age_days = age
                                result.is_newly_registered = age < 30
                                break
                            except:
                                continue
                    except:
                        pass
                elif 'expir' in line_lower and 'date' in line_lower:
                    result.whois_expiration_date = line.split(':', 1)[1].strip()
    except Exception as e:
        if not result.error:
            result.error = f"WHOIS lookup failed: {e}"
    
    return result


@dataclass
class CertificateInfo:
    """SSL/TLS certificate information"""
    subject: str = ""
    issuer: str = ""
    valid_from: str = ""
    valid_until: str = ""
    days_until_expiry: int = -1
    is_expired: bool = False
    is_self_signed: bool = False
    is_recently_issued: bool = False  # < 30 days
    san_domains: List[str] = field(default_factory=list)
    issuer_org: str = ""
    error: str = ""


def get_certificate_info(domain: str, port: int = 443) -> CertificateInfo:
    """
    Get SSL/TLS certificate information for a domain.
    """
    import ssl
    import socket
    from datetime import datetime
    
    result = CertificateInfo()
    
    # Clean domain
    try:
        parsed = urlparse(domain if '://' in domain else f'https://{domain}')
        clean_domain = parsed.netloc or parsed.path
        clean_domain = clean_domain.split(':')[0]
    except:
        clean_domain = domain
    
    try:
        context = ssl.create_default_context()
        context.check_hostname = False
        context.verify_mode = ssl.CERT_NONE  # Allow self-signed
        
        with socket.create_connection((clean_domain, port), timeout=10) as sock:
            with context.wrap_socket(sock, server_hostname=clean_domain) as ssock:
                cert = ssock.getpeercert(binary_form=False)
                
                if not cert:
                    # Get binary cert for self-signed
                    cert_bin = ssock.getpeercert(binary_form=True)
                    if cert_bin:
                        result.error = "Certificate details not available (possibly self-signed)"
                        result.is_self_signed = True
                    return result
                
                # Parse subject
                subject = dict(x[0] for x in cert.get('subject', []))
                result.subject = subject.get('commonName', '')
                
                # Parse issuer
                issuer = dict(x[0] for x in cert.get('issuer', []))
                result.issuer = issuer.get('commonName', '')
                result.issuer_org = issuer.get('organizationName', '')
                
                # Check if self-signed
                if result.subject == result.issuer:
                    result.is_self_signed = True
                
                # Parse validity dates
                not_before = cert.get('notBefore', '')
                not_after = cert.get('notAfter', '')
                
                result.valid_from = not_before
                result.valid_until = not_after
                
                # Calculate days until expiry
                try:
                    # SSL date format: 'Sep 15 00:00:00 2023 GMT'
                    expiry = datetime.strptime(not_after, '%b %d %H:%M:%S %Y %Z')
                    issued = datetime.strptime(not_before, '%b %d %H:%M:%S %Y %Z')
                    
                    result.days_until_expiry = (expiry - datetime.now()).days
                    result.is_expired = result.days_until_expiry < 0
                    
                    # Check if recently issued (< 30 days)
                    days_since_issue = (datetime.now() - issued).days
                    result.is_recently_issued = days_since_issue < 30
                except:
                    pass
                
                # Get Subject Alternative Names
                san = cert.get('subjectAltName', [])
                result.san_domains = [name for typ, name in san if typ == 'DNS']
                
    except ssl.SSLError as e:
        result.error = f"SSL error: {e}"
    except socket.timeout:
        result.error = "Connection timeout"
    except socket.gaierror as e:
        result.error = f"DNS resolution failed: {e}"
    except Exception as e:
        result.error = f"Certificate check failed: {e}"
    
    return result


def _simple_whois_lookup(domain: str) -> str:
    """
    Perform a simple WHOIS lookup using socket.
    Returns raw WHOIS data or empty string on failure.
    """
    import socket
    
    # Determine WHOIS server based on TLD
    tld = domain.split('.')[-1].lower()
    whois_servers = {
        'com': 'whois.verisign-grs.com',
        'net': 'whois.verisign-grs.com',
        'org': 'whois.pir.org',
        'info': 'whois.afilias.net',
        'io': 'whois.nic.io',
        'co': 'whois.nic.co',
        'me': 'whois.nic.me',
        'biz': 'whois.biz',
        'us': 'whois.nic.us',
        'uk': 'whois.nic.uk',
        'de': 'whois.denic.de',
        'ru': 'whois.tcinet.ru',
    }
    
    whois_server = whois_servers.get(tld, f'whois.nic.{tld}')
    
    try:
        sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
        sock.settimeout(5)
        sock.connect((whois_server, 43))
        sock.send((domain + '\r\n').encode())
        
        response = b''
        while True:
            data = sock.recv(4096)
            if not data:
                break
            response += data
        
        sock.close()
        return response.decode('utf-8', errors='ignore')
    except:
        return ""


def generate_report_markdown(
    detections: List[Detection],
    source_url: str,
    redirect_chain: List[RedirectHop],
    content_buffer: List[ScannableContent],
    failed_resources: Dict[str, str] = None,
    extracted_iocs: Optional[ExtractedIOCs] = None,
    extracted_payloads: Optional[ExtractedPayloads] = None,
    defang: bool = True
) -> str:
    """Generate a Markdown report of scan results"""
    from datetime import datetime
    
    if failed_resources is None:
        failed_resources = {}
    
    lines = []
    lines.append("# EKFiddle Threat Scanner Report")
    lines.append(f"\nGenerated: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}")
    lines.append("")
    
    # Summary
    lines.append("## Executive Summary")
    lines.append("")
    source_display = defang_url(source_url) if defang else source_url
    lines.append(f"- Target: {source_display}")
    
    severity_counts = {'high': 0, 'med': 0, 'low': 0}
    for d in detections:
        severity_counts[d.rule.severity] = severity_counts.get(d.rule.severity, 0) + 1
    
    if detections:
        threat_level = "HIGH RISK" if severity_counts['high'] > 0 else ("MEDIUM RISK" if severity_counts['med'] > 0 else "LOW RISK")
    else:
        threat_level = "CLEAN"
    
    lines.append(f"- Threat Level: {threat_level}")
    lines.append(f"- Detections: {len(detections)} (HIGH: {severity_counts['high']}, MED: {severity_counts['med']}, LOW: {severity_counts['low']})")
    lines.append(f"- Resources Scanned: {len(content_buffer)}")
    lines.append("")
    
    # Redirect Chain
    if len(redirect_chain) > 1:
        lines.append("## Redirect Chain")
        lines.append("")
        for i, hop in enumerate(redirect_chain, 1):
            hop_url = defang_url(hop.url) if defang else hop.url
            lines.append(f"{i}. [{hop.status_code}] {hop_url}")
        lines.append("")
    
    # Detections
    if detections:
        lines.append("## Detections")
        lines.append("")
        
        by_source = {}
        for d in detections:
            key = (d.source_url, d.source_type)
            if key not in by_source:
                by_source[key] = []
            by_source[key].append(d)
        
        for (src_url, src_type), dets in by_source.items():
            src_display = defang_url(src_url) if defang else src_url
            lines.append(f"### [{src_type.upper()}] {src_display}")
            lines.append("")
            
            for det in dets:
                lines.append(f"- [{det.rule.severity.upper()}] {det.rule.name}")
                if det.rule.description:
                    lines.append(f"  - Description: {det.rule.description}")
                if det.matches:
                    lines.append(f"  - Matches: {len(det.matches)}")
                    for m in det.matches[:3]:
                        m_display = truncate_match(m, 80)
                        if defang:
                            m_display = defang_string(m_display)
                        lines.append(f"    - {m_display}")
            lines.append("")
    
    # Payloads
    if extracted_payloads and not extracted_payloads.is_empty():
        lines.append("## Extracted Payloads")
        lines.append("")
        
        if extracted_payloads.powershell_encoded:
            lines.append("### Decoded PowerShell Commands")
            lines.append("")
            for enc, dec in extracted_payloads.powershell_encoded[:5]:
                dec_display = defang_string(dec) if defang else dec
                lines.append(f"```")
                lines.append(dec_display[:500])
                lines.append(f"```")
                lines.append("")
        
        if extracted_payloads.download_urls:
            lines.append("### Suspicious Download URLs")
            lines.append("")
            for url in extracted_payloads.download_urls[:10]:
                url_display = defang_url(url) if defang else url
                lines.append(f"- {url_display}")
            lines.append("")
    
    # IOCs
    if extracted_iocs and not extracted_iocs.is_empty():
        lines.append("## Extracted IOCs")
        lines.append("")
        
        if extracted_iocs.ipv4_addresses:
            lines.append("### IPv4 Addresses")
            lines.append("")
            for ip in sorted(list(extracted_iocs.ipv4_addresses))[:20]:
                lines.append(f"- {ip}")
            lines.append("")
        
        if extracted_iocs.domains:
            lines.append("### Domains")
            lines.append("")
            for domain in sorted(list(extracted_iocs.domains))[:20]:
                lines.append(f"- {defang_url(domain) if defang else domain}")
            lines.append("")
        
        if extracted_iocs.urls:
            lines.append("### URLs")
            lines.append("")
            for url in sorted(list(extracted_iocs.urls))[:15]:
                lines.append(f"- {defang_url(url) if defang else url}")
            lines.append("")
    
    # Resources
    lines.append("## Resources Analyzed")
    lines.append("")
    external_items = [i for i in content_buffer if i.content_type in ('script', 'iframe', 'css')]
    lines.append(f"- External Resources: {len(external_items)}")
    lines.append(f"- Failed Resources: {len(failed_resources)}")
    lines.append("")
    
    return "\n".join(lines)


def generate_report_html(
    detections: List[Detection],
    source_url: str,
    redirect_chain: List[RedirectHop],
    content_buffer: List[ScannableContent],
    failed_resources: Dict[str, str] = None,
    extracted_iocs: Optional[ExtractedIOCs] = None,
    extracted_payloads: Optional[ExtractedPayloads] = None,
    defang: bool = True
) -> str:
    """Generate an HTML report of scan results"""
    from datetime import datetime
    
    if failed_resources is None:
        failed_resources = {}
    
    severity_counts = {'high': 0, 'med': 0, 'low': 0}
    for d in detections:
        severity_counts[d.rule.severity] = severity_counts.get(d.rule.severity, 0) + 1
    
    if detections:
        threat_level = "HIGH RISK" if severity_counts['high'] > 0 else ("MEDIUM RISK" if severity_counts['med'] > 0 else "LOW RISK")
        threat_color = "#dc3545" if severity_counts['high'] > 0 else ("#ffc107" if severity_counts['med'] > 0 else "#28a745")
    else:
        threat_level = "CLEAN"
        threat_color = "#28a745"
    
    source_display = defang_url(source_url) if defang else source_url
    
    html = f'''<!DOCTYPE html>
<html lang="en">
<head>
    <meta charset="UTF-8">
    <meta name="viewport" content="width=device-width, initial-scale=1.0">
    <title>EKFiddle Threat Scanner Report</title>
    <style>
        body {{ font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, sans-serif; margin: 40px; background: #f5f5f5; }}
        .container {{ max-width: 1200px; margin: 0 auto; background: white; padding: 30px; border-radius: 8px; box-shadow: 0 2px 4px rgba(0,0,0,0.1); }}
        h1 {{ color: #333; border-bottom: 2px solid #007bff; padding-bottom: 10px; }}
        h2 {{ color: #555; margin-top: 30px; }}
        .summary {{ background: #f8f9fa; padding: 20px; border-radius: 8px; margin: 20px 0; }}
        .threat-level {{ font-size: 24px; font-weight: bold; color: {threat_color}; }}
        .detection {{ background: #fff3cd; border-left: 4px solid #ffc107; padding: 15px; margin: 10px 0; }}
        .detection.high {{ background: #f8d7da; border-left-color: #dc3545; }}
        .detection.med {{ background: #fff3cd; border-left-color: #ffc107; }}
        .detection.low {{ background: #d1ecf1; border-left-color: #17a2b8; }}
        .severity {{ font-weight: bold; padding: 2px 8px; border-radius: 4px; color: white; }}
        .severity.high {{ background: #dc3545; }}
        .severity.med {{ background: #ffc107; color: #333; }}
        .severity.low {{ background: #17a2b8; }}
        .match {{ font-family: monospace; background: #e9ecef; padding: 5px 10px; margin: 5px 0; border-radius: 4px; word-break: break-all; }}
        .ioc-list {{ column-count: 2; column-gap: 20px; }}
        .ioc-list li {{ break-inside: avoid; }}
        code {{ background: #e9ecef; padding: 2px 6px; border-radius: 3px; font-family: monospace; }}
        .timestamp {{ color: #666; font-size: 14px; }}
    </style>
</head>
<body>
    <div class="container">
        <h1>EKFiddle Threat Scanner Report</h1>
        <p class="timestamp">Generated: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}</p>
        
        <div class="summary">
            <h2>Executive Summary</h2>
            <p><strong>Target:</strong> <code>{source_display}</code></p>
            <p><strong>Threat Level:</strong> <span class="threat-level">{threat_level}</span></p>
            <p><strong>Detections:</strong> {len(detections)} total (HIGH: {severity_counts['high']}, MED: {severity_counts['med']}, LOW: {severity_counts['low']})</p>
            <p><strong>Resources Scanned:</strong> {len(content_buffer)}</p>
        </div>
'''
    
    # Detections
    if detections:
        html += '<h2>Detections</h2>\n'
        for d in sorted(detections, key=lambda x: {'high': 0, 'med': 1, 'low': 2}.get(x.rule.severity, 3)):
            sev = d.rule.severity
            src_display = defang_url(d.source_url) if defang else d.source_url
            html += f'<div class="detection {sev}">\n'
            html += f'<span class="severity {sev}">{sev.upper()}</span> <strong>{d.rule.name}</strong>\n'
            html += f'<p><small>Source: {d.source_type} - {truncate_url(src_display, 80)}</small></p>\n'
            if d.rule.description:
                html += f'<p>{d.rule.description}</p>\n'
            if d.matches:
                html += '<div class="match">'
                m = truncate_match(d.matches[0], 200)
                if defang:
                    m = defang_string(m)
                html += m
                if len(d.matches) > 1:
                    html += f' <small>(+{len(d.matches)-1} more matches)</small>'
                html += '</div>\n'
            html += '</div>\n'
    else:
        html += '<h2>Detections</h2>\n<p>No threats detected.</p>\n'
    
    # Payloads
    if extracted_payloads and not extracted_payloads.is_empty():
        html += '<h2>Extracted Payloads</h2>\n'
        if extracted_payloads.download_urls:
            html += '<h3>Suspicious Download URLs</h3>\n<ul>\n'
            for url in extracted_payloads.download_urls[:10]:
                url_display = defang_url(url) if defang else url
                html += f'<li><code>{url_display}</code></li>\n'
            html += '</ul>\n'
    
    # IOCs
    if extracted_iocs and not extracted_iocs.is_empty():
        html += '<h2>Extracted IOCs</h2>\n'
        if extracted_iocs.ipv4_addresses:
            html += f'<h3>IPv4 Addresses ({len(extracted_iocs.ipv4_addresses)})</h3>\n<ul class="ioc-list">\n'
            for ip in sorted(list(extracted_iocs.ipv4_addresses))[:20]:
                html += f'<li><code>{ip}</code></li>\n'
            html += '</ul>\n'
        if extracted_iocs.domains:
            html += f'<h3>Domains ({len(extracted_iocs.domains)})</h3>\n<ul class="ioc-list">\n'
            for domain in sorted(list(extracted_iocs.domains))[:20]:
                html += f'<li><code>{defang_url(domain) if defang else domain}</code></li>\n'
            html += '</ul>\n'
    
    html += '''
    </div>
</body>
</html>'''
    
    return html


def scan_single_content(
    scannable: ScannableContent,
    rules: List[EKFiddleRule],
    verbose: bool = False,
    deobfuscate: bool = True
) -> List[Detection]:
    """
    Scan a single ScannableContent item with all rules.
    """
    detections = []
    content = scannable.content
    if not content:
        return detections
    
    # Apply basic decoding
    content_decoded = decode_content(content)
    
    # Apply JavaScript deobfuscation for script content
    if deobfuscate and scannable.content_type in ('script', 'inline_script', 'main_page', 'data_uri'):
        content_decoded = deobfuscate_javascript(content_decoded)
    
    content_hash = compute_content_hash(content)
    
    # Extract URLs for URI rules (from both original and deobfuscated content)
    urls_in_content = extract_urls_from_content(content_decoded)
    urls_in_content.add(scannable.url)
    
    # Build headers string for Headers rules
    headers_str = "\n".join(f"{k}: {v}" for k, v in scannable.headers.items())
    
    for rule in rules:
        matched = False
        matches = []
        
        try:
            if rule.rule_type == 'SourceCode':
                matched, matches = check_and_pattern(rule, content_decoded)
                
            elif rule.rule_type == 'URI':
                for url in urls_in_content:
                    url_matched, url_matches = check_and_pattern(rule, url)
                    if url_matched:
                        matched = True
                        matches.extend(url_matches if url_matches else [url])
                
            elif rule.rule_type == 'Headers':
                matched, matches = check_and_pattern(rule, headers_str)
                
            elif rule.rule_type == 'IP':
                matched, matches = check_and_pattern(rule, content_decoded)
                
            elif rule.rule_type == 'Hash':
                if rule.pattern.lower() in content_hash.lower():
                    matched = True
                    matches = [content_hash]
            
            if matched and matches:
                unique_matches = list(dict.fromkeys(matches))
                detection = Detection(
                    rule=rule,
                    matches=unique_matches,
                    source_url=scannable.url,
                    source_type=scannable.content_type
                )
                detections.append(detection)
                
                if verbose:
                    print(f"    [+] {scannable.content_type}: [{rule.severity.upper()}] {rule.name}")
                    
        except Exception as e:
            if verbose:
                print(f"    [!] Error with rule '{rule.name}': {e}")
            continue
    
    return detections


def scan_content_buffer(
    content_buffer: List[ScannableContent],
    rules: List[EKFiddleRule],
    verbose: bool = False
) -> List[Detection]:
    """
    Scan all content in the buffer with rules.
    """
    all_detections = []
    
    for scannable in content_buffer:
        detections = scan_single_content(scannable, rules, verbose)
        all_detections.extend(detections)
    
    return all_detections


def defang_url(url: str) -> str:
    """Defang a URL for safe display/storage"""
    return url.replace(".", "[.]").replace("http", "hxxp")


# Google-related domains to skip scanning (reduces noise from trusted CDNs)
SKIP_DOMAINS = (
    'googleapis.com',
    'google.com',
    'google-analytics.com',
    'googletagmanager.com',
    'googlesyndication.com',
    'googleadservices.com',
    'gstatic.com',
    'googleusercontent.com',
    'youtube.com',
    'ytimg.com',
    'ggpht.com',
    'doubleclick.net',
    'google.co.',
    'recaptcha.net',
)


def is_skipped_domain(url: str) -> bool:
    """Check if URL belongs to a domain we should skip scanning"""
    try:
        parsed = urlparse(url)
        host = parsed.netloc.lower()
        for domain in SKIP_DOMAINS:
            if host == domain or host.endswith('.' + domain) or ('.' + domain) in host:
                return True
        return False
    except Exception:
        return False


def defang_string(s: str) -> str:
    """Defang URLs and IPs in a string"""
    result = re.sub(r'(https?)', r'hxxp', s, flags=re.IGNORECASE)
    result = re.sub(r'\.(?=[a-zA-Z])', '[.]', result)
    return result


def truncate_match(match: str, max_length: int = 100) -> str:
    """Truncate a match string for display"""
    if len(match) <= max_length:
        return match
    return match[:max_length] + "..."


def truncate_url(url: str, max_length: int = 70) -> str:
    """Truncate URL for display"""
    if len(url) <= max_length:
        return url
    return url[:max_length] + "..."


def output_results_enhanced(
    detections: List[Detection],
    source_url: str,
    redirect_chain: List[RedirectHop],
    content_buffer: List[ScannableContent],
    failed_resources: Dict[str, str] = None,
    defang: bool = True,
    output_file: Optional[str] = None,
    show_matches: bool = True,
    show_redirects: bool = True,
    extracted_iocs: Optional[ExtractedIOCs] = None,
    extracted_payloads: Optional[ExtractedPayloads] = None
):
    """Output results grouped by resource source"""
    
    if failed_resources is None:
        failed_resources = {}
    
    print("\n" + "=" * 70)
    print("THREAT DETECTION RESULTS")
    print("=" * 70)
    
    source_display = defang_url(source_url) if defang else source_url
    print(f"\nSource: {source_display}")
    
    # Show redirect chain
    if show_redirects and len(redirect_chain) > 1:
        print(f"\nRedirect Chain:")
        for i, hop in enumerate(redirect_chain, 1):
            hop_url = defang_url(hop.url) if defang else hop.url
            print(f"  {i}. {truncate_url(hop_url, 65)} -> {hop.status_code}")
    
    # Show all external resources loaded (full list)
    external_types = ('script', 'iframe', 'css', 'css_import')
    external_items = [item for item in content_buffer if item.content_type in external_types]
    
    if external_items:
        print(f"\nExternal Resources Loaded: {len(external_items)}")
        for item in external_items:
            item_url = defang_url(item.url) if defang else item.url
            print(f"  - {truncate_url(item_url, 65)} ({item.content_type})")
    
    # Show failed resources if any
    if failed_resources:
        print(f"\nFailed Resources: {len(failed_resources)}")
        for url, reason in list(failed_resources.items())[:10]:
            url_display = defang_url(url) if defang else url
            print(f"  - {truncate_url(url_display, 60)} ({reason})")
        if len(failed_resources) > 10:
            print(f"  ... and {len(failed_resources) - 10} more")
    
    print("-" * 70)
    
    if not detections:
        print("\n[*] No threats detected")
        print("\n" + "-" * 70)
        print("Summary: 0 rules triggered")
        print("=" * 70)
        return
    
    # Detections header - make it stand out
    print("\n" + "*" * 70)
    print("*" + " " * 20 + "!! DETECTIONS FOUND !!" + " " * 24 + "*")
    print("*" * 70)
    
    # Group detections by source
    by_source = {}
    for d in detections:
        key = (d.source_url, d.source_type)
        if key not in by_source:
            by_source[key] = []
        by_source[key].append(d)
    
    output_lines = []
    
    # Sort sources: main_page first, then others
    source_order = {'main_page': 0, 'redirect': 1, 'inline_script': 2, 'data_uri': 3, 'script': 4, 'iframe': 5, 'css': 6}
    sorted_sources = sorted(by_source.keys(), key=lambda x: (source_order.get(x[1], 99), x[0]))
    
    for source_url, source_type in sorted_sources:
        source_detections = by_source[(source_url, source_type)]
        
        # Section header
        type_label = source_type.upper().replace('_', ' ')
        source_display = defang_url(source_url) if defang else source_url
        print(f"\n[{type_label}] {truncate_url(source_display, 65)}")
        
        # Sort by severity
        severity_order = {'high': 0, 'med': 1, 'low': 2}
        source_detections.sort(key=lambda d: (severity_order.get(d.rule.severity, 3), d.rule.name))
        
        for detection in source_detections:
            rule = detection.rule
            severity_label = f"[{rule.severity.upper()}]"
            
            print(f"\n  {severity_label} {rule.name}")
            print(f"    Type: {rule.rule_type}")
            
            if rule.description:
                print(f"    Description: {rule.description}")
            
            if show_matches and detection.matches:
                print("    Matches:")
                for match in detection.matches[:5]:
                    match_display = truncate_match(match)
                    if defang:
                        match_display = defang_string(match_display)
                    print(f"      - {match_display}")
                
                if len(detection.matches) > 5:
                    print(f"      ... and {len(detection.matches) - 5} more")
            
            # For file output
            output_lines.append(f"{source_type}|{source_url}|{rule.severity.upper()}|{rule.name}")
            for match in detection.matches:
                output_lines.append(f"  MATCH: {match}")
    
    # Display extracted payloads if available
    if extracted_payloads and not extracted_payloads.is_empty():
        print("\n" + "-" * 70)
        print("EXTRACTED PAYLOADS")
        print("-" * 70)
        
        if extracted_payloads.powershell_commands:
            print(f"\nPowerShell Commands ({len(extracted_payloads.powershell_commands)}):")
            for cmd in extracted_payloads.powershell_commands[:5]:
                cmd_display = truncate_match(cmd, 150)
                if defang:
                    cmd_display = defang_string(cmd_display)
                print(f"  - {cmd_display}")
            if len(extracted_payloads.powershell_commands) > 5:
                print(f"  ... and {len(extracted_payloads.powershell_commands) - 5} more")
        
        if extracted_payloads.powershell_encoded:
            print(f"\nDecoded PowerShell ({len(extracted_payloads.powershell_encoded)}):")
            for encoded, decoded in extracted_payloads.powershell_encoded[:3]:
                decoded_display = truncate_match(decoded, 200)
                if defang:
                    decoded_display = defang_string(decoded_display)
                print(f"  [DECODED] {decoded_display}")
            if len(extracted_payloads.powershell_encoded) > 3:
                print(f"  ... and {len(extracted_payloads.powershell_encoded) - 3} more")
        
        if extracted_payloads.msiexec_commands:
            print(f"\nMSIEXEC Commands ({len(extracted_payloads.msiexec_commands)}):")
            for cmd in extracted_payloads.msiexec_commands[:5]:
                cmd_display = truncate_match(cmd, 150)
                if defang:
                    cmd_display = defang_string(cmd_display)
                print(f"  - {cmd_display}")
        
        if extracted_payloads.mshta_commands:
            print(f"\nMSHTA Commands ({len(extracted_payloads.mshta_commands)}):")
            for cmd in extracted_payloads.mshta_commands[:5]:
                cmd_display = truncate_match(cmd, 150)
                if defang:
                    cmd_display = defang_string(cmd_display)
                print(f"  - {cmd_display}")
        
        if extracted_payloads.download_urls:
            print(f"\nSuspicious Download URLs ({len(extracted_payloads.download_urls)}):")
            for url in extracted_payloads.download_urls[:10]:
                url_display = defang_url(url) if defang else url
                print(f"  - {url_display}")
            if len(extracted_payloads.download_urls) > 10:
                print(f"  ... and {len(extracted_payloads.download_urls) - 10} more")
    
    # Display extracted IOCs if available
    if extracted_iocs and not extracted_iocs.is_empty():
        print("\n" + "-" * 70)
        print("EXTRACTED IOCs")
        print("-" * 70)
        
        if extracted_iocs.ipv4_addresses:
            print(f"\nIPv4 Addresses ({len(extracted_iocs.ipv4_addresses)}):")
            for ip in sorted(list(extracted_iocs.ipv4_addresses))[:15]:
                print(f"  - {ip}")
            if len(extracted_iocs.ipv4_addresses) > 15:
                print(f"  ... and {len(extracted_iocs.ipv4_addresses) - 15} more")
        
        if extracted_iocs.domains:
            print(f"\nDomains ({len(extracted_iocs.domains)}):")
            for domain in sorted(list(extracted_iocs.domains))[:15]:
                domain_display = defang_url(domain) if defang else domain
                print(f"  - {domain_display}")
            if len(extracted_iocs.domains) > 15:
                print(f"  ... and {len(extracted_iocs.domains) - 15} more")
        
        if extracted_iocs.urls:
            print(f"\nURLs ({len(extracted_iocs.urls)}):")
            for url in sorted(list(extracted_iocs.urls))[:10]:
                url_display = defang_url(url) if defang else url
                print(f"  - {truncate_url(url_display, 80)}")
            if len(extracted_iocs.urls) > 10:
                print(f"  ... and {len(extracted_iocs.urls) - 10} more")
        
        if extracted_iocs.email_addresses:
            print(f"\nEmail Addresses ({len(extracted_iocs.email_addresses)}):")
            for email in sorted(list(extracted_iocs.email_addresses))[:10]:
                print(f"  - {email}")
        
        if extracted_iocs.bitcoin_addresses:
            print(f"\nBitcoin Addresses ({len(extracted_iocs.bitcoin_addresses)}):")
            for addr in sorted(list(extracted_iocs.bitcoin_addresses))[:5]:
                print(f"  - {addr}")
        
        if extracted_iocs.ethereum_addresses:
            print(f"\nEthereum Addresses ({len(extracted_iocs.ethereum_addresses)}):")
            for addr in sorted(list(extracted_iocs.ethereum_addresses))[:5]:
                print(f"  - {addr}")
        
        if extracted_iocs.file_hashes_sha256:
            print(f"\nSHA-256 Hashes ({len(extracted_iocs.file_hashes_sha256)}):")
            for h in sorted(list(extracted_iocs.file_hashes_sha256))[:5]:
                print(f"  - {h}")
    
    # Summary
    print("\n" + "-" * 70)
    
    severity_counts = {'high': 0, 'med': 0, 'low': 0}
    for d in detections:
        severity_counts[d.rule.severity] = severity_counts.get(d.rule.severity, 0) + 1
    
    high_count = severity_counts.get('high', 0)
    med_count = severity_counts.get('med', 0)
    low_count = severity_counts.get('low', 0)
    
    print(f"Summary: {len(detections)} rule(s) triggered across {len(by_source)} resource(s)")
    print(f"  HIGH: {high_count} | MED: {med_count} | LOW: {low_count}")
    
    if extracted_iocs and not extracted_iocs.is_empty():
        print(f"  IOCs Extracted: {extracted_iocs.total_count()}")
    
    if extracted_payloads and not extracted_payloads.is_empty():
        payload_count = (len(extracted_payloads.powershell_commands) + 
                        len(extracted_payloads.powershell_encoded) +
                        len(extracted_payloads.msiexec_commands) + 
                        len(extracted_payloads.download_urls))
        print(f"  Payloads Extracted: {payload_count}")
    
    print("=" * 70)
    
    if output_file:
        with open(output_file, 'a') as f:
            f.write(f"\n# Source: {source_url}\n")
            for line in output_lines:
                f.write(line + '\n')
        print(f"\n[+] Results appended to: {output_file}")


def output_results_json(
    detections: List[Detection],
    source_url: str,
    redirect_chain: List[RedirectHop],
    content_buffer: List[ScannableContent],
    failed_resources: Dict[str, str] = None,
    extracted_iocs: Optional[ExtractedIOCs] = None,
    extracted_payloads: Optional[ExtractedPayloads] = None,
    defang: bool = True
):
    """Output results in JSON format"""
    import json
    
    if failed_resources is None:
        failed_resources = {}
    
    result = {
        'source_url': defang_url(source_url) if defang else source_url,
        'redirect_chain': [
            {
                'url': defang_url(hop.url) if defang else hop.url,
                'status_code': hop.status_code
            }
            for hop in redirect_chain
        ],
        'resources_loaded': len([i for i in content_buffer if i.content_type in ('script', 'iframe', 'css', 'css_import')]),
        'resources_failed': len(failed_resources),
        'detections': [],
        'summary': {
            'total_detections': len(detections),
            'high': 0,
            'med': 0,
            'low': 0
        }
    }
    
    # Add detections
    for d in detections:
        result['summary'][d.rule.severity] = result['summary'].get(d.rule.severity, 0) + 1
        detection_obj = {
            'rule_name': d.rule.name,
            'rule_type': d.rule.rule_type,
            'severity': d.rule.severity,
            'source_url': defang_url(d.source_url) if defang else d.source_url,
            'source_type': d.source_type,
            'matches': [defang_string(m) if defang else m for m in d.matches[:10]]
        }
        if d.rule.description:
            detection_obj['description'] = d.rule.description
        result['detections'].append(detection_obj)
    
    # Add IOCs if extracted
    if extracted_iocs and not extracted_iocs.is_empty():
        result['iocs'] = export_iocs_json(extracted_iocs)
    
    # Add payloads if extracted
    if extracted_payloads and not extracted_payloads.is_empty():
        result['payloads'] = {
            'powershell_commands': [defang_string(c) if defang else c for c in extracted_payloads.powershell_commands],
            'powershell_decoded': [
                {'encoded': e[:50], 'decoded': defang_string(d) if defang else d}
                for e, d in extracted_payloads.powershell_encoded
            ],
            'msiexec_commands': [defang_string(c) if defang else c for c in extracted_payloads.msiexec_commands],
            'mshta_commands': [defang_string(c) if defang else c for c in extracted_payloads.mshta_commands],
            'download_urls': [defang_url(u) if defang else u for u in extracted_payloads.download_urls]
        }
    
    print(json.dumps(result, indent=2))


def process_single_target(
    target: str,
    rules: List[EKFiddleRule],
    verbose: bool = False,
    timeout: int = 30,
    resource_timeout: int = 10,
    max_depth: int = 2,
    fetch_resources_flag: bool = True,
    show_redirects: bool = True,
    defang: bool = True,
    output_file: Optional[str] = None,
    show_matches: bool = True,
    extract_iocs_flag: bool = False,
    extract_payloads_flag: bool = False,
    include_benign_iocs: bool = False,
    json_output: bool = False,
    quiet: bool = False,
    proxy: Optional[str] = None,
    enrich_domains: bool = False
) -> dict:
    """
    Process a single domain/URL with enhanced resource loading.
    Returns dict with detections, final_url, iocs, payloads, and enrichment data.
    """
    url = normalize_url(target)
    if not url:
        if not quiet:
            print(f"[!] Invalid target: {target}")
        return {
            'detections': [],
            'final_url': "",
            'redirect_chain': [],
            'content_buffer': [],
            'failed_resources': {},
            'extracted_iocs': None,
            'extracted_payloads': None
        }
    
    if not quiet:
        print(f"[*] Fetching: {url}")
    
    # Fetch with redirect tracking
    content, headers, redirect_chain, final_url = fetch_with_redirect_tracking(url, timeout, proxy=proxy)
    
    if not content:
        if not quiet:
            print(f"[!] Failed to fetch content from {url}")
        return {
            'detections': [],
            'final_url': url,
            'redirect_chain': redirect_chain,
            'content_buffer': [],
            'failed_resources': {},
            'extracted_iocs': None,
            'extracted_payloads': None
        }
    
    if not quiet:
        print(f"[*] Received {len(content)} bytes")
    
    # Show redirect info
    if not quiet and len(redirect_chain) > 1:
        print(f"[*] Followed {len(redirect_chain) - 1} redirect(s) to: {final_url}")
    
    # Extract resources from HTML
    if not quiet:
        print(f"[*] Extracting resources...")
    resources = extract_resources(content, final_url)
    
    resource_count = (len(resources.scripts) + len(resources.iframes) + 
                     len(resources.css) + len(resources.inline_scripts) + 
                     len(resources.data_uris))
    if not quiet:
        print(f"[*] Found {resource_count} resource(s): "
              f"{len(resources.scripts)} scripts, "
              f"{len(resources.iframes)} iframes, "
              f"{len(resources.css)} CSS, "
              f"{len(resources.inline_scripts)} inline scripts, "
              f"{len(resources.data_uris)} data URIs")
    
    # Fetch external resources
    fetched_resources = {}
    failed_resources = {}
    if fetch_resources_flag and (resources.scripts or resources.iframes or resources.css):
        external_count = len(resources.scripts) + len(resources.iframes) + len(resources.css)
        if not quiet:
            print(f"[*] Fetching {external_count} external resource(s) (timeout: {resource_timeout}s each)...")
        fetched_resources, failed_resources = fetch_resources(resources, final_url, resource_timeout, verbose=verbose, proxy=proxy)
    
    # Build content buffer
    content_buffer = build_content_buffer(
        content, final_url, headers,
        resources, fetched_resources, redirect_chain
    )
    
    if not quiet:
        print(f"[*] Scanning {len(content_buffer)} content item(s) with {len(rules)} rules...")
    
    # Scan all content
    detections = scan_content_buffer(content_buffer, rules, verbose)
    
    if not quiet:
        if detections:
            print(f"[+] Found {len(detections)} threat indicator(s)")
        else:
            print(f"[-] No threats detected")
    
    # Analyze forms for credential harvesting
    suspicious_forms = []
    all_forms = analyze_forms(content, final_url)
    for form in all_forms:
        if form.has_password_field or form.has_credit_card_field or form.is_suspicious:
            suspicious_forms.append(form)
    
    if not quiet and suspicious_forms:
        print(f"[!] Found {len(suspicious_forms)} suspicious form(s)")
        for form in suspicious_forms:
            if form.has_password_field:
                print(f"    - Login form -> {truncate_url(form.action_url, 50)}")
            if form.has_credit_card_field:
                print(f"    - Payment form -> {truncate_url(form.action_url, 50)}")
            if form.is_cross_origin:
                print(f"      WARNING: Cross-origin submission!")
    
    # Detect JavaScript-based redirects
    js_redirects = []
    for item in content_buffer:
        if item.content_type in ('main_page', 'script', 'inline_script'):
            js_redirects.extend(detect_js_redirects(item.content))
    
    if not quiet and js_redirects:
        print(f"[!] Found {len(js_redirects)} JavaScript redirect(s)")
        for redir in js_redirects[:3]:
            url_display = defang_url(redir.target_url) if defang else redir.target_url
            print(f"    - [{redir.redirect_type}] {truncate_url(url_display, 50)}")
        if len(js_redirects) > 3:
            print(f"    ... and {len(js_redirects) - 3} more")
    
    # Extract IOCs from all content
    extracted_iocs = None
    if extract_iocs_flag:
        if not quiet:
            print(f"[*] Extracting IOCs...")
        iocs_list = []
        for item in content_buffer:
            iocs_list.append(extract_iocs(item.content, include_benign_iocs))
        extracted_iocs = merge_iocs(iocs_list)
        if not quiet and not extracted_iocs.is_empty():
            print(f"[+] Extracted {extracted_iocs.total_count()} IOC(s)")
    
    # Extract payloads from all content
    extracted_payloads = None
    if extract_payloads_flag:
        if not quiet:
            print(f"[*] Extracting payloads...")
        payloads_list = []
        for item in content_buffer:
            payloads_list.append(extract_payloads(item.content))
        extracted_payloads = merge_payloads(payloads_list)
        if not quiet and not extracted_payloads.is_empty():
            payload_count = (len(extracted_payloads.powershell_commands) + 
                           len(extracted_payloads.powershell_encoded) +
                           len(extracted_payloads.msiexec_commands) + 
                           len(extracted_payloads.mshta_commands) +
                           len(extracted_payloads.download_urls))
            print(f"[+] Extracted {payload_count} payload(s)")
    
    # Output results (skip if quiet mode)
    if not quiet:
        if json_output:
            output_results_json(
                detections, final_url, redirect_chain, content_buffer,
                failed_resources=failed_resources,
                extracted_iocs=extracted_iocs,
                extracted_payloads=extracted_payloads,
                defang=defang
            )
        else:
            output_results_enhanced(
                detections, final_url, redirect_chain, content_buffer,
                failed_resources=failed_resources,
                defang=defang, output_file=output_file,
                show_matches=show_matches, show_redirects=show_redirects,
                extracted_iocs=extracted_iocs,
                extracted_payloads=extracted_payloads
            )
    
    # Domain enrichment if requested
    domain_enrichment = None
    cert_info = None
    if enrich_domains:
        if not quiet:
            print(f"[*] Enriching domain information...")
        try:
            parsed = urlparse(final_url)
            target_domain = parsed.netloc
            domain_enrichment = enrich_domain(target_domain)
            if not quiet:
                if domain_enrichment.ip_addresses:
                    print(f"[+] IP: {', '.join(domain_enrichment.ip_addresses[:3])}")
                if domain_enrichment.whois_registrar:
                    print(f"[+] Registrar: {domain_enrichment.whois_registrar}")
                if domain_enrichment.domain_age_days >= 0:
                    age_str = f"{domain_enrichment.domain_age_days} days"
                    if domain_enrichment.is_newly_registered:
                        age_str += " (NEWLY REGISTERED - HIGH RISK)"
                    print(f"[+] Domain Age: {age_str}")
            
            # Certificate check for HTTPS
            if final_url.startswith('https://'):
                if not quiet:
                    print(f"[*] Checking SSL certificate...")
                cert_info = get_certificate_info(target_domain)
                if not quiet:
                    if cert_info.issuer:
                        print(f"[+] Certificate Issuer: {cert_info.issuer}")
                    if cert_info.is_self_signed:
                        print(f"[!] WARNING: Self-signed certificate")
                    if cert_info.is_expired:
                        print(f"[!] WARNING: Certificate EXPIRED")
                    elif cert_info.days_until_expiry >= 0:
                        print(f"[+] Certificate Valid For: {cert_info.days_until_expiry} days")
                    if cert_info.is_recently_issued:
                        print(f"[!] NOTE: Recently issued certificate (< 30 days)")
        except Exception as e:
            if not quiet:
                print(f"[!] Enrichment failed: {e}")
    
    # Return all data needed for reporting
    return {
        'detections': detections,
        'final_url': final_url,
        'redirect_chain': redirect_chain,
        'content_buffer': content_buffer,
        'failed_resources': failed_resources,
        'extracted_iocs': extracted_iocs,
        'extracted_payloads': extracted_payloads,
        'domain_enrichment': domain_enrichment,
        'certificate_info': cert_info,
        'suspicious_forms': suspicious_forms,
        'js_redirects': js_redirects
    }


def process_file(
    filepath: str,
    rules: List[EKFiddleRule],
    verbose: bool = False,
    timeout: int = 30,
    resource_timeout: int = 10,
    max_depth: int = 2,
    fetch_resources_flag: bool = True,
    show_redirects: bool = True,
    defang: bool = True,
    output_file: Optional[str] = None,
    show_matches: bool = True,
    extract_iocs_flag: bool = False,
    extract_payloads_flag: bool = False,
    include_benign_iocs: bool = False,
    json_output: bool = False,
    quiet: bool = False,
    proxy: Optional[str] = None,
    enrich_domains: bool = False
):
    """Process a file containing list of domains/URLs"""
    
    if not os.path.exists(filepath):
        if not quiet:
            print(f"[!] Error: File '{filepath}' not found")
        return
    
    with open(filepath, 'r') as f:
        targets = [line.strip() for line in f if line.strip() and not line.startswith('#')]
    
    if not quiet:
        print(f"[*] Processing {len(targets)} target(s) from {filepath}")
        print(f"[*] Using {len(rules)} detection rules")
    
    # Clear output file if specified
    if output_file:
        with open(output_file, 'w') as f:
            f.write("# EKFiddle Scanner Results\n")
    
    all_detections = []
    
    for i, target in enumerate(targets, 1):
        if not quiet:
            print(f"\n{'='*70}")
            print(f"[{i}/{len(targets)}] Processing: {target}")
            print('='*70)
        
        result = process_single_target(
            target, rules, verbose, timeout, resource_timeout,
            max_depth, fetch_resources_flag, show_redirects,
            defang, output_file, show_matches,
            extract_iocs_flag, extract_payloads_flag, include_benign_iocs,
            json_output, quiet, proxy, enrich_domains
        )
        all_detections.extend(result.get('detections', []))
    
    # Final summary
    print("\n" + "=" * 70)
    print("SCAN COMPLETE")
    print("=" * 70)
    print(f"Targets scanned: {len(targets)}")
    print(f"Total detections: {len(all_detections)}")
    
    if all_detections:
        severity_counts = {'high': 0, 'med': 0, 'low': 0}
        for d in all_detections:
            severity_counts[d.rule.severity] = severity_counts.get(d.rule.severity, 0) + 1
        print(f"  HIGH: {severity_counts.get('high', 0)} | MED: {severity_counts.get('med', 0)} | LOW: {severity_counts.get('low', 0)}")


def process_autoscan(
    filepath: str,
    rules: List[EKFiddleRule],
    timeout: int = 30,
    resource_timeout: int = 10,
    fetch_resources_flag: bool = True,
    proxy: Optional[str] = None,
    verbose: bool = False,
    output_file: Optional[str] = None
) -> None:
    """
    Batch scan targets from file and output clean results.
    Default: target + rules only. Verbose: includes matched content snippets.
    """
    from datetime import datetime
    
    if not os.path.exists(filepath):
        print(f"[!] Error: File '{filepath}' not found")
        return
    
    with open(filepath, 'r', encoding='utf-8') as f:
        content = f.read()
    
    # Parse targets: support newline and comma delimited
    targets = []
    for line in content.replace(',', '\n').split('\n'):
        target = line.strip()
        if target and not target.startswith('#'):
            targets.append(target)
    
    if not targets:
        print("[!] No valid targets found in input file")
        return
    
    # Use custom output file if provided, otherwise generate from input filename
    if output_file:
        output_file_path = output_file
    else:
        input_basename = os.path.splitext(os.path.basename(filepath))[0]
        output_file_path = f"{input_basename}_results.txt"
    
    print(f"[*] Autoscan: {len(targets)} target(s) from {filepath}")
    print(f"[*] Output: {output_file_path}")
    print(f"[*] Rules: {len(rules)} loaded")
    if verbose:
        print("[*] Mode: Verbose (including matched content)")
    print()
    
    # Track results
    results_with_detections = []
    total_scanned = 0
    total_clean = 0
    total_failed = 0
    
    # Write header to output file
    with open(output_file_path, 'w', encoding='utf-8') as f:
        f.write("=" * 80 + "\n")
        if verbose:
            f.write("AUTOSCAN RESULTS (VERBOSE)\n")
        else:
            f.write("AUTOSCAN RESULTS\n")
        f.write(f"Input: {os.path.basename(filepath)}\n")
        f.write(f"Scan Date: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}\n")
        f.write("=" * 80 + "\n\n")
    
    # Process each target
    for i, target in enumerate(targets, 1):
        # Progress indicator
        sys.stdout.write(f"\r[{i}/{len(targets)}] Scanning: {target[:50]:<50}")
        sys.stdout.flush()
        
        # Scan silently with error handling
        try:
            result = process_single_target(
                target, rules,
                verbose=False,
                timeout=timeout,
                resource_timeout=resource_timeout,
                fetch_resources_flag=fetch_resources_flag,
                quiet=True,
                proxy=proxy,
                json_output=False,
                show_matches=False
            )
        except Exception as e:
            # Log error but continue scanning
            sys.stdout.write(f"\r[{i}/{len(targets)}] Error scanning {target[:40]}: {str(e)[:30]}\n")
            total_scanned += 1
            total_failed += 1
            continue
        
        total_scanned += 1
        detections = result.get('detections', [])
        
        if not detections:
            total_clean += 1
            continue
        
        # Collect results for targets with detections
        final_url = result.get('final_url', target)
        
        # Group detections by severity
        detection_lines = []
        severity_order = {'high': 0, 'med': 1, 'low': 2}
        sorted_detections = sorted(detections, key=lambda d: (severity_order.get(d.rule.severity, 3), d.rule.name))
        
        seen_rules = set()
        for det in sorted_detections:
            rule_key = det.rule.name
            if rule_key in seen_rules:
                continue
            seen_rules.add(rule_key)
            
            severity_label = det.rule.severity.upper()
            detection_lines.append(f"  [{severity_label}] {det.rule.name}")
            
            if verbose and det.matches:
                match_content = det.matches[0][:100]
                match_display = defang_string(match_content)
                if len(det.matches[0]) > 100:
                    match_display += "..."
                detection_lines.append(f"         Match: {match_display}")
        
        # Write to file immediately
        with open(output_file_path, 'a', encoding='utf-8') as f:
            display_target = defang_url(target)
            f.write(f"{display_target}\n")
            for line in detection_lines:
                f.write(f"{line}\n")
            f.write("\n")
        
        results_with_detections.append(target)
    
    # Clear progress line
    sys.stdout.write("\r" + " " * 80 + "\r")
    sys.stdout.flush()
    
    # Update header with final counts
    total_detections = len(results_with_detections)
    
    # Read current content and update header
    with open(output_file_path, 'r', encoding='utf-8') as f:
        content = f.read()
    
    # Find and update the summary line
    lines = content.split('\n')
    for i, line in enumerate(lines):
        if line.startswith("Scan Date:"):
            summary = f"Scanned: {total_scanned} | Detections: {total_detections} | Clean: {total_clean}"
            if total_failed > 0:
                summary += f" | Failed: {total_failed}"
            lines.insert(i + 1, summary)
            break
    
    # Add footer
    lines.append("=" * 80)
    lines.append(f"End of scan - {total_detections} target(s) with detections")
    lines.append("=" * 80)
    
    with open(output_file_path, 'w', encoding='utf-8') as f:
        f.write('\n'.join(lines))
    
    # Console summary
    print(f"[+] Scan complete")
    print(f"    Scanned: {total_scanned}")
    print(f"    Detections: {total_detections}")
    print(f"    Clean: {total_clean}")
    if total_failed > 0:
        print(f"    Failed: {total_failed}")
    print(f"[+] Results written to: {output_file_path}")


def filter_rules(
    rules: List[EKFiddleRule],
    severity_filter: str = "all",
    type_filter: str = "all"
) -> List[EKFiddleRule]:
    """Filter rules by severity and type"""
    filtered = rules
    
    if severity_filter != "all":
        severities = [s.strip().lower() for s in severity_filter.split(',')]
        filtered = [r for r in filtered if r.severity in severities]
    
    if type_filter != "all":
        types = [t.strip().lower() for t in type_filter.split(',')]
        filtered = [r for r in filtered if r.rule_type.lower() in types]
    
    return filtered


# =============================================================================
# API SERVER
# =============================================================================

class EKFiddleAPIServer:
    """
    REST API server for EKFiddle Scanner.
    Simple, transparent API for internal use.
    """
    
    def __init__(self, rules: List[EKFiddleRule], api_key: Optional[str] = None,
                 host: str = '127.0.0.1', port: int = 8080):
        global FLASK_AVAILABLE, Flask, flask_request, jsonify
        
        # Ensure Flask is available
        if not FLASK_AVAILABLE:
            print("[*] Installing Flask for API server mode...")
            if install_package("flask"):
                from flask import Flask as FlaskClass, request as flask_req, jsonify as json_resp
                Flask = FlaskClass
                flask_request = flask_req
                jsonify = json_resp
                FLASK_AVAILABLE = True
            else:
                raise RuntimeError("Flask is required for API server mode")
        
        self.app = Flask(__name__)
        self.rules = rules
        self.api_key = api_key
        self.host = host
        self.port = port
        self._setup_routes()
    
    def _check_api_key(self):
        """Validate API key if configured."""
        if not self.api_key:
            return True, None
        
        provided_key = flask_request.headers.get('X-API-Key', '')
        if provided_key != self.api_key:
            return False, (jsonify({
                'success': False,
                'error': 'Invalid or missing API key. Use X-API-Key header.'
            }), 401)
        return True, None
    
    def _setup_routes(self):
        """Setup API routes."""
        
        @self.app.after_request
        def add_headers(response):
            response.headers['Access-Control-Allow-Origin'] = '*'
            response.headers['Access-Control-Allow-Methods'] = 'GET, POST, OPTIONS'
            response.headers['Access-Control-Allow-Headers'] = 'Content-Type, X-API-Key'
            return response
        
        # =====================================================================
        # GET /health - Health check
        # =====================================================================
        @self.app.route('/health', methods=['GET'])
        def health():
            return jsonify({
                'status': 'ok',
                'rules_loaded': len(self.rules)
            })
        
        # =====================================================================
        # GET /rules - List detection rules
        # =====================================================================
        @self.app.route('/rules', methods=['GET'])
        def rules_list():
            valid, error = self._check_api_key()
            if not valid:
                return error
            
            by_severity = {'high': 0, 'med': 0, 'low': 0}
            by_type = {}
            for rule in self.rules:
                by_severity[rule.severity] = by_severity.get(rule.severity, 0) + 1
                by_type[rule.rule_type] = by_type.get(rule.rule_type, 0) + 1
            
            return jsonify({
                'total': len(self.rules),
                'by_severity': by_severity,
                'by_type': by_type
            })
        
        # =====================================================================
        # GET /scan?url=... - Simple scan via URL parameter (browser-friendly)
        # =====================================================================
        @self.app.route('/scan', methods=['GET'])
        def scan_get():
            valid, error = self._check_api_key()
            if not valid:
                return error
            
            target = flask_request.args.get('url', '').strip()
            if not target:
                return jsonify({
                    'success': False,
                    'error': 'Missing url parameter. Usage: /scan?url=example.com'
                }), 400
            
            # Quick scan with defaults
            result = self._execute_scan(target, {})
            return jsonify(result)
        
        # =====================================================================
        # POST /scan - Full scan with options
        # =====================================================================
        @self.app.route('/scan', methods=['POST', 'OPTIONS'])
        def scan_post():
            if flask_request.method == 'OPTIONS':
                return '', 204
            
            valid, error = self._check_api_key()
            if not valid:
                return error
            
            try:
                data = flask_request.get_json() or {}
            except:
                return jsonify({'success': False, 'error': 'Invalid JSON'}), 400
            
            # Support both 'url' and 'target' for convenience
            target = data.get('url') or data.get('target', '')
            if not target:
                return jsonify({
                    'success': False,
                    'error': 'Missing url field'
                }), 400
            
            options = {
                'fetch_resources': data.get('fetch_resources', True),
                'extract_iocs': data.get('extract_iocs', False),
                'extract_payloads': data.get('extract_payloads', False),
                'timeout': data.get('timeout', 30),
                'resource_timeout': data.get('resource_timeout', 10),
            }
            
            result = self._execute_scan(target, options)
            return jsonify(result)
    
    def _execute_scan(self, target: str, options: dict) -> dict:
        """
        Execute scan and return clean, organized results.
        """
        from datetime import datetime
        
        scan_start = time.time()
        scanned_at = datetime.utcnow().strftime('%Y-%m-%dT%H:%M:%SZ')
        
        # Options
        fetch_resources_opt = options.get('fetch_resources', True)
        extract_iocs_opt = options.get('extract_iocs', False)
        extract_payloads_opt = options.get('extract_payloads', False)
        timeout = options.get('timeout', 30)
        resource_timeout = options.get('resource_timeout', 10)
        
        # Normalize URL
        url = normalize_url(target)
        if not url:
            return {
                'success': False,
                'error': 'Invalid URL',
                'target': {'input': target}
            }
        
        # Fetch content
        fetch_start = time.time()
        try:
            content, headers, redirect_chain, final_url = fetch_with_redirect_tracking(url, timeout)
        except Exception as e:
            return {
                'success': False,
                'error': f'Fetch failed: {str(e)}',
                'target': {'input': target, 'normalized': url}
            }
        fetch_time = int((time.time() - fetch_start) * 1000)
        
        if not content:
            return {
                'success': False,
                'error': 'No content received',
                'target': {'input': target, 'final_url': defang_url(final_url)}
            }
        
        # Extract and fetch resources
        resources = extract_resources(content, final_url)
        fetched = {}
        failed = {}
        
        if fetch_resources_opt:
            resource_start = time.time()
            fetched, failed = fetch_resources_func(
                resources, final_url, resource_timeout, verbose=False
            )
            resource_time = int((time.time() - resource_start) * 1000)
        else:
            resource_time = 0
        
        # Build content buffer and scan
        content_buffer = build_content_buffer(
            content, final_url, headers,
            resources, fetched, redirect_chain
        )
        
        scan_start_inner = time.time()
        detections = scan_content_buffer(content_buffer, self.rules, verbose=False)
        scan_time = int((time.time() - scan_start_inner) * 1000)
        
        # Count by severity
        counts = {'high': 0, 'med': 0, 'low': 0}
        for d in detections:
            counts[d.rule.severity] = counts.get(d.rule.severity, 0) + 1
        
        # Determine threat level and is_malicious
        is_malicious = counts['high'] > 0 or counts['med'] > 0
        if counts['high'] > 0:
            threat_level = 'HIGH'
        elif counts['med'] > 0:
            threat_level = 'MEDIUM'
        elif counts['low'] > 0:
            threat_level = 'LOW'
        else:
            threat_level = 'CLEAN'
        
        # Group detections by severity - rule info is prominent
        detections_grouped = {'high': [], 'med': [], 'low': []}
        for d in detections:
            det_info = {
                'rule_name': d.rule.name,
                'rule_description': d.rule.description or '',
                'rule_type': d.rule.rule_type,
                'found_in': defang_url(d.source_url),
                'content_type': d.source_type,
                'matched_content': defang_string(d.matches[0][:200]) if d.matches else ''
            }
            detections_grouped[d.rule.severity].append(det_info)
        
        # Build redirect chain info
        redirects = []
        for hop in redirect_chain:
            redirects.append({
                'url': defang_url(hop.url),
                'status': hop.status_code
            })
        
        # Extract IOCs if requested
        iocs = None
        if extract_iocs_opt:
            iocs_list = [extract_iocs_func(item.content, False) for item in content_buffer]
            merged = merge_iocs(iocs_list)
            if not merged.is_empty():
                iocs = {
                    'ipv4': list(merged.ipv4_addresses)[:20],
                    'domains': list(merged.domains)[:20],
                    'urls': list(merged.urls)[:10],
                    'emails': list(merged.email_addresses)[:10]
                }
        
        # Extract payloads if requested
        payloads = None
        if extract_payloads_opt:
            payloads_list = [extract_payloads_func(item.content) for item in content_buffer]
            merged_p = merge_payloads(payloads_list)
            if not merged_p.is_empty():
                payloads = {
                    'powershell': merged_p.powershell_commands[:5],
                    'msiexec': merged_p.msiexec_commands[:5],
                    'download_urls': [defang_url(u) for u in merged_p.download_urls[:10]]
                }
        
        total_time = int((time.time() - scan_start) * 1000)
        
        # Build clean response
        response = {
            'success': True,
            'scanned_at': scanned_at,
            
            'target': {
                'input': target,
                'final_url': defang_url(final_url),
                'redirects': len(redirect_chain) - 1 if len(redirect_chain) > 1 else 0
            },
            
            'result': {
                'is_malicious': is_malicious,
                'threat_level': threat_level,
                'detection_count': len(detections),
                'high': counts['high'],
                'med': counts['med'],
                'low': counts['low']
            },
            
            'detections': detections_grouped,
            
            'resources': {
                'scanned': len(content_buffer),
                'external_fetched': len(fetched),
                'failed': len(failed)
            },
            
            'timing': {
                'total_ms': total_time,
                'fetch_ms': fetch_time,
                'resources_ms': resource_time,
                'scan_ms': scan_time
            }
        }
        
        # Add redirect chain if there were redirects
        if len(redirect_chain) > 1:
            response['redirect_chain'] = redirects
        
        # Add optional data if extracted
        if iocs:
            response['iocs'] = iocs
        if payloads:
            response['payloads'] = payloads
        
        return response
    
    def run(self, debug: bool = False):
        """Start the API server."""
        print()
        print("=" * 50)
        print("  EKFiddle Scanner API Server")
        print("=" * 50)
        print(f"  Host: {self.host}")
        print(f"  Port: {self.port}")
        print(f"  Rules: {len(self.rules)} loaded")
        print(f"  Auth:  {'API key required' if self.api_key else 'Open access'}")
        print()
        print("  Endpoints:")
        print(f"    GET  /health        - Server status")
        print(f"    GET  /rules         - List detection rules")
        print(f"    GET  /scan?url=...  - Quick scan (browser-friendly)")
        print(f"    POST /scan          - Full scan with options")
        print()
        print("  Quick test:")
        print(f"    curl http://{self.host}:{self.port}/scan?url=example.com")
        print("=" * 50)
        print()
        
        self.app.run(host=self.host, port=self.port, debug=debug, threaded=True)


# Alias functions to avoid naming conflicts with parameters
fetch_resources_func = lambda *args, **kwargs: fetch_resources(*args, **kwargs)
extract_iocs_func = lambda content, include_benign: extract_iocs(content, include_benign)
extract_payloads_func = lambda content: extract_payloads(content)


def main():
    parser = argparse.ArgumentParser(
        description='EKFiddle Threat Scanner - Scans web pages and resources using EKFiddle regex rules',
        formatter_class=argparse.RawDescriptionHelpFormatter,
        epilog="""
Examples:
  Single domain or URL:
    python ekfiddle_scanner.py -d example.com
    python ekfiddle_scanner.py -d "https://example.com/suspicious/page?id=123"

  With custom rules:
    python ekfiddle_scanner.py -d example.com -r CustomRegexes.txt

  File with list of domains/URLs:
    python ekfiddle_scanner.py -l domains.txt -o results.txt

  Skip external resources (faster):
    python ekfiddle_scanner.py -d example.com --no-resources

  Filter by severity:
    python ekfiddle_scanner.py -d example.com --severity high,med

API Server Mode:
  Start API server:
    python ekfiddle_scanner.py --server

  Custom port with API key:
    python ekfiddle_scanner.py --server --port 5000 --api-key mysecretkey

  Endpoints:
    GET  /health        - Server status
    GET  /rules         - List detection rules
    GET  /scan?url=...  - Quick scan (browser-friendly)
    POST /scan          - Full scan with options

  Quick test:
    curl http://localhost:8080/scan?url=example.com

Resource Loading:
  The scanner fetches and scans:
  - Main page HTML content
  - All redirect chain pages
  - External JavaScript files
  - Iframe content
  - External CSS
  - Inline scripts and styles
  - Base64-encoded data URIs

Severity levels:
  - HIGH: High confidence malicious indicators
  - MED: Medium confidence, may need manual review
  - LOW: Low confidence, potential false positives
        """
    )
    
    group = parser.add_mutually_exclusive_group(required=False)
    group.add_argument('--domain', '-d',
                      help='Single domain or full URL to analyze')
    
    group.add_argument('--list', '-l',
                      help='Text file containing list of domains/URLs (one per line)')
    
    group.add_argument('--autoscan', '-a',
                      metavar='FILE',
                      help='Batch scan targets from file, output clean results to FILE_results.txt')
    
    # Note: --server is added separately to allow it to work independently
    
    parser.add_argument('--rules', '-r',
                       default=None,
                       help='Path to EKFiddle rules file (auto-detects CustomRegexes.txt or Regex_Rules.txt)')
    
    parser.add_argument('--output', '-o',
                       default=None,
                       help='Output file for results (default: stdout only)')
    
    parser.add_argument('--severity',
                       default='all',
                       help='Filter by severity: high, med, low, or all (default: all)')
    
    parser.add_argument('--types',
                       default='all',
                       help='Filter by rule type: sourcecode, uri, headers, ip, hash, or all')
    
    # New resource-related options
    parser.add_argument('--no-resources',
                       action='store_true',
                       help='Skip fetching external resources (faster, main page only)')
    
    parser.add_argument('--max-depth',
                       type=int,
                       default=2,
                       help='Max depth for fetching nested resources (default: 2)')
    
    parser.add_argument('--resource-timeout',
                       type=int,
                       default=10,
                       help='Timeout for each resource fetch in seconds (default: 10)')
    
    parser.add_argument('--show-redirects',
                       action='store_true',
                       default=True,
                       help='Show full redirect chain in output (default: True)')
    
    parser.add_argument('--no-redirects',
                       action='store_true',
                       help='Hide redirect chain in output')
    
    parser.add_argument('--no-defang',
                       action='store_true',
                       help='Do not defang URLs in output (show raw URLs)')
    
    parser.add_argument('--no-matches',
                       action='store_true',
                       help='Do not show matched content in output')
    
    parser.add_argument('--verbose', '-v',
                       action='store_true',
                       help='Verbose output')
    
    parser.add_argument('--timeout', '-t',
                       type=int,
                       default=30,
                       help='Request timeout in seconds (default: 30)')
    
    # IOC and payload extraction options
    parser.add_argument('--extract-iocs',
                       action='store_true',
                       help='Extract and display IOCs (IPs, domains, URLs, emails, crypto addresses)')
    
    parser.add_argument('--extract-payloads',
                       action='store_true',
                       help='Extract and display malicious payloads (PowerShell, MSIEXEC, etc.)')
    
    parser.add_argument('--ioc-export',
                       metavar='FILE',
                       help='Export IOCs to file (format determined by extension: .csv, .json, .stix.json)')
    
    parser.add_argument('--include-benign-iocs',
                       action='store_true',
                       help='Include known benign domains/IPs in IOC extraction')
    
    # Output format options
    parser.add_argument('--json',
                       action='store_true',
                       help='Output results in JSON format')
    
    parser.add_argument('--quiet', '-q',
                       action='store_true',
                       help='Quiet mode - only output detections (for scripting)')
    
    # Proxy support
    parser.add_argument('--proxy',
                       metavar='URL',
                       help='HTTP/SOCKS proxy URL (e.g., http://127.0.0.1:8080 or socks5://127.0.0.1:9050)')
    
    # Report generation
    parser.add_argument('--report',
                       metavar='FILE',
                       help='Generate report file (format by extension: .md, .html, .json)')
    
    # Domain enrichment
    parser.add_argument('--enrich',
                       action='store_true',
                       help='Enrich domains with DNS and WHOIS information')
    
    # API Server mode
    parser.add_argument('--server',
                       action='store_true',
                       help='Start in API server mode instead of CLI scanning')
    
    parser.add_argument('--host',
                       default='127.0.0.1',
                       help='API server host address (default: 127.0.0.1)')
    
    parser.add_argument('--port',
                       type=int,
                       default=8080,
                       help='API server port (default: 8080)')
    
    parser.add_argument('--api-key',
                       metavar='KEY',
                       help='API key for server authentication (optional)')
    
    args = parser.parse_args()
    
    # Enable IOC extraction if export is requested
    if args.ioc_export:
        args.extract_iocs = True
    
    if not args.quiet:
        print("[*] EKFiddle Threat Scanner (Enhanced)")
        print("[*] Loading detection rules...\n")
    
    # Determine rules file path
    script_dir = os.path.dirname(os.path.abspath(__file__))
    
    if args.rules:
        # User specified a rules file
        rules_path = args.rules
        if not os.path.isabs(rules_path):
            rules_path_script = os.path.join(script_dir, rules_path)
            if os.path.exists(rules_path_script):
                rules_path = rules_path_script
            elif not os.path.exists(rules_path):
                print(f"[!] Error: Rules file '{args.rules}' not found")
                print(f"    Checked: {rules_path}")
                print(f"    Checked: {rules_path_script}")
                sys.exit(1)
    else:
        # Auto-detect rules file: CustomRegexes.txt first, then Regex_Rules.txt
        rules_path = None
        for candidate in ['CustomRegexes.txt', 'Regex_Rules.txt']:
            # Check in script directory
            path_in_script_dir = os.path.join(script_dir, candidate)
            if os.path.exists(path_in_script_dir):
                rules_path = path_in_script_dir
                break
            # Check in current directory
            if os.path.exists(candidate):
                rules_path = candidate
                break
        
        if not rules_path:
            if not args.quiet:
                print("[!] Error: No rules file found")
                print(f"    Checked for CustomRegexes.txt and Regex_Rules.txt in:")
                print(f"    - {script_dir}")
                print(f"    - {os.getcwd()}")
            sys.exit(1)
    
    # Parse rules
    rules = parse_ekfiddle_rules(rules_path)
    
    if not rules:
        if not args.quiet:
            print("[!] Error: No valid rules loaded")
        sys.exit(1)
    
    if not args.quiet:
        print(f"[+] Loaded {len(rules)} rules from {rules_path}")
    
    # Filter rules
    rules = filter_rules(rules, args.severity, args.types)
    
    if not rules:
        if not args.quiet:
            print("[!] Error: No rules match the specified filters")
        sys.exit(1)
    
    if not args.quiet:
        if args.severity != "all" or args.types != "all":
            print(f"[*] After filtering: {len(rules)} rules")
        
        # Count by type
        type_counts = {}
        for r in rules:
            type_counts[r.rule_type] = type_counts.get(r.rule_type, 0) + 1
        print(f"[*] Rule types: {type_counts}")
        
        # Show resource fetching status
        if args.no_resources:
            print("[*] Resource fetching: DISABLED (main page only)")
        else:
            print(f"[*] Resource fetching: ENABLED (timeout: {args.resource_timeout}s)")
        
        if args.proxy:
            print(f"[*] Using proxy: {args.proxy}")
        
        if args.extract_iocs:
            print("[*] IOC extraction: ENABLED")
        
        if args.extract_payloads:
            print("[*] Payload extraction: ENABLED")
        
        print()
    
    # Handle API Server mode
    if args.server:
        print("[*] Starting API Server mode...")
        try:
            server = EKFiddleAPIServer(
                rules=rules,
                api_key=args.api_key,
                host=args.host,
                port=args.port
            )
            server.run()
        except KeyboardInterrupt:
            print("\n[*] Server stopped")
        except Exception as e:
            print(f"[!] Server error: {e}")
            sys.exit(1)
        return
    
    # Handle Autoscan mode
    if args.autoscan:
        process_autoscan(
            args.autoscan, rules,
            timeout=args.timeout,
            resource_timeout=args.resource_timeout,
            fetch_resources_flag=not args.no_resources,
            proxy=args.proxy,
            verbose=args.verbose,
            output_file=args.output
        )
        return
    
    # Check that domain or list is specified for CLI mode
    if not args.domain and not args.list:
        print("[!] Error: One of --domain, --list, --autoscan, or --server is required")
        print("    Use --help for usage information")
        sys.exit(1)
    
    show_redirects = not args.no_redirects
    
    # Process targets
    extracted_iocs = None
    
    if args.domain:
        scan_result = process_single_target(
            args.domain, rules,
            verbose=args.verbose,
            timeout=args.timeout,
            resource_timeout=args.resource_timeout,
            max_depth=args.max_depth,
            fetch_resources_flag=not args.no_resources,
            show_redirects=show_redirects,
            defang=not args.no_defang,
            output_file=args.output,
            show_matches=not args.no_matches,
            extract_iocs_flag=args.extract_iocs,
            extract_payloads_flag=args.extract_payloads,
            include_benign_iocs=args.include_benign_iocs,
            json_output=args.json,
            quiet=args.quiet,
            proxy=args.proxy,
            enrich_domains=args.enrich
        )
        extracted_iocs = scan_result.get('extracted_iocs')
    elif args.list:
        process_file(
            args.list, rules,
            verbose=args.verbose,
            timeout=args.timeout,
            resource_timeout=args.resource_timeout,
            max_depth=args.max_depth,
            fetch_resources_flag=not args.no_resources,
            show_redirects=show_redirects,
            defang=not args.no_defang,
            output_file=args.output,
            show_matches=not args.no_matches,
            extract_iocs_flag=args.extract_iocs,
            extract_payloads_flag=args.extract_payloads,
            include_benign_iocs=args.include_benign_iocs,
            json_output=args.json,
            quiet=args.quiet,
            proxy=args.proxy,
            enrich_domains=args.enrich
        )
    
    # Export IOCs if requested
    if args.ioc_export and extracted_iocs and not extracted_iocs.is_empty():
        export_path = args.ioc_export
        if export_path.endswith('.csv'):
            export_iocs_csv(extracted_iocs, export_path)
            if not args.quiet:
                print(f"\n[+] IOCs exported to: {export_path}")
        elif export_path.endswith('.stix.json'):
            import json
            stix_data = export_iocs_stix(extracted_iocs, args.domain or 'batch')
            with open(export_path, 'w') as f:
                json.dump(stix_data, f, indent=2)
            if not args.quiet:
                print(f"\n[+] IOCs exported to STIX format: {export_path}")
        elif export_path.endswith('.json'):
            import json
            ioc_data = export_iocs_json(extracted_iocs)
            with open(export_path, 'w') as f:
                json.dump(ioc_data, f, indent=2)
            if not args.quiet:
                print(f"\n[+] IOCs exported to: {export_path}")
    
    # Generate report if requested
    if args.report and args.domain and scan_result:
        report_path = args.report
        defang = not args.no_defang
        
        if report_path.endswith('.html'):
            report_content = generate_report_html(
                scan_result.get('detections', []),
                scan_result.get('final_url', args.domain),
                scan_result.get('redirect_chain', []),
                scan_result.get('content_buffer', []),
                scan_result.get('failed_resources', {}),
                scan_result.get('extracted_iocs'),
                scan_result.get('extracted_payloads'),
                defang
            )
            with open(report_path, 'w', encoding='utf-8') as f:
                f.write(report_content)
            if not args.quiet:
                print(f"\n[+] HTML report generated: {report_path}")
        
        elif report_path.endswith('.md'):
            report_content = generate_report_markdown(
                scan_result.get('detections', []),
                scan_result.get('final_url', args.domain),
                scan_result.get('redirect_chain', []),
                scan_result.get('content_buffer', []),
                scan_result.get('failed_resources', {}),
                scan_result.get('extracted_iocs'),
                scan_result.get('extracted_payloads'),
                defang
            )
            with open(report_path, 'w', encoding='utf-8') as f:
                f.write(report_content)
            if not args.quiet:
                print(f"\n[+] Markdown report generated: {report_path}")
        
        elif report_path.endswith('.json'):
            import json
            report_data = {
                'source_url': scan_result.get('final_url', args.domain),
                'detections': [
                    {
                        'rule_name': d.rule.name,
                        'severity': d.rule.severity,
                        'rule_type': d.rule.rule_type,
                        'source_url': d.source_url,
                        'matches': d.matches[:10]
                    }
                    for d in scan_result.get('detections', [])
                ],
                'iocs': export_iocs_json(scan_result.get('extracted_iocs')) if scan_result.get('extracted_iocs') else None
            }
            with open(report_path, 'w', encoding='utf-8') as f:
                json.dump(report_data, f, indent=2)
            if not args.quiet:
                print(f"\n[+] JSON report generated: {report_path}")


if __name__ == "__main__":
    main()