From ef33e7a3e5f6c564590afb7e014f5457e932c10c Mon Sep 17 00:00:00 2001 From: Chris Llanwarne Date: Thu, 15 May 2025 11:29:29 -0400 Subject: [PATCH 01/33] trivy CLI starters --- .gitignore | 3 +- README.md | 39 +++++- cli/README.md | 86 +++++++++++++ cli/__init__.py | 3 + cli/cli.py | 136 ++++++++++++++++++++ requirements.txt | 6 +- tools/__init__.py | 3 + tools/findings.py | 84 ++++++++++++ tools/github.py | 58 +++++++++ tools/poam.py | 171 +++++++++++++++++++++++++ tools/trivy/alerts.py | 149 +++++++++++++++++++++ tools/trivy/importer.py | 71 ++++++++++ tools/trivy/trivy_alerts_poaminator.py | 143 +++++++++++++++++++++ tools/utils.py | 17 +++ 14 files changed, 966 insertions(+), 3 deletions(-) create mode 100644 cli/README.md create mode 100644 cli/__init__.py create mode 100755 cli/cli.py create mode 100644 tools/__init__.py create mode 100644 tools/findings.py create mode 100644 tools/github.py create mode 100644 tools/poam.py create mode 100644 tools/trivy/alerts.py create mode 100644 tools/trivy/importer.py create mode 100644 tools/trivy/trivy_alerts_poaminator.py create mode 100644 tools/utils.py diff --git a/.gitignore b/.gitignore index e3101ee..9171af1 100644 --- a/.gitignore +++ b/.gitignore @@ -176,4 +176,5 @@ cython_debug/ .pypirc # Ignore any findings.db files -findings.db \ No newline at end of file +findings.db +working/ diff --git a/README.md b/README.md index 0b9d06f..cee093d 100644 --- a/README.md +++ b/README.md @@ -15,6 +15,7 @@ A Streamlit-based application for tracking and managing security findings from w - Interactive data visualization - Export capabilities for active and resolved findings - Modern, responsive UI inspired by hail.is +- Command line tools for automation and data management ## Installation @@ -40,7 +41,43 @@ source venv/bin/activate # On Windows: venv\Scripts\activate pip install -r requirements.txt ``` -## Usage +## Command Line Tools + +The application includes command line tools for automation and data management. These tools are available through the `cli.py` script in the `cli` directory. + +### Authentication + +The tools that interact with Google services use Application Default Credentials (ADC). To set up authentication: + +1. Using gcloud (recommended for development): +```bash +gcloud auth application-default login +``` + +2. Or using a service account (recommended for production): +```bash +export GOOGLE_APPLICATION_CREDENTIALS="/path/to/service-account-key.json" +``` + +### Available Commands + +1. Download Google Sheets: +```bash +# Using a Google Sheets URL +./cli/cli.py download-gsheet "https://docs.google.com/spreadsheets/d/YOUR_SHEET_ID" + +# Or using just the file ID +./cli/cli.py download-gsheet "YOUR_SHEET_ID" +``` + +The downloaded files will be saved to the `working` directory in the project root. + +To see all available commands and their options: +```bash +./cli/cli.py --help +``` + +## Web Application Usage 1. Start the Streamlit application (make sure you're in the repository root directory): ```bash diff --git a/cli/README.md b/cli/README.md new file mode 100644 index 0000000..927ade8 --- /dev/null +++ b/cli/README.md @@ -0,0 +1,86 @@ +# Security Findings CLI + +Command-line interface for managing security findings from Trivy scans. + +## Installation + +Ensure you have Python 3.x installed and the required dependencies: + +```bash +pip install -r requirements.txt +``` + +## Commands + +### Download Alerts + +Download Trivy alerts from GitHub's code scanning API: + +```bash +./cli.py download-alerts +``` + +This command: +- Downloads Trivy alerts from GitHub's code scanning API +- Saves them as a JSON file in the working directory +- Requires either: + 1. GitHub CLI (`gh`) installed and authenticated via `gh auth login` + 2. GitHub token provided via `GITHUB_TOKEN` environment variable + +### Convert Alerts + +Convert downloaded GitHub Trivy alerts from JSON to CSV format: + +```bash +./cli.py convert-alerts +``` + +This command: +- Takes a JSON file containing GitHub code scanning alerts +- Converts the alerts to a CSV format suitable for findings tracking +- Saves the output as a CSV file in the working directory + +### Import and View Alerts + +Import alerts from CSV and display the first entry in YAML format: + +```bash +./cli.py import-alerts +``` + +This command: +- Reads a CSV file containing Trivy alerts +- Converts each row into a Finding object +- Displays the first finding in YAML format for review + +### Preview Trivy POAMs + +Preview POAMs from an Excel file: + +```bash +./cli.py preview-trivy [--limit ] +``` + +This command: +- Reads POAMs from an Excel file +- Displays a preview of the first n entries (default: 5) +- Requires an Excel file with an "Open POA&M Items" sheet and headers in row 5 + +## Example Workflow + +1. Download alerts from GitHub: + ```bash + ./cli.py download-alerts + ``` + +2. Convert the downloaded JSON to CSV: + ```bash + ./cli.py convert-alerts alerts_20240513.json + ``` + +3. Import and verify the converted alerts: + ```bash + ./cli.py import-alerts working/trivy_alerts_20240513_180947.csv + ``` + +Each command includes error handling and will provide helpful error messages if something goes wrong. \ No newline at end of file diff --git a/cli/__init__.py b/cli/__init__.py new file mode 100644 index 0000000..0bf5620 --- /dev/null +++ b/cli/__init__.py @@ -0,0 +1,3 @@ +""" +CLI package for security tracker command line interface. +""" \ No newline at end of file diff --git a/cli/cli.py b/cli/cli.py new file mode 100755 index 0000000..4cfc568 --- /dev/null +++ b/cli/cli.py @@ -0,0 +1,136 @@ +#!/usr/bin/env python3 + +import click +import sys +import os +from pathlib import Path +import yaml +from datetime import datetime + +# Add the project root to the Python path +sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + +from tools.poam import PoamFile +from tools.github import download_trivy_alerts +from tools.trivy.alerts import convert_alerts_to_poam +from tools.trivy.importer import import_alerts_from_csv + +@click.group() +def cli(): + """Security findings management CLI.""" + pass + +@cli.command() +@click.argument('file_path', type=click.Path(exists=True)) +@click.option('--limit', '-n', default=5, help='Number of POAMs to preview') +def preview_trivy(file_path, limit): + """Preview Trivy POAMs from an Excel file. + + FILE_PATH should be the path to your POAM Excel file. + The file must contain an "Open POA&M Items" sheet with headers in row 5. + """ + try: + poam_file = PoamFile(file_path) + preview = poam_file.preview_trivy_poams(limit) + click.echo(preview) + except Exception as e: + click.echo(f"Error: {str(e)}", err=True) + sys.exit(1) + +@cli.command() +def download_alerts(): + """Download Trivy alerts from GitHub code scanning API. + + REPO: Optional GitHub repository in owner/name format (e.g. 'owner/repo') + If not provided, defaults to configured repository + + Requires one of: + 1. GitHub CLI (gh) to be installed and authenticated via 'gh auth login' + 2. GitHub token provided via --token option or GITHUB_TOKEN environment variable + + The alerts will be saved as a JSON file in the working directory. + """ + try: + output_file = download_trivy_alerts() + click.echo(f"Successfully downloaded alerts to: {output_file}") + except Exception as e: + click.echo(f"Error: {str(e)}", err=True) + sys.exit(1) + +@cli.command() +@click.argument('alerts_file', type=click.Path(exists=True)) +def convert_alerts(alerts_file): + """Convert GitHub Trivy alerts JSON to POAM CSV format. + + ALERTS_FILE should be a JSON file containing GitHub code scanning alerts. + The file can be obtained using the download-alerts command. + + The converted POAM data will be saved as a CSV file in the working directory. + """ + try: + alerts_path = Path(alerts_file) + output_file = convert_alerts_to_poam(alerts_path) + click.echo(f"Successfully converted alerts to POAM format: {output_file}") + except Exception as e: + click.echo(f"Error: {str(e)}", err=True) + sys.exit(1) + +@cli.command() +@click.argument('csv_file', type=click.Path(exists=True, path_type=Path)) +def import_alerts(csv_file: Path): + """ + Import Trivy alerts from a CSV file and display the first entry in YAML format. + + CSV_FILE: Path to the CSV file containing Trivy alerts + """ + try: + findings = import_alerts_from_csv(csv_file) + if not findings: + click.echo("No findings found in CSV file", err=True) + sys.exit(1) + + # Get the first finding and convert to dict for YAML output + first_finding = findings[0] + finding_dict = { + 'finding_id': first_finding.finding_id, + 'controls': first_finding.controls, + 'weakness_name': first_finding.weakness_name, + 'weakness_description': first_finding.weakness_description, + 'weakness_detector_source': first_finding.weakness_detector_source, + 'weakness_source_identifier': first_finding.weakness_source_identifier, + 'asset_identifier': first_finding.asset_identifier, + 'point_of_contact': first_finding.point_of_contact, + 'resources_required': first_finding.resources_required, + 'overall_remediation_plan': first_finding.overall_remediation_plan, + 'original_detection_date': first_finding.original_detection_date.strftime("%Y-%m-%d"), + 'scheduled_completion_date': first_finding.scheduled_completion_date.strftime("%Y-%m-%d"), + 'planned_milestones': first_finding.planned_milestones, + 'milestone_changes': first_finding.milestone_changes, + 'status_date': first_finding.status_date.strftime("%Y-%m-%d"), + 'vendor_dependency': first_finding.vendor_dependency, + 'last_vendor_check_in_date': first_finding.last_vendor_check_in_date.strftime("%Y-%m-%d") if first_finding.last_vendor_check_in_date else None, + 'vendor_dependent_product_name': first_finding.vendor_dependent_product_name, + 'original_risk_rating': first_finding.original_risk_rating, + 'adjusted_risk_rating': first_finding.adjusted_risk_rating, + 'risk_adjustment': first_finding.risk_adjustment, + 'false_positive': first_finding.false_positive, + 'operational_requirement': first_finding.operational_requirement, + 'deviation_rationale': first_finding.deviation_rationale, + 'supporting_documents': first_finding.supporting_documents, + 'comments': first_finding.comments, + 'auto_approve': first_finding.auto_approve, + 'binding_operational_directive_22_01_tracking': first_finding.binding_operational_directive_22_01_tracking, + 'binding_operational_directive_22_01_due_date': first_finding.binding_operational_directive_22_01_due_date.strftime("%Y-%m-%d") if first_finding.binding_operational_directive_22_01_due_date else None, + 'cve': first_finding.cve, + 'service_name': first_finding.service_name + } + + # Output as YAML + click.echo(yaml.dump(finding_dict, sort_keys=False)) + + except Exception as e: + click.echo(f"Error importing alerts: {str(e)}", err=True) + sys.exit(1) + +if __name__ == '__main__': + cli() \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index 4079643..fa07206 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,4 +2,8 @@ streamlit>=1.43.0; python_version >= "3.9" and python_version < "3.13" pandas==2.2.1; python_version >= "3.9" and python_version < "3.13" numpy==1.26.4; python_version >= "3.9" and python_version < "3.13" plotly==5.19.0; python_version >= "3.9" and python_version < "3.13" -python-dateutil==2.8.2; python_version >= "3.9" and python_version < "3.13" \ No newline at end of file +python-dateutil==2.8.2; python_version >= "3.9" and python_version < "3.13" +click>=8.1.7; python_version >= "3.9" and python_version < "3.13" +openpyxl>=3.1.2; python_version >= "3.9" and python_version < "3.13" +PyYAML>=6.0.1; python_version >= "3.9" and python_version < "3.13" +jq>=1.6.0; python_version >= "3.9" and python_version < "3.13" \ No newline at end of file diff --git a/tools/__init__.py b/tools/__init__.py new file mode 100644 index 0000000..6b1e51a --- /dev/null +++ b/tools/__init__.py @@ -0,0 +1,3 @@ +""" +Tools package for security tracker CLI utilities. +""" \ No newline at end of file diff --git a/tools/findings.py b/tools/findings.py new file mode 100644 index 0000000..9e47f5a --- /dev/null +++ b/tools/findings.py @@ -0,0 +1,84 @@ +""" +Module for handling security findings data structures. +""" +from dataclasses import dataclass +from datetime import datetime +from typing import Optional +import pandas as pd + +@dataclass +class Finding: + """Represents a single security finding.""" + finding_id: str + controls: str + weakness_name: str + weakness_description: str + weakness_detector_source: str + weakness_source_identifier: str + asset_identifier: str + point_of_contact: str + resources_required: Optional[str] + overall_remediation_plan: str + original_detection_date: datetime + scheduled_completion_date: datetime + planned_milestones: str + milestone_changes: str + status_date: datetime + vendor_dependency: str + last_vendor_check_in_date: Optional[datetime] + vendor_dependent_product_name: str + original_risk_rating: str + adjusted_risk_rating: Optional[str] + risk_adjustment: str + false_positive: str + operational_requirement: str + deviation_rationale: Optional[str] + supporting_documents: Optional[str] + comments: Optional[str] + auto_approve: str + binding_operational_directive_22_01_tracking: str + binding_operational_directive_22_01_due_date: Optional[datetime] + cve: Optional[str] + service_name: str + + @classmethod + def from_dict(cls, data: dict) -> 'Finding': + """Create a Finding from a dictionary, handling timestamp conversion.""" + # Convert pandas timestamps to datetime + date_fields = [ + 'original_detection_date', + 'scheduled_completion_date', + 'status_date', + 'last_vendor_check_in_date', + 'binding_operational_directive_22_01_due_date' + ] + + for field in date_fields: + if field in data and pd.notna(data[field]): + if isinstance(data[field], pd._libs.tslibs.timestamps.Timestamp): + data[field] = data[field].to_pydatetime() + else: + data[field] = None + + # Convert NaN to None for optional fields + for key, value in data.items(): + if pd.isna(value): + data[key] = None + elif isinstance(value, float) and key != 'comments': # Keep numeric comments + data[key] = str(int(value)) if value.is_integer() else str(value) + elif isinstance(value, str): + data[key] = value.strip() + + # Rename ID field if necessary + if 'POAM ID' in data: + data['finding_id'] = data.pop('POAM ID') + elif 'Alert ID' in data: + data['finding_id'] = data.pop('Alert ID') + + # Convert keys to snake_case + converted_data = {} + for key, value in data.items(): + snake_key = ''.join(['_' + c.lower() if c.isupper() else c.lower() for c in key]).lstrip('_') + converted_data[snake_key] = value + + return cls(**converted_data) \ No newline at end of file diff --git a/tools/github.py b/tools/github.py new file mode 100644 index 0000000..f981e0d --- /dev/null +++ b/tools/github.py @@ -0,0 +1,58 @@ +""" +Tool for interacting with GitHub APIs. +""" +import json +import subprocess +from datetime import datetime +from pathlib import Path + +from .utils import ensure_working_dir + +def download_trivy_alerts() -> Path: + """ + Download Trivy alerts from GitHub code scanning API. + Uses gh CLI tool to handle authentication and pagination. + + Returns: + Path to the downloaded JSON file + """ + working_dir = ensure_working_dir() + + # Generate timestamp for the filename + timestamp = datetime.now().strftime("%Y%m%d_%H%M%S") + output_file = working_dir / f"trivy_alerts_{timestamp}.json" + + try: + # Run gh command and capture output + cmd = [ + "gh", "api", "--paginate", + "-X", "GET", + "/repos/hail-is/hail/code-scanning/alerts", + "-f", "q=branch:main tool:Trivy" + ] + + result = subprocess.run( + cmd, + capture_output=True, + text=True, + check=True + ) + + # Parse and save the JSON response + alerts = json.loads(result.stdout) + output_file.write_text(json.dumps(alerts, indent=2)) + + return output_file + + except subprocess.CalledProcessError as e: + raise Exception( + f"Failed to download alerts. Make sure:\n" + f"1. The gh CLI tool is installed\n" + f"2. You are authenticated with gh auth login\n" + f"3. You have access to the hail-is/hail repository\n" + f"\nError: {e.stderr}" + ) + except json.JSONDecodeError as e: + raise Exception(f"Failed to parse GitHub API response: {e}") + except Exception as e: + raise Exception(f"Unexpected error downloading alerts: {e}") diff --git a/tools/poam.py b/tools/poam.py new file mode 100644 index 0000000..5b1e57e --- /dev/null +++ b/tools/poam.py @@ -0,0 +1,171 @@ +""" +Tool for handling POAM Excel files. +""" +import pandas as pd +import re +import yaml +from pathlib import Path +from dataclasses import dataclass +from datetime import datetime +from typing import Optional + +@dataclass +class PoamEntry: + """Represents a single POAM entry.""" + poam_id: str + controls: str + weakness_name: str + weakness_description: str + weakness_detector_source: str + weakness_source_identifier: str + asset_identifier: str + point_of_contact: str + resources_required: Optional[str] + overall_remediation_plan: str + original_detection_date: datetime + scheduled_completion_date: datetime + planned_milestones: str + milestone_changes: str + status_date: datetime + vendor_dependency: str + last_vendor_check_in_date: Optional[datetime] + vendor_dependent_product_name: str + original_risk_rating: str + adjusted_risk_rating: Optional[str] + risk_adjustment: str + false_positive: str + operational_requirement: str + deviation_rationale: Optional[str] + supporting_documents: Optional[str] + comments: Optional[str] + auto_approve: str + binding_operational_directive_22_01_tracking: str + binding_operational_directive_22_01_due_date: Optional[datetime] + cve: Optional[str] + service_name: str + + @classmethod + def from_dict(cls, data: dict) -> 'PoamEntry': + """Create a PoamEntry from a dictionary, handling timestamp conversion.""" + # Convert pandas timestamps to datetime + date_fields = [ + 'original_detection_date', + 'scheduled_completion_date', + 'status_date', + 'last_vendor_check_in_date', + 'binding_operational_directive_22_01_due_date' + ] + + for field in date_fields: + if field in data and pd.notna(data[field]): + if isinstance(data[field], pd._libs.tslibs.timestamps.Timestamp): + data[field] = data[field].to_pydatetime() + else: + data[field] = None + + # Convert NaN to None for optional fields + for key, value in data.items(): + if pd.isna(value): + data[key] = None + elif isinstance(value, float) and key != 'comments': # Keep numeric comments + data[key] = str(int(value)) if value.is_integer() else str(value) + elif isinstance(value, str): + data[key] = value.strip() + + # Rename POAM ID field if necessary + if 'POAM ID' in data: + data['poam_id'] = data.pop('POAM ID') + + # Convert keys to snake_case + converted_data = {} + for key, value in data.items(): + snake_key = ''.join(['_' + c.lower() if c.isupper() else c.lower() for c in key]).lstrip('_') + converted_data[snake_key] = value + + return cls(**converted_data) + +class PoamFile: + """Handler for POAM Excel files with specific support for Trivy findings.""" + + def __init__(self, file_path: str): + """ + Initialize a POAM file handler. + + Args: + file_path: Path to the XLSX file + """ + self.file_path = Path(file_path) + if not self.file_path.exists(): + raise FileNotFoundError(f"POAM file not found: {file_path}") + + # Load the Excel file + self.workbook = pd.ExcelFile(self.file_path) + + # Validate required sheet exists + if "Open POA&M Items" not in self.workbook.sheet_names: + raise ValueError('Excel file must contain "Open POA&M Items" sheet') + + # Load the data with headers in row 5 (0-based index is 4) + self.df = pd.read_excel( + self.workbook, + sheet_name="Open POA&M Items", + header=4, # 0-based index for row 5 + engine='openpyxl' + ) + + def get_trivy_poams(self) -> pd.DataFrame: + """ + Filter and return Trivy POAMs. + + Returns: + DataFrame containing only Trivy POAMs + """ + # Pattern matches YYYY-TRIVYXXXX where XXXX is 4 or more digits + trivy_pattern = r'^\d{4}-TRIVY\d{4,}$' + + # Filter for POAM IDs matching the Trivy pattern + return self.df[self.df['POAM ID'].str.match(trivy_pattern, na=False)] + + def get_trivy_poam_entries(self, limit: Optional[int] = None) -> list[PoamEntry]: + """ + Get Trivy POAMs as PoamEntry objects. + + Args: + limit: Optional number of entries to return + + Returns: + List of PoamEntry objects + """ + df = self.get_trivy_poams() + if limit: + df = df.head(limit) + + return [PoamEntry.from_dict(row) for _, row in df.iterrows()] + + def preview_trivy_poams(self, limit: int = 5) -> str: + """ + Get a YAML preview of the first N Trivy POAMs. + + Args: + limit: Number of POAMs to preview (default 5) + + Returns: + YAML formatted string of the POAMs + """ + entries = self.get_trivy_poam_entries(limit) + + # Convert to dict format expected in output + preview_data = [] + for entry in entries: + # Convert datetime objects to strings + entry_dict = {} + for field, value in entry.__dict__.items(): + if isinstance(value, datetime): + value = value.strftime('%Y-%m-%d') + # Convert snake_case back to Title Case for keys + key = ' '.join(word.capitalize() for word in field.split('_')) + entry_dict[key] = value + preview_data.append(entry_dict) + + # Convert to YAML with proper formatting + return yaml.dump(preview_data, sort_keys=False, allow_unicode=True) \ No newline at end of file diff --git a/tools/trivy/alerts.py b/tools/trivy/alerts.py new file mode 100644 index 0000000..135eff5 --- /dev/null +++ b/tools/trivy/alerts.py @@ -0,0 +1,149 @@ +""" +Tool for converting Trivy alerts to POAM format. +""" +import csv +import json +from datetime import datetime, timedelta +from pathlib import Path +import jq + +from ..utils import ensure_working_dir + +# JQ query to transform GitHub alerts into POAM format +ALERTS_TO_POAM_QUERY = """ +.[] | { + "_state": .state, + "Alert ID": .number, + "Controls": "RA-5", + "Weakness Name": .rule.description, + "Weakness Description": .rule.full_description, + "Weakness Detector Source": .html_url, + "Weakness Source Identifier": (.tool.name + " " + .tool.version), + "Asset Identifier": .rule.most_recent_instance.location.path, + "Point of Contact": "Chris Llanwarne", + "Resources Required": "None", + "Overall Remediation Plan": "Perform necessary updates to resolve the vulnerability", + "Original Detection Date": .created_at, + "Status Date": .updated_at, + "Last Vendor Check-in Date": .rule.updated_at, + "Scheduled Completion Date": "DATE", + "AGENCY Scheduled Completion Date": "DATE", + "Planned Milestones": "DATE: Perform necessary updates to resolve the vulnerability", + "Milestone Changes": "", + "Vendor Dependency": "Yes", + "Vendor Dependent Product Name": "Ubuntu", + "Original Risk Rating": .rule.security_severity_level, + "Adjusted Risk Rating": "", + "Risk Adjustment": "", + "False Positive": "No", + "Operational Requirement": "No", + "Deviation Rationale": "", + "Supporting Documents": "", + "Comments": .most_recent_instance.message.text, + "Auto-Approve": "No", + "Binding Operational Directive 22-01 tracking": "", + "Binding Operational Directive 22-01 Due Date": "", + "CVE": .rule.id, + "Service Name": "Hail Batch" +}""" + +# POAM CSV field names in order +FIELDNAMES = [ + "Alert ID", "Controls", "Weakness Name", "Weakness Description", + "Weakness Detector Source", "Weakness Source Identifier", "Asset Identifier", + "Point of Contact", "Resources Required", "Overall Remediation Plan", + "Original Detection Date", "Scheduled Completion Date", + "AGENCY Scheduled Completion Date", "Planned Milestones", "Milestone Changes", + "Status Date", "Vendor Dependency", "Last Vendor Check-in Date", + "Vendor Dependent Product Name", "Original Risk Rating", "Adjusted Risk Rating", + "Risk Adjustment", "False Positive", "Operational Requirement", + "Deviation Rationale", "Supporting Documents", "Comments", "Auto-Approve", + "Binding Operational Directive 22-01 tracking", + "Binding Operational Directive 22-01 Due Date", "CVE", "Service Name" +] + +def date_plus(iso_date_string: str, days_to_add: int) -> str: + """ + Parses an ISO date string, adds days, and formats it to MM/DD/YY. + + Args: + iso_date_string: The ISO date string to parse (e.g., "2023-10-26T12:00:00Z") + days_to_add: The number of days to add (can be positive or negative) + + Returns: + The formatted date string (MM/DD/YY), or None if parsing fails + """ + try: + date_object = datetime.fromisoformat(iso_date_string.replace("Z", "+00:00")) + modified_date = date_object + timedelta(days=days_to_add) + return modified_date.strftime("%m/%d/%y") + except ValueError as e: + raise ValueError(f"Invalid ISO date string format: {iso_date_string}") from e + +def convert_alerts_to_poam(alerts_file: Path) -> Path: + """ + Convert GitHub Trivy alerts JSON to POAM CSV format. + + Args: + alerts_file: Path to the JSON file containing GitHub alerts + + Returns: + Path to the generated CSV file + """ + # Load alerts data + alerts_data = json.loads(alerts_file.read_text()) + + # Compile and run JQ query + alerts_jq = jq.compile(ALERTS_TO_POAM_QUERY) + jq_results = alerts_jq.input_value(alerts_data) + rows: list[dict] = [] + + # Process each alert + for row in jq_results.all(): + # Skip non-Trivy and closed alerts + if row["Weakness Source Identifier"][:5] != "Trivy" or row.pop("_state") != "open": + continue + + # Parse message for asset information + message = { + kv[0]: (kv[1] if len(kv) > 1 else "") + for kv in [line.split(": ") for line in row["Comments"].split("\n")] + } + + if "Image" not in message or "Package" not in message: + continue + + # Update asset identifier + row["Asset Identifier"] = f"{message['Image']} ({message['Package']})" + + # Handle dates and intervals + orig_date = row["Original Detection Date"] + status_date = row["Status Date"] + sev = row["Original Risk Rating"].lower() + + # Calculate fix date based on severity + fix_intervals = {"high": 14, "medium": 90, "low": 180} + fix_interval = fix_intervals.get(sev, 0) + fix_date = date_plus(orig_date, fix_interval) + + # Update all dates + row["Original Detection Date"] = date_plus(orig_date, 0) + row["Status Date"] = date_plus(status_date, 0) + row["Last Vendor Check-in Date"] = date_plus(status_date, 0) + row["Scheduled Completion Date"] = fix_date + row["AGENCY Scheduled Completion Date"] = fix_date + row["Planned Milestones"] = row["Planned Milestones"].replace("DATE", fix_date) + + rows.append(row) + + # Generate output filename with timestamp + timestamp = datetime.now().strftime("%Y%m%d_%H%M%S") + output_file = ensure_working_dir() / f"trivy_alerts_{timestamp}.csv" + + # Write CSV file + with output_file.open('w', newline='') as csvfile: + writer = csv.DictWriter(csvfile, fieldnames=FIELDNAMES) + writer.writeheader() + writer.writerows(rows) + + return output_file \ No newline at end of file diff --git a/tools/trivy/importer.py b/tools/trivy/importer.py new file mode 100644 index 0000000..0027f44 --- /dev/null +++ b/tools/trivy/importer.py @@ -0,0 +1,71 @@ +""" +Tool for importing Trivy alerts from CSV format into Finding objects. +""" +import csv +from datetime import datetime +from pathlib import Path +from typing import List + +from ..findings import Finding + +def parse_date(date_str: str) -> datetime: + """Parse a date string in MM/DD/YY format into a datetime object.""" + try: + return datetime.strptime(date_str, "%m/%d/%y") + except ValueError as e: + raise ValueError(f"Invalid date format (expected MM/DD/YY): {date_str}") from e + +def import_alerts_from_csv(csv_file: Path) -> List[Finding]: + """ + Import Trivy alerts from a CSV file and convert them to Finding objects. + + Args: + csv_file: Path to the CSV file containing Trivy alerts + + Returns: + List of Finding objects + """ + entries = [] + + with csv_file.open('r', newline='') as f: + reader = csv.DictReader(f) + + for row in reader: + # Convert CSV field names to Finding field names + entry_data = { + 'finding_id': row['Alert ID'], + 'controls': row['Controls'], + 'weakness_name': row['Weakness Name'], + 'weakness_description': row['Weakness Description'], + 'weakness_detector_source': row['Weakness Detector Source'], + 'weakness_source_identifier': row['Weakness Source Identifier'], + 'asset_identifier': row['Asset Identifier'], + 'point_of_contact': row['Point of Contact'], + 'resources_required': row['Resources Required'] or None, + 'overall_remediation_plan': row['Overall Remediation Plan'], + 'original_detection_date': parse_date(row['Original Detection Date']), + 'scheduled_completion_date': parse_date(row['Scheduled Completion Date']), + 'planned_milestones': row['Planned Milestones'], + 'milestone_changes': row['Milestone Changes'], + 'status_date': parse_date(row['Status Date']), + 'vendor_dependency': row['Vendor Dependency'], + 'last_vendor_check_in_date': parse_date(row['Last Vendor Check-in Date']) if row['Last Vendor Check-in Date'] else None, + 'vendor_dependent_product_name': row['Vendor Dependent Product Name'], + 'original_risk_rating': row['Original Risk Rating'], + 'adjusted_risk_rating': row['Adjusted Risk Rating'] or None, + 'risk_adjustment': row['Risk Adjustment'], + 'false_positive': row['False Positive'], + 'operational_requirement': row['Operational Requirement'], + 'deviation_rationale': row['Deviation Rationale'] or None, + 'supporting_documents': row['Supporting Documents'] or None, + 'comments': row['Comments'] or None, + 'auto_approve': row['Auto-Approve'], + 'binding_operational_directive_22_01_tracking': row['Binding Operational Directive 22-01 tracking'], + 'binding_operational_directive_22_01_due_date': parse_date(row['Binding Operational Directive 22-01 Due Date']) if row['Binding Operational Directive 22-01 Due Date'] else None, + 'cve': row['CVE'] or None, + 'service_name': row['Service Name'] + } + + entries.append(Finding(**entry_data)) + + return entries \ No newline at end of file diff --git a/tools/trivy/trivy_alerts_poaminator.py b/tools/trivy/trivy_alerts_poaminator.py new file mode 100644 index 0000000..8142956 --- /dev/null +++ b/tools/trivy/trivy_alerts_poaminator.py @@ -0,0 +1,143 @@ +import csv +import json +import jq +from datetime import datetime, timedelta + + +def date_plus(iso_date_string, days_to_add): + """ + Parses an ISO date string, adds days, and formats it to a custom date string. + + Args: + iso_date_string: The ISO date string to parse (e.g., "2023-10-26T12:00:00Z"). + days_to_add: The number of days to add (can be positive or negative). + output_format: The desired output date format string (e.g., "%Y-%m-%d"). + + Returns: + The formatted date string, or None if parsing fails. + """ + try: + date_object = datetime.fromisoformat(iso_date_string.replace("Z", "+00:00")) + except ValueError: + print("Error: Invalid ISO date string format.") + return None + + modified_date = date_object + timedelta(days=days_to_add) + formatted_date = modified_date.strftime("%m/%d/%y") + return formatted_date + + +fieldnames = [ + "Alert ID", + "Controls", + "Weakness Name", + "Weakness Description", + "Weakness Detector Source", + "Weakness Source Identifier", + "Asset Identifier", + "Point of Contact", + "Resources Required", + "Overall Remediation Plan", + "Original Detection Date", + "Scheduled Completion Date", + "AGENCY Scheduled Completion Date", + "Planned Milestones", + "Milestone Changes", + "Status Date", + "Vendor Dependency", + "Last Vendor Check-in Date", + "Vendor Dependent Product Name", + "Original Risk Rating", + "Adjusted Risk Rating", + "Risk Adjustment", + "False Positive", + "Operational Requirement", + "Deviation Rationale", + "Supporting Documents", + "Comments", + "Auto-Approve", + "Binding Operational Directive 22-01 tracking", + "Binding Operational Directive 22-01 Due Date", + "CVE", + "Service Name", +] + +with open("alerts.json") as inf: + alerts_data = json.load(inf) + +alerts_jq = jq.compile(""" +.[] | { + "_state": .state, + "POAM ID": .number, + "Controls": "RA-5", + "Weakness Name": .rule.description, + "Weakness Description": .rule.full_description, + "Weakness Detector Source": .html_url, + "Weakness Source Identifier": (.tool.name + " " + .tool.version), + "Asset Identifier": .rule.most_recent_instance.location.path, + "Point of Contact": "Chris Llanwarne", + "Resources Required": "None", + "Overall Remediation Plan": "Perform necessary updates to resolve the vulnerability", + "Original Detection Date": .created_at, + "Status Date": .updated_at, + "Last Vendor Check-in Date": .rule.updated_at, + "Scheduled Completion Date": "DATE", + "AGENCY Scheduled Completion Date": "DATE", + "Planned Milestones": "DATE: Perform necessary updates to resolve the vulnerability", + "Milestone Changes": "", + "Vendor Dependency": "Yes", + "Vendor Dependent Product Name": "Ubuntu", + "Original Risk Rating": .rule.security_severity_level, + "Adjusted Risk Rating": "", + "Risk Adjustment": "", + "False Positive": "No", + "Operational Requirement": "No", + "Deviation Rationale": "", + "Supporting Documents": "", + "Comments": .most_recent_instance.message.text, + "Auto-Approve": "No", + "Binding Operational Directive 22-01 tracking": "", + "Binding Operational Directive 22-01 Due Date": "", + "CVE": .rule.id, + "Service Name": "Hail Batch" +}""") + +jq_results = alerts_jq.input_value(alerts_data) +rows: list[dict] = [] + +for row in jq_results.all(): + if row["Weakness Source Identifier"][:5] != "Trivy": + continue + state = row["_state"] + del row["_state"] + if state != "open": + continue + message = { + kv[0]: (kv[1] if len(kv) > 1 else "") + for kv in [line.split(": ") for line in row["Comments"].split("\n")] + } + if "Image" not in message: + print(message) + print(repr(row)) + row["Asset Identifier"] = f"{message['Image']} ({message['Package']})" + orig_date = row["Original Detection Date"] + status_date = row["Status Date"] + sev = row["Original Risk Rating"] + fix_intervals = {"high": 14, "medium": 90, "low": 180} + fix_interval = fix_intervals.get(sev) or 0 + fix_date = date_plus(orig_date, fix_interval) + row["Original Detection Date"] = date_plus(orig_date, 0) + row["Status Date"] = date_plus(status_date, 0) + row["Last Vendor Check-in Date"] = date_plus(status_date, 0) + row["Scheduled Completion Date"] = date_plus(orig_date, 0) + row["Original Detection Date"] = date_plus(orig_date, 0) + row["Scheduled Completion Date"] = fix_date + row["AGENCY Scheduled Completion Date"] = fix_date + row["Planned Milestones"] = row["Planned Milestones"].replace("DATE", fix_date) + + rows.append(row) + +with open("gh-alerts.csv", "w", newline="") as csvfile: + writer = csv.DictWriter(csvfile, fieldnames=fieldnames) + writer.writeheader() + writer.writerows(rows) diff --git a/tools/utils.py b/tools/utils.py new file mode 100644 index 0000000..1979816 --- /dev/null +++ b/tools/utils.py @@ -0,0 +1,17 @@ +""" +Common utility functions used across tools. +""" +import os +from pathlib import Path + +def ensure_working_dir() -> Path: + """ + Ensure the working directory exists and return its path. + The working directory is used for temporary files and downloads. + + Returns: + Path object for the working directory + """ + working_dir = Path(os.getcwd()) / 'working' + working_dir.mkdir(exist_ok=True) + return working_dir \ No newline at end of file From 3e30f4dd0441fe55c3a22b5dcd1e7363242ad61f Mon Sep 17 00:00:00 2001 From: Chris Llanwarne Date: Thu, 15 May 2025 13:47:00 -0400 Subject: [PATCH 02/33] diff working --- cli/README.md | 70 ++++++- cli/cli.py | 33 ++++ tests/__init__.py | 3 + tests/conftest.py | 10 + tests/test_diff.py | 200 +++++++++++++++++++ tests/test_poam.py | 36 ++++ tools/__init__.py | 4 + tools/poam.py | 50 ++++- tools/trivy/alerts.py | 2 +- tools/trivy/diff.py | 139 +++++++++++++ tools/trivy/trivy_alerts_poaminator.py | 263 +++++++++++++------------ 11 files changed, 673 insertions(+), 137 deletions(-) create mode 100644 tests/__init__.py create mode 100644 tests/conftest.py create mode 100644 tests/test_diff.py create mode 100644 tests/test_poam.py create mode 100644 tools/trivy/diff.py diff --git a/cli/README.md b/cli/README.md index 927ade8..3851366 100644 --- a/cli/README.md +++ b/cli/README.md @@ -10,6 +10,50 @@ Ensure you have Python 3.x installed and the required dependencies: pip install -r requirements.txt ``` +## Testing + +To run the tests, first install pytest and coverage tools: + +```bash +pip install pytest pytest-cov +``` + +The project uses a standard Python test layout: +``` +security-tracker-app/ +├── tests/ +│ ├── __init__.py +│ ├── conftest.py # Test configuration and fixtures +│ └── test_poam.py # Tests for POAM functionality +├── tools/ +│ ├── __init__.py +│ ├── poam.py +│ └── ... +└── cli/ + └── ... +``` + +Then run the tests: + +```bash +# Run all tests +pytest tests/ + +# Run specific test file +pytest tests/test_poam.py + +# Run with verbose output +pytest tests/test_poam.py -v + +# Run tests and show coverage +pytest tests/ --cov=tools +``` + +The test suite includes: +- Unit tests for data conversion utilities +- Field name handling for POAM entries +- Edge cases for text formatting + ## Commands ### Download Alerts @@ -53,6 +97,25 @@ This command: - Converts each row into a Finding object - Displays the first finding in YAML format for review +### Compare Alerts with POAMs + +Compare current Trivy alerts against existing POAMs: + +```bash +./cli.py alerts-diff +``` + +This command: +- Reads existing POAMs from an Excel file +- Compares them against current findings from a CSV file +- Shows: + - New findings that need POAMs created + - Existing findings that already have POAMs (with confidence scores) + - Closed POAMs that no longer have corresponding findings +- Matching is done based on: + - Weakness name similarity + - Asset identifier matching + ### Preview Trivy POAMs Preview POAMs from an Excel file: @@ -78,7 +141,12 @@ This command: ./cli.py convert-alerts alerts_20240513.json ``` -3. Import and verify the converted alerts: +3. Compare new alerts against existing POAMs: + ```bash + ./cli.py alerts-diff existing_poams.xlsx working/trivy_alerts_20240513_180947.csv + ``` + +4. Import and verify specific alerts: ```bash ./cli.py import-alerts working/trivy_alerts_20240513_180947.csv ``` diff --git a/cli/cli.py b/cli/cli.py index 4cfc568..35d8faa 100755 --- a/cli/cli.py +++ b/cli/cli.py @@ -14,6 +14,7 @@ from tools.github import download_trivy_alerts from tools.trivy.alerts import convert_alerts_to_poam from tools.trivy.importer import import_alerts_from_csv +from tools.trivy.diff import compare_findings_to_trivy_poams @click.group() def cli(): @@ -132,5 +133,37 @@ def import_alerts(csv_file: Path): click.echo(f"Error importing alerts: {str(e)}", err=True) sys.exit(1) +@cli.command() +@click.argument('poam_file', type=click.Path(exists=True, path_type=Path)) +@click.argument('alerts_csv', type=click.Path(exists=True, path_type=Path)) +def alerts_diff(poam_file: Path, alerts_csv: Path): + """ + Compare Trivy alerts from CSV against existing POAMs. + + POAM_FILE: Excel file containing existing POAMs + ALERTS_CSV: CSV file containing current Trivy alerts + + Shows: + - New findings that need POAMs created + - Existing findings that already have POAMs + - Closed POAMs that no longer have corresponding findings + """ + try: + # Import findings from CSV + findings = import_alerts_from_csv(alerts_csv) + if not findings: + click.echo("No findings found in CSV file", err=True) + sys.exit(1) + + # Compare findings against POAMs + diff = compare_findings_to_trivy_poams(findings, poam_file) + + # Print results + diff.print_summary() + + except Exception as e: + click.echo(f"Error comparing alerts: {str(e)}", err=True) + sys.exit(1) + if __name__ == '__main__': cli() \ No newline at end of file diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..33fc16c --- /dev/null +++ b/tests/__init__.py @@ -0,0 +1,3 @@ +""" +Test package initialization. +""" \ No newline at end of file diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..d6a8cb5 --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,10 @@ +""" +Test configuration and fixtures. +""" +import os +import sys +from pathlib import Path + +# Add the project root directory to the Python path +project_root = Path(__file__).parent.parent +sys.path.append(str(project_root)) \ No newline at end of file diff --git a/tests/test_diff.py b/tests/test_diff.py new file mode 100644 index 0000000..d1b49f2 --- /dev/null +++ b/tests/test_diff.py @@ -0,0 +1,200 @@ +""" +Tests for the diff logic between Findings and POAMs. +""" +from datetime import datetime +from pathlib import Path + +from tools.findings import Finding +from tools.poam import PoamEntry +from tools.trivy.diff import ( + _is_exact_match, + _is_asset_covered, + _find_matching_poam, + compare_findings_to_poams, +) + +def create_test_finding(finding_id: str, weakness_name: str, asset_identifier: str) -> Finding: + """Helper to create a test Finding with minimal required fields.""" + return Finding( + finding_id=finding_id, + controls="Test controls", + weakness_name=weakness_name, + weakness_description="Test description", + weakness_detector_source="Trivy", + weakness_source_identifier="TEST-001", + asset_identifier=asset_identifier, + point_of_contact="test@example.com", + resources_required=None, + overall_remediation_plan="Test plan", + original_detection_date=datetime.now(), + scheduled_completion_date=datetime.now(), + planned_milestones="Test milestones", + milestone_changes="None", + status_date=datetime.now(), + vendor_dependency="None", + last_vendor_check_in_date=None, + vendor_dependent_product_name="None", + original_risk_rating="Low", + adjusted_risk_rating=None, + risk_adjustment="None", + false_positive="No", + operational_requirement="No", + deviation_rationale=None, + supporting_documents=None, + comments=None, + auto_approve="No", + binding_operational_directive_22_01_tracking="No", + binding_operational_directive_22_01_due_date=None, + cve=None, + service_name="test-service" + ) + +def create_test_poam(poam_id: str, weakness_name: str, asset_identifier: str) -> PoamEntry: + """Helper to create a test PoamEntry with minimal required fields.""" + return PoamEntry( + poam_id=poam_id, + controls="Test controls", + weakness_name=weakness_name, + weakness_description="Test description", + weakness_detector_source="Trivy", + weakness_source_identifier="TEST-001", + asset_identifier=asset_identifier, + point_of_contact="test@example.com", + resources_required=None, + overall_remediation_plan="Test plan", + original_detection_date=datetime.now(), + scheduled_completion_date=datetime.now(), + planned_milestones="Test milestones", + milestone_changes="None", + status_date=datetime.now(), + vendor_dependency="None", + last_vendor_check_in_date=None, + vendor_dependent_product_name="None", + original_risk_rating="Low", + adjusted_risk_rating=None, + risk_adjustment="None", + false_positive="No", + operational_requirement="No", + deviation_rationale=None, + supporting_documents=None, + comments=None, + auto_approve="No", + binding_operational_directive_22_01_tracking="No", + binding_operational_directive_22_01_due_date=None, + cve=None, + service_name="test-service" + ) + +def test_exact_match(): + """Test the exact matching function.""" + assert _is_exact_match("Test String", "test string") + assert _is_exact_match("test string ", "test string") + assert not _is_exact_match("test string", "different string") + assert not _is_exact_match("", None) + assert not _is_exact_match(None, "test") + +def test_asset_covered(): + """Test the asset coverage function.""" + # Single asset matches + assert _is_asset_covered("app-1", "app-1") + assert _is_asset_covered("app-1", "App-1") + + # Asset list includes the finding's asset + assert _is_asset_covered("app-1", "app-1, app-2, app-3") + assert _is_asset_covered("app-2", "app-1,app-2,app-3") + + # Asset not in list + assert not _is_asset_covered("app-4", "app-1, app-2, app-3") + + # Edge cases + assert not _is_asset_covered("", "app-1") + assert not _is_asset_covered("app-1", "") + assert not _is_asset_covered(None, "app-1") + assert not _is_asset_covered("app-1", None) + +def test_find_matching_poam(): + """Test finding matching POAMs.""" + finding = create_test_finding( + finding_id="TRIVY-001", + weakness_name="SQL Injection; CVE-2023-1234", + asset_identifier="app-1" + ) + + # Exact match + matching_poam = create_test_poam( + poam_id="POAM-001", + weakness_name="SQL Injection; CVE-2023-1234", + asset_identifier="app-1" + ) + + # Same weakness, different asset + different_asset_poam = create_test_poam( + poam_id="POAM-002", + weakness_name="SQL Injection; CVE-2023-1234", + asset_identifier="app-2" + ) + + # Different weakness, same asset + different_weakness_poam = create_test_poam( + poam_id="POAM-003", + weakness_name="XSS; CVE-2023-5678", + asset_identifier="app-1" + ) + + # POAM covering multiple assets + multi_asset_poam = create_test_poam( + poam_id="POAM-004", + weakness_name="SQL Injection; CVE-2023-1234", + asset_identifier="app-1, app-2, app-3" + ) + + # Test single POAM matching + match = _find_matching_poam(finding, [matching_poam]) + assert match is not None + assert match.poam.poam_id == "POAM-001" + + # Test no match for different asset + match = _find_matching_poam(finding, [different_asset_poam]) + assert match is None + + # Test no match for different weakness + match = _find_matching_poam(finding, [different_weakness_poam]) + assert match is None + + # Test match with multi-asset POAM + match = _find_matching_poam(finding, [multi_asset_poam]) + assert match is not None + assert match.poam.poam_id == "POAM-004" + + # Test finding best match from multiple POAMs + all_poams = [different_asset_poam, different_weakness_poam, matching_poam, multi_asset_poam] + match = _find_matching_poam(finding, all_poams) + assert match is not None + assert match.poam.poam_id == "POAM-001" # Should match the first valid match + +def test_compare_findings_to_poams(): + """Test the full comparison logic.""" + # Create test findings + findings = [ + create_test_finding("TRIVY-001", "SQL Injection; CVE-2023-1234", "app-1"), + create_test_finding("TRIVY-002", "XSS; CVE-2023-5678", "app-2"), + create_test_finding("TRIVY-003", "CSRF; CVE-2023-9012", "app-3") + ] + + # Create test POAMs + poams = [ + create_test_poam("POAM-001", "SQL Injection; CVE-2023-1234", "app-1, app-4"), + create_test_poam("POAM-002", "XSS; CVE-2023-5678", "app-5"), + create_test_poam("POAM-003", "Buffer Overflow; CVE-2023-3456", "app-1"), + create_test_poam("POAM-004", "CSRF; CVE-2023-9012", "app-1, app-2, app-3"), + ] + + diff = compare_findings_to_poams(findings, poams) + + # Verify the results + assert {f.finding_id for f in diff.new_findings} == {"TRIVY-002"} + + assert {match.poam.poam_id for match in diff.existing_matches} == {"POAM-001", "POAM-004"} + assert {match.finding.finding_id for match in diff.existing_matches} == {"TRIVY-001", "TRIVY-003"} + + assert {poam.poam_id for poam in diff.closed_poams} == {"POAM-002", "POAM-003"} diff --git a/tests/test_poam.py b/tests/test_poam.py new file mode 100644 index 0000000..605f4e5 --- /dev/null +++ b/tests/test_poam.py @@ -0,0 +1,36 @@ +""" +Tests for the POAM module. +""" +import pytest +from tools.poam import convert_to_snake_case + +@pytest.mark.parametrize("input_str,expected", [ + ("Weakness Name", "weakness_name"), + ("POAM ID", "poam_id"), + ("Point of Contact", "point_of_contact"), + ("CVE", "cve"), + ("Auto-Approve", "auto_approve"), + ("Binding Operational Directive 22-01 tracking", "binding_operational_directive_22_01_tracking"), + ("Last Vendor Check-in Date", "last_vendor_check_in_date"), + ("Resources Required", "resources_required"), + ("Overall Remediation Plan", "overall_remediation_plan"), + ("Original Detection Date", "original_detection_date"), + ("Status Date", "status_date"), + ("Vendor Dependency", "vendor_dependency"), + ("Original Risk Rating", "original_risk_rating"), + ("False Positive", "false_positive"), + ("Operational Requirement", "operational_requirement"), + ("Supporting Documents", "supporting_documents"), + ("Comments", "comments"), + ("Service Name", "service_name"), + # Edge cases + ("", ""), + ("alreadysnakecase", "alreadysnakecase"), + ("UPPER CASE", "upper_case"), + ("Mixed Case With-Hyphen", "mixed_case_with_hyphen"), + (" Spaces Around ", "spaces_around"), + ("multiple spaces between", "multiple_spaces_between"), +]) +def test_convert_to_snake_case(input_str, expected): + """Test the convert_to_snake_case function with various inputs.""" + assert convert_to_snake_case(input_str) == expected \ No newline at end of file diff --git a/tools/__init__.py b/tools/__init__.py index 6b1e51a..48ad642 100644 --- a/tools/__init__.py +++ b/tools/__init__.py @@ -1,3 +1,7 @@ """ Tools package for security tracker CLI utilities. +""" + +""" +Tools package initialization. """ \ No newline at end of file diff --git a/tools/poam.py b/tools/poam.py index 5b1e57e..6deea86 100644 --- a/tools/poam.py +++ b/tools/poam.py @@ -9,7 +9,37 @@ from datetime import datetime from typing import Optional -@dataclass +def convert_to_snake_case(text: str) -> str: + """ + Convert a string to snake_case format. + + Args: + text: Input string in any format (e.g., "Weakness Name", "POAM ID", etc.) + + Returns: + String converted to snake_case format + + Examples: + >>> convert_to_snake_case("Weakness Name") + 'weakness_name' + >>> convert_to_snake_case("POAM ID") + 'poam_id' + >>> convert_to_snake_case("Auto-Approve") + 'auto_approve' + """ + if not text: + return text + + # Replace hyphens with spaces + text = text.replace('-', ' ') + + # Normalize spaces (remove extra spaces) + text = ' '.join(text.split()) + + # Convert to lowercase and replace spaces with underscores + return text.strip().lower().replace(' ', '_') + +@dataclass(frozen=True) class PoamEntry: """Represents a single POAM entry.""" poam_id: str @@ -44,6 +74,16 @@ class PoamEntry: cve: Optional[str] service_name: str + def __hash__(self) -> int: + """Make PoamEntry hashable based on its poam_id.""" + return hash(self.poam_id) + + def __eq__(self, other) -> bool: + """Define equality based on poam_id.""" + if not isinstance(other, PoamEntry): + return NotImplemented + return self.poam_id == other.poam_id + @classmethod def from_dict(cls, data: dict) -> 'PoamEntry': """Create a PoamEntry from a dictionary, handling timestamp conversion.""" @@ -77,10 +117,10 @@ def from_dict(cls, data: dict) -> 'PoamEntry': data['poam_id'] = data.pop('POAM ID') # Convert keys to snake_case - converted_data = {} - for key, value in data.items(): - snake_key = ''.join(['_' + c.lower() if c.isupper() else c.lower() for c in key]).lstrip('_') - converted_data[snake_key] = value + converted_data = { + convert_to_snake_case(key): value + for key, value in data.items() + } return cls(**converted_data) diff --git a/tools/trivy/alerts.py b/tools/trivy/alerts.py index 135eff5..2190b82 100644 --- a/tools/trivy/alerts.py +++ b/tools/trivy/alerts.py @@ -15,7 +15,7 @@ "_state": .state, "Alert ID": .number, "Controls": "RA-5", - "Weakness Name": .rule.description, + "Weakness Name": (.rule.description + "; " + .rule.id), "Weakness Description": .rule.full_description, "Weakness Detector Source": .html_url, "Weakness Source Identifier": (.tool.name + " " + .tool.version), diff --git a/tools/trivy/diff.py b/tools/trivy/diff.py new file mode 100644 index 0000000..1f00a5c --- /dev/null +++ b/tools/trivy/diff.py @@ -0,0 +1,139 @@ +""" +Module for comparing Trivy findings against existing POAMs. +""" +from dataclasses import dataclass +from typing import List, Optional +from pathlib import Path + +from ..findings import Finding +from ..poam import PoamFile, PoamEntry + +@dataclass +class FindingPoamMatch: + """Represents a match between a finding and an existing POAM.""" + finding: Finding + poam: PoamEntry + +@dataclass +class TrivyAlertsDiff: + """Represents the difference between current findings and existing POAMs.""" + new_findings: List[Finding] # Findings without corresponding POAMs + existing_matches: List[FindingPoamMatch] # Findings matched to existing POAMs + closed_poams: List[PoamEntry] # POAMs without corresponding findings + + def print_summary(self, max_preview: int = 10) -> None: + """Print a human-readable summary of the diff.""" + # Print new findings + print("\n=== New Findings ===") + print(f"Count: {len(self.new_findings)}") + if self.new_findings: + finding_ids = [finding.finding_id for finding in self.new_findings] + print(f"Finding IDs: {', '.join(finding_ids)}") + + # Print existing matches + print("\n=== Existing Matches ===") + print(f"Count: {len(self.existing_matches)}") + if self.existing_matches: + matches = [f"{match.finding.finding_id} -> {match.poam.poam_id}" + for match in self.existing_matches[:max_preview]] + print(f"Preview of matches: {', '.join(matches)}") + if len(self.existing_matches) > max_preview: + print(f"... and {len(self.existing_matches) - max_preview} more") + + # Print closed POAMs + print("\n=== Closed POAMs ===") + print(f"Count: {len(self.closed_poams)}") + if self.closed_poams: + poam_ids = [poam.poam_id for poam in self.closed_poams] + print(f"POAM IDs no longer active: {', '.join(poam_ids)}") + +def _is_exact_match(text1: str, text2: str) -> bool: + """ + Check if two strings match exactly (case-insensitive). + + Args: + text1: First text string + text2: Second text string + + Returns: + True if strings match exactly (ignoring case), False otherwise + """ + if not text1 or not text2: + return False + return text1.lower().strip() == text2.lower().strip() + +def _is_asset_covered(finding_asset: str, poam_assets: str) -> bool: + """ + Check if the finding's asset is included in the POAM's asset list. + + Args: + finding_asset: Asset identifier from the finding + poam_assets: Asset identifier field from the POAM (may contain multiple assets) + + Returns: + True if the finding's asset is contained within the POAM's asset list + """ + if not finding_asset or not poam_assets: + return False + return finding_asset.lower().strip() in poam_assets.lower().strip() + +def _find_matching_poam(finding: Finding, poams: List[PoamEntry]) -> Optional[FindingPoamMatch]: + """Find a matching POAM for a given finding based on exact weakness name match and asset coverage.""" + for poam in poams: + # Weakness name must match exactly, and the finding's asset must be included in the POAM's assets + if (_is_exact_match(finding.weakness_name, poam.weakness_name) and + _is_asset_covered(finding.asset_identifier, poam.asset_identifier)): + return FindingPoamMatch(finding=finding, poam=poam) + + return None + + +def compare_findings_to_trivy_poams(findings: List[Finding], poam_file: Path) -> TrivyAlertsDiff: + """ + Compare a list of findings against Trivy POAMs. + + Args: + findings: List of current findings from Trivy + poam_file: Path to Excel file containing Trivy POAMs + + Returns: + TrivyAlertsDiff containing new, existing, and closed findings + """ + # Load Trivy POAMs + poam_entries = PoamFile(poam_file).get_trivy_poam_entries() + return compare_findings_to_poams(findings, poam_entries) + + +def compare_findings_to_poams(findings: List[Finding], poam_entries: List[PoamEntry]) -> TrivyAlertsDiff: + """ + Compare a list of findings against existing POAMs. + + Args: + findings: List of current findings from Trivy + poam_file: Path to Excel file containing existing POAMs + + Returns: + TrivyAlertsDiff containing new, existing, and closed findings + """ + # Track which POAMs are matched + matched_poams = set() + new_findings = [] + existing_matches = [] + + # Find matches for each finding + for finding in findings: + match = _find_matching_poam(finding, poam_entries) + if match: + existing_matches.append(match) + matched_poams.add(match.poam) + else: + new_findings.append(finding) + + # Find closed POAMs (those without matches) + closed_poams = [poam for poam in poam_entries if poam not in matched_poams] + + return TrivyAlertsDiff( + new_findings=new_findings, + existing_matches=existing_matches, + closed_poams=closed_poams + ) \ No newline at end of file diff --git a/tools/trivy/trivy_alerts_poaminator.py b/tools/trivy/trivy_alerts_poaminator.py index 8142956..bf05239 100644 --- a/tools/trivy/trivy_alerts_poaminator.py +++ b/tools/trivy/trivy_alerts_poaminator.py @@ -1,143 +1,146 @@ -import csv -import json -import jq -from datetime import datetime, timedelta +# For reference only: the original script that generated the alerts.csv file. +# For the current script that is wired into the CLI, see trivy/alerts.py +# import csv +# import json +# import jq +# from datetime import datetime, timedelta -def date_plus(iso_date_string, days_to_add): - """ - Parses an ISO date string, adds days, and formats it to a custom date string. - Args: - iso_date_string: The ISO date string to parse (e.g., "2023-10-26T12:00:00Z"). - days_to_add: The number of days to add (can be positive or negative). - output_format: The desired output date format string (e.g., "%Y-%m-%d"). +# def date_plus(iso_date_string, days_to_add): +# """ +# Parses an ISO date string, adds days, and formats it to a custom date string. - Returns: - The formatted date string, or None if parsing fails. - """ - try: - date_object = datetime.fromisoformat(iso_date_string.replace("Z", "+00:00")) - except ValueError: - print("Error: Invalid ISO date string format.") - return None +# Args: +# iso_date_string: The ISO date string to parse (e.g., "2023-10-26T12:00:00Z"). +# days_to_add: The number of days to add (can be positive or negative). +# output_format: The desired output date format string (e.g., "%Y-%m-%d"). - modified_date = date_object + timedelta(days=days_to_add) - formatted_date = modified_date.strftime("%m/%d/%y") - return formatted_date +# Returns: +# The formatted date string, or None if parsing fails. +# """ +# try: +# date_object = datetime.fromisoformat(iso_date_string.replace("Z", "+00:00")) +# except ValueError: +# print("Error: Invalid ISO date string format.") +# return None +# modified_date = date_object + timedelta(days=days_to_add) +# formatted_date = modified_date.strftime("%m/%d/%y") +# return formatted_date -fieldnames = [ - "Alert ID", - "Controls", - "Weakness Name", - "Weakness Description", - "Weakness Detector Source", - "Weakness Source Identifier", - "Asset Identifier", - "Point of Contact", - "Resources Required", - "Overall Remediation Plan", - "Original Detection Date", - "Scheduled Completion Date", - "AGENCY Scheduled Completion Date", - "Planned Milestones", - "Milestone Changes", - "Status Date", - "Vendor Dependency", - "Last Vendor Check-in Date", - "Vendor Dependent Product Name", - "Original Risk Rating", - "Adjusted Risk Rating", - "Risk Adjustment", - "False Positive", - "Operational Requirement", - "Deviation Rationale", - "Supporting Documents", - "Comments", - "Auto-Approve", - "Binding Operational Directive 22-01 tracking", - "Binding Operational Directive 22-01 Due Date", - "CVE", - "Service Name", -] -with open("alerts.json") as inf: - alerts_data = json.load(inf) +# fieldnames = [ +# "Alert ID", +# "Controls", +# "Weakness Name", +# "Weakness Description", +# "Weakness Detector Source", +# "Weakness Source Identifier", +# "Asset Identifier", +# "Point of Contact", +# "Resources Required", +# "Overall Remediation Plan", +# "Original Detection Date", +# "Scheduled Completion Date", +# "AGENCY Scheduled Completion Date", +# "Planned Milestones", +# "Milestone Changes", +# "Status Date", +# "Vendor Dependency", +# "Last Vendor Check-in Date", +# "Vendor Dependent Product Name", +# "Original Risk Rating", +# "Adjusted Risk Rating", +# "Risk Adjustment", +# "False Positive", +# "Operational Requirement", +# "Deviation Rationale", +# "Supporting Documents", +# "Comments", +# "Auto-Approve", +# "Binding Operational Directive 22-01 tracking", +# "Binding Operational Directive 22-01 Due Date", +# "CVE", +# "Service Name", +# ] -alerts_jq = jq.compile(""" -.[] | { - "_state": .state, - "POAM ID": .number, - "Controls": "RA-5", - "Weakness Name": .rule.description, - "Weakness Description": .rule.full_description, - "Weakness Detector Source": .html_url, - "Weakness Source Identifier": (.tool.name + " " + .tool.version), - "Asset Identifier": .rule.most_recent_instance.location.path, - "Point of Contact": "Chris Llanwarne", - "Resources Required": "None", - "Overall Remediation Plan": "Perform necessary updates to resolve the vulnerability", - "Original Detection Date": .created_at, - "Status Date": .updated_at, - "Last Vendor Check-in Date": .rule.updated_at, - "Scheduled Completion Date": "DATE", - "AGENCY Scheduled Completion Date": "DATE", - "Planned Milestones": "DATE: Perform necessary updates to resolve the vulnerability", - "Milestone Changes": "", - "Vendor Dependency": "Yes", - "Vendor Dependent Product Name": "Ubuntu", - "Original Risk Rating": .rule.security_severity_level, - "Adjusted Risk Rating": "", - "Risk Adjustment": "", - "False Positive": "No", - "Operational Requirement": "No", - "Deviation Rationale": "", - "Supporting Documents": "", - "Comments": .most_recent_instance.message.text, - "Auto-Approve": "No", - "Binding Operational Directive 22-01 tracking": "", - "Binding Operational Directive 22-01 Due Date": "", - "CVE": .rule.id, - "Service Name": "Hail Batch" -}""") +# with open("alerts.json") as inf: +# alerts_data = json.load(inf) -jq_results = alerts_jq.input_value(alerts_data) -rows: list[dict] = [] +# alerts_jq = jq.compile(""" +# .[] | { +# "_state": .state, +# "POAM ID": .number, +# "Controls": "RA-5", +# "Weakness Name": .rule.description, +# "Weakness Description": .rule.full_description, +# "Weakness Detector Source": .html_url, +# "Weakness Source Identifier": (.tool.name + " " + .tool.version), +# "Asset Identifier": .rule.most_recent_instance.location.path, +# "Point of Contact": "Chris Llanwarne", +# "Resources Required": "None", +# "Overall Remediation Plan": "Perform necessary updates to resolve the vulnerability", +# "Original Detection Date": .created_at, +# "Status Date": .updated_at, +# "Last Vendor Check-in Date": .rule.updated_at, +# "Scheduled Completion Date": "DATE", +# "AGENCY Scheduled Completion Date": "DATE", +# "Planned Milestones": "DATE: Perform necessary updates to resolve the vulnerability", +# "Milestone Changes": "", +# "Vendor Dependency": "Yes", +# "Vendor Dependent Product Name": "Ubuntu", +# "Original Risk Rating": .rule.security_severity_level, +# "Adjusted Risk Rating": "", +# "Risk Adjustment": "", +# "False Positive": "No", +# "Operational Requirement": "No", +# "Deviation Rationale": "", +# "Supporting Documents": "", +# "Comments": .most_recent_instance.message.text, +# "Auto-Approve": "No", +# "Binding Operational Directive 22-01 tracking": "", +# "Binding Operational Directive 22-01 Due Date": "", +# "CVE": .rule.id, +# "Service Name": "Hail Batch" +# }""") -for row in jq_results.all(): - if row["Weakness Source Identifier"][:5] != "Trivy": - continue - state = row["_state"] - del row["_state"] - if state != "open": - continue - message = { - kv[0]: (kv[1] if len(kv) > 1 else "") - for kv in [line.split(": ") for line in row["Comments"].split("\n")] - } - if "Image" not in message: - print(message) - print(repr(row)) - row["Asset Identifier"] = f"{message['Image']} ({message['Package']})" - orig_date = row["Original Detection Date"] - status_date = row["Status Date"] - sev = row["Original Risk Rating"] - fix_intervals = {"high": 14, "medium": 90, "low": 180} - fix_interval = fix_intervals.get(sev) or 0 - fix_date = date_plus(orig_date, fix_interval) - row["Original Detection Date"] = date_plus(orig_date, 0) - row["Status Date"] = date_plus(status_date, 0) - row["Last Vendor Check-in Date"] = date_plus(status_date, 0) - row["Scheduled Completion Date"] = date_plus(orig_date, 0) - row["Original Detection Date"] = date_plus(orig_date, 0) - row["Scheduled Completion Date"] = fix_date - row["AGENCY Scheduled Completion Date"] = fix_date - row["Planned Milestones"] = row["Planned Milestones"].replace("DATE", fix_date) +# jq_results = alerts_jq.input_value(alerts_data) +# rows: list[dict] = [] - rows.append(row) +# for row in jq_results.all(): +# if row["Weakness Source Identifier"][:5] != "Trivy": +# continue +# state = row["_state"] +# del row["_state"] +# if state != "open": +# continue +# message = { +# kv[0]: (kv[1] if len(kv) > 1 else "") +# for kv in [line.split(": ") for line in row["Comments"].split("\n")] +# } +# if "Image" not in message: +# print(message) +# print(repr(row)) +# row["Asset Identifier"] = f"{message['Image']} ({message['Package']})" +# orig_date = row["Original Detection Date"] +# status_date = row["Status Date"] +# sev = row["Original Risk Rating"] +# fix_intervals = {"high": 14, "medium": 90, "low": 180} +# fix_interval = fix_intervals.get(sev) or 0 +# fix_date = date_plus(orig_date, fix_interval) +# row["Original Detection Date"] = date_plus(orig_date, 0) +# row["Status Date"] = date_plus(status_date, 0) +# row["Last Vendor Check-in Date"] = date_plus(status_date, 0) +# row["Scheduled Completion Date"] = date_plus(orig_date, 0) +# row["Original Detection Date"] = date_plus(orig_date, 0) +# row["Scheduled Completion Date"] = fix_date +# row["AGENCY Scheduled Completion Date"] = fix_date +# row["Planned Milestones"] = row["Planned Milestones"].replace("DATE", fix_date) -with open("gh-alerts.csv", "w", newline="") as csvfile: - writer = csv.DictWriter(csvfile, fieldnames=fieldnames) - writer.writeheader() - writer.writerows(rows) +# rows.append(row) + +# with open("gh-alerts.csv", "w", newline="") as csvfile: +# writer = csv.DictWriter(csvfile, fieldnames=fieldnames) +# writer.writeheader() +# writer.writerows(rows) From 56cf4c27a8483ef4f2b71e61ef144275f6315f99 Mon Sep 17 00:00:00 2001 From: Chris Llanwarne Date: Thu, 15 May 2025 13:58:45 -0400 Subject: [PATCH 03/33] Reopened findings too --- tests/test_diff.py | 36 ++++++++++++++++++++++---------- tools/poam.py | 50 +++++++++++++++++++++++++++++++++++++-------- tools/trivy/diff.py | 44 +++++++++++++++++++++++++++++---------- 3 files changed, 100 insertions(+), 30 deletions(-) diff --git a/tests/test_diff.py b/tests/test_diff.py index d1b49f2..a03a0ef 100644 --- a/tests/test_diff.py +++ b/tests/test_diff.py @@ -176,25 +176,39 @@ def test_compare_findings_to_poams(): """Test the full comparison logic.""" # Create test findings findings = [ - create_test_finding("TRIVY-001", "SQL Injection; CVE-2023-1234", "app-1"), - create_test_finding("TRIVY-002", "XSS; CVE-2023-5678", "app-2"), - create_test_finding("TRIVY-003", "CSRF; CVE-2023-9012", "app-3") + create_test_finding("TRIVY-001", "SQL Injection; CVE-2023-1234", "app-1"), # Should match open POAM + create_test_finding("TRIVY-002", "XSS; CVE-2023-5678", "app-2"), # Should be new (no match) + create_test_finding("TRIVY-003", "CSRF; CVE-2023-9012", "app-3"), # Should match open POAM + create_test_finding("TRIVY-004", "RCE; CVE-2023-4567", "app-4"), # Should match closed POAM ] - # Create test POAMs - poams = [ - create_test_poam("POAM-001", "SQL Injection; CVE-2023-1234", "app-1, app-4"), - create_test_poam("POAM-002", "XSS; CVE-2023-5678", "app-5"), - create_test_poam("POAM-003", "Buffer Overflow; CVE-2023-3456", "app-1"), - create_test_poam("POAM-004", "CSRF; CVE-2023-9012", "app-1, app-2, app-3"), + # Create test open POAMs + open_poams = [ + create_test_poam("POAM-001", "SQL Injection; CVE-2023-1234", "app-1, app-4"), # Should match TRIVY-001 + create_test_poam("POAM-002", "XSS; CVE-2023-5678", "app-5"), # Should be closed (no match) + create_test_poam("POAM-003", "Buffer Overflow; CVE-2023-3456", "app-1"), # Should be closed (no match) + create_test_poam("POAM-004", "CSRF; CVE-2023-9012", "app-1, app-2, app-3"), # Should match TRIVY-003 ] - diff = compare_findings_to_poams(findings, poams) + # Create test closed POAMs + closed_poams = [ + create_test_poam("POAM-005", "RCE; CVE-2023-4567", "app-4"), # Should match TRIVY-004 (reopened) + create_test_poam("POAM-006", "XSS; CVE-2023-8901", "app-6"), # Should stay closed (no match) + ] + + # Compare findings to POAMs + diff = compare_findings_to_poams(findings, open_poams, closed_poams) - # Verify the results + # Verify new findings assert {f.finding_id for f in diff.new_findings} == {"TRIVY-002"} + # Verify existing matches assert {match.poam.poam_id for match in diff.existing_matches} == {"POAM-001", "POAM-004"} assert {match.finding.finding_id for match in diff.existing_matches} == {"TRIVY-001", "TRIVY-003"} + # Verify reopened findings + assert {match.poam.poam_id for match in diff.reopened_findings} == {"POAM-005"} + assert {match.finding.finding_id for match in diff.reopened_findings} == {"TRIVY-004"} + + # Verify closed POAMs assert {poam.poam_id for poam in diff.closed_poams} == {"POAM-002", "POAM-003"} diff --git a/tools/poam.py b/tools/poam.py index 6deea86..d1386e4 100644 --- a/tools/poam.py +++ b/tools/poam.py @@ -145,13 +145,23 @@ def __init__(self, file_path: str): if "Open POA&M Items" not in self.workbook.sheet_names: raise ValueError('Excel file must contain "Open POA&M Items" sheet') - # Load the data with headers in row 5 (0-based index is 4) + # Load the open POAMs data with headers in row 5 (0-based index is 4) self.df = pd.read_excel( self.workbook, sheet_name="Open POA&M Items", header=4, # 0-based index for row 5 engine='openpyxl' ) + + # Load closed POAMs if available + self.closed_df = None + if "Closed POA&M Items" in self.workbook.sheet_names: + self.closed_df = pd.read_excel( + self.workbook, + sheet_name="Closed POA&M Items", + header=4, # 0-based index for row 5 + engine='openpyxl' + ) def get_trivy_poams(self) -> pd.DataFrame: """ @@ -166,7 +176,23 @@ def get_trivy_poams(self) -> pd.DataFrame: # Filter for POAM IDs matching the Trivy pattern return self.df[self.df['POAM ID'].str.match(trivy_pattern, na=False)] - def get_trivy_poam_entries(self, limit: Optional[int] = None) -> list[PoamEntry]: + def get_closed_trivy_poams(self) -> pd.DataFrame: + """ + Filter and return closed Trivy POAMs. + + Returns: + DataFrame containing only closed Trivy POAMs, or empty DataFrame if no closed POAMs exist + """ + if self.closed_df is None: + return pd.DataFrame() + + # Pattern matches YYYY-TRIVYXXXX where XXXX is 4 or more digits + trivy_pattern = r'^\d{4}-TRIVY\d{4,}$' + + # Filter for POAM IDs matching the Trivy pattern + return self.closed_df[self.closed_df['POAM ID'].str.match(trivy_pattern, na=False)] + + def get_trivy_poam_entries(self, limit: Optional[int] = None) -> tuple[list[PoamEntry], list[PoamEntry]]: """ Get Trivy POAMs as PoamEntry objects. @@ -174,13 +200,21 @@ def get_trivy_poam_entries(self, limit: Optional[int] = None) -> list[PoamEntry] limit: Optional number of entries to return Returns: - List of PoamEntry objects + Tuple of (open_poams, closed_poams) where each is a list of PoamEntry objects """ - df = self.get_trivy_poams() + # Get open POAMs + open_df = self.get_trivy_poams() + if limit: + open_df = open_df.head(limit) + open_poams = [PoamEntry.from_dict(row) for _, row in open_df.iterrows()] + + # Get closed POAMs + closed_df = self.get_closed_trivy_poams() if limit: - df = df.head(limit) + closed_df = closed_df.head(limit) + closed_poams = [PoamEntry.from_dict(row) for _, row in closed_df.iterrows()] - return [PoamEntry.from_dict(row) for _, row in df.iterrows()] + return open_poams, closed_poams def preview_trivy_poams(self, limit: int = 5) -> str: """ @@ -192,11 +226,11 @@ def preview_trivy_poams(self, limit: int = 5) -> str: Returns: YAML formatted string of the POAMs """ - entries = self.get_trivy_poam_entries(limit) + open_entries, closed_entries = self.get_trivy_poam_entries(limit) # Convert to dict format expected in output preview_data = [] - for entry in entries: + for entry in open_entries: # Convert datetime objects to strings entry_dict = {} for field, value in entry.__dict__.items(): diff --git a/tools/trivy/diff.py b/tools/trivy/diff.py index 1f00a5c..c5a555e 100644 --- a/tools/trivy/diff.py +++ b/tools/trivy/diff.py @@ -20,6 +20,7 @@ class TrivyAlertsDiff: new_findings: List[Finding] # Findings without corresponding POAMs existing_matches: List[FindingPoamMatch] # Findings matched to existing POAMs closed_poams: List[PoamEntry] # POAMs without corresponding findings + reopened_findings: List[FindingPoamMatch] # Findings that match previously closed POAMs def print_summary(self, max_preview: int = 10) -> None: """Print a human-readable summary of the diff.""" @@ -40,6 +41,16 @@ def print_summary(self, max_preview: int = 10) -> None: if len(self.existing_matches) > max_preview: print(f"... and {len(self.existing_matches) - max_preview} more") + # Print reopened findings + print("\n=== Reopened Findings ===") + print(f"Count: {len(self.reopened_findings)}") + if self.reopened_findings: + matches = [f"{match.finding.finding_id} -> {match.poam.poam_id}" + for match in self.reopened_findings[:max_preview]] + print(f"Preview of matches: {', '.join(matches)}") + if len(self.reopened_findings) > max_preview: + print(f"... and {len(self.reopened_findings) - max_preview} more") + # Print closed POAMs print("\n=== Closed POAMs ===") print(f"Count: {len(self.closed_poams)}") @@ -97,43 +108,54 @@ def compare_findings_to_trivy_poams(findings: List[Finding], poam_file: Path) -> poam_file: Path to Excel file containing Trivy POAMs Returns: - TrivyAlertsDiff containing new, existing, and closed findings + TrivyAlertsDiff containing new, existing, closed, and reopened findings """ # Load Trivy POAMs - poam_entries = PoamFile(poam_file).get_trivy_poam_entries() - return compare_findings_to_poams(findings, poam_entries) + poam_file_handler = PoamFile(poam_file) + open_poams, closed_poams = poam_file_handler.get_trivy_poam_entries() + return compare_findings_to_poams(findings, open_poams, closed_poams) -def compare_findings_to_poams(findings: List[Finding], poam_entries: List[PoamEntry]) -> TrivyAlertsDiff: +def compare_findings_to_poams(findings: List[Finding], + open_poams: List[PoamEntry], + closed_poams: List[PoamEntry]) -> TrivyAlertsDiff: """ Compare a list of findings against existing POAMs. Args: findings: List of current findings from Trivy - poam_file: Path to Excel file containing existing POAMs + open_poams: List of open POAMs + closed_poams: List of closed POAMs Returns: - TrivyAlertsDiff containing new, existing, and closed findings + TrivyAlertsDiff containing new, existing, closed, and reopened findings """ # Track which POAMs are matched matched_poams = set() new_findings = [] existing_matches = [] + reopened_findings = [] - # Find matches for each finding + # First check for matches against open POAMs for finding in findings: - match = _find_matching_poam(finding, poam_entries) + match = _find_matching_poam(finding, open_poams) if match: existing_matches.append(match) matched_poams.add(match.poam) else: - new_findings.append(finding) + # If no match in open POAMs, check closed POAMs + closed_match = _find_matching_poam(finding, closed_poams) + if closed_match: + reopened_findings.append(closed_match) + else: + new_findings.append(finding) # Find closed POAMs (those without matches) - closed_poams = [poam for poam in poam_entries if poam not in matched_poams] + closed_poams = [poam for poam in open_poams if poam not in matched_poams] return TrivyAlertsDiff( new_findings=new_findings, existing_matches=existing_matches, - closed_poams=closed_poams + closed_poams=closed_poams, + reopened_findings=reopened_findings ) \ No newline at end of file From 1610ad07ebf663f5174e30ecca33b73f10f8e11e Mon Sep 17 00:00:00 2001 From: Chris Llanwarne Date: Thu, 15 May 2025 15:30:50 -0400 Subject: [PATCH 04/33] diff to json output --- cli/cli.py | 7 ++ tests/test_poam_generator.py | 146 +++++++++++++++++++++++++++++ tools/trivy/diff.py | 126 ++++++++++++++++++++++++- tools/trivy/poam_generator.py | 172 ++++++++++++++++++++++++++++++++++ 4 files changed, 447 insertions(+), 4 deletions(-) create mode 100644 tests/test_poam_generator.py create mode 100644 tools/trivy/poam_generator.py diff --git a/cli/cli.py b/cli/cli.py index 35d8faa..a83a548 100755 --- a/cli/cli.py +++ b/cli/cli.py @@ -1,5 +1,6 @@ #!/usr/bin/env python3 +import json import click import sys import os @@ -160,6 +161,12 @@ def alerts_diff(poam_file: Path, alerts_csv: Path): # Print results diff.print_summary() + + # JSON output file path: + json_output_file = alerts_csv.with_suffix('.diff.json') + with open(json_output_file, 'w') as f: + json.dump(diff.to_json(), f) + click.echo(f"JSON output saved to: {json_output_file}") except Exception as e: click.echo(f"Error comparing alerts: {str(e)}", err=True) diff --git a/tests/test_poam_generator.py b/tests/test_poam_generator.py new file mode 100644 index 0000000..bfde1e6 --- /dev/null +++ b/tests/test_poam_generator.py @@ -0,0 +1,146 @@ +""" +Tests for POAM generation functionality. +""" +from datetime import datetime +import pytest +from tools.findings import Finding +from tools.trivy.poam_generator import ( + parse_trivy_id, + get_next_trivy_id, + findings_to_poam, + group_findings_by_weakness, + generate_poams_from_findings +) + +def create_test_finding(finding_id: str, weakness_name: str, asset_identifier: str) -> Finding: + """Helper to create a test Finding with minimal required fields.""" + return Finding( + finding_id=finding_id, + controls="Test controls", + weakness_name=weakness_name, + weakness_description="Test description", + weakness_detector_source="Trivy", + weakness_source_identifier="TEST-001", + asset_identifier=asset_identifier, + point_of_contact="test@example.com", + resources_required=None, + overall_remediation_plan="Test plan", + original_detection_date=datetime.now(), + scheduled_completion_date=datetime.now(), + planned_milestones="Test milestones", + milestone_changes="None", + status_date=datetime.now(), + vendor_dependency="None", + last_vendor_check_in_date=None, + vendor_dependent_product_name="None", + original_risk_rating="Low", + adjusted_risk_rating=None, + risk_adjustment="None", + false_positive="No", + operational_requirement="No", + deviation_rationale=None, + supporting_documents=None, + comments=None, + auto_approve="No", + binding_operational_directive_22_01_tracking="No", + binding_operational_directive_22_01_due_date=None, + cve=None, + service_name="test-service" + ) + +def test_parse_trivy_id(): + """Test parsing Trivy POAM IDs.""" + # Test valid IDs + assert parse_trivy_id("2024-TRIVY0001") == (2024, 1) + assert parse_trivy_id("2023-TRIVY9999") == (2023, 9999) + + # Test invalid IDs + with pytest.raises(ValueError): + parse_trivy_id("invalid") + with pytest.raises(ValueError): + parse_trivy_id("2024-TRIVY") + with pytest.raises(ValueError): + parse_trivy_id("2024-trivy0001") + +def test_get_next_trivy_id(): + """Test generating next Trivy POAM ID.""" + # Test with no existing IDs + assert get_next_trivy_id([], 2024) == "2024-TRIVY0001" + + # Test with existing IDs + existing_ids = ["2024-TRIVY0001", "2024-TRIVY0002", "2023-TRIVY0001"] + assert get_next_trivy_id(existing_ids, 2024) == "2024-TRIVY0003" + + # Test with invalid IDs in the list + existing_ids = ["2024-TRIVY0001", "invalid", "2024-TRIVY0003"] + assert get_next_trivy_id(existing_ids, 2024) == "2024-TRIVY0004" + +def test_findings_to_poam(): + """Test converting findings to a POAM.""" + findings = [ + create_test_finding("TRIVY-001", "SQL Injection", "app-1"), + create_test_finding("TRIVY-002", "SQL Injection", "app-2") + ] + + poam = findings_to_poam(findings, "2024-TRIVY0001") + + # Verify basic fields + assert poam.poam_id == "2024-TRIVY0001" + assert poam.weakness_name == "SQL Injection" + assert poam.asset_identifier == "app-1, app-2" + assert poam.comments == "TRIVY-001, TRIVY-002" + + # Test with different weakness names + findings = [ + create_test_finding("TRIVY-001", "SQL Injection", "app-1"), + create_test_finding("TRIVY-002", "XSS", "app-2") + ] + with pytest.raises(ValueError): + findings_to_poam(findings, "2024-TRIVY0001") + + # Test with empty findings list + with pytest.raises(ValueError): + findings_to_poam([], "2024-TRIVY0001") + +def test_group_findings_by_weakness(): + """Test grouping findings by weakness name.""" + findings = [ + create_test_finding("TRIVY-001", "SQL Injection", "app-1"), + create_test_finding("TRIVY-002", "XSS", "app-2"), + create_test_finding("TRIVY-003", "SQL Injection", "app-3") + ] + + groups = group_findings_by_weakness(findings) + + assert len(groups) == 2 + assert len(groups["SQL Injection"]) == 2 + assert len(groups["XSS"]) == 1 + assert {f.finding_id for f in groups["SQL Injection"]} == {"TRIVY-001", "TRIVY-003"} + assert {f.finding_id for f in groups["XSS"]} == {"TRIVY-002"} + +def test_generate_poams_from_findings(): + """Test generating POAMs from findings.""" + findings = [ + create_test_finding("TRIVY-001", "SQL Injection", "app-1"), + create_test_finding("TRIVY-002", "XSS", "app-2"), + create_test_finding("TRIVY-003", "SQL Injection", "app-3") + ] + + existing_poam_ids = ["2024-TRIVY0001", "2024-TRIVY0002"] + result = generate_poams_from_findings(findings, existing_poam_ids, 2024) + + assert len(result) == 2 # Two groups: SQL Injection and XSS + + # Check SQL Injection group + sql_group = next(r for r in result if r[1].weakness_name == "SQL Injection") + assert len(sql_group[0]) == 2 # Two findings + assert sql_group[1].poam_id == "2024-TRIVY0003" + assert sql_group[1].asset_identifier == "app-1, app-3" + assert sql_group[1].comments == "TRIVY-001, TRIVY-003" + + # Check XSS group + xss_group = next(r for r in result if r[1].weakness_name == "XSS") + assert len(xss_group[0]) == 1 # One finding + assert xss_group[1].poam_id == "2024-TRIVY0004" + assert xss_group[1].asset_identifier == "app-2" + assert xss_group[1].comments == "TRIVY-002" \ No newline at end of file diff --git a/tools/trivy/diff.py b/tools/trivy/diff.py index c5a555e..c397557 100644 --- a/tools/trivy/diff.py +++ b/tools/trivy/diff.py @@ -2,11 +2,13 @@ Module for comparing Trivy findings against existing POAMs. """ from dataclasses import dataclass -from typing import List, Optional +from typing import List, Optional, Tuple, Dict, Any from pathlib import Path +from datetime import datetime from ..findings import Finding from ..poam import PoamFile, PoamEntry +from .poam_generator import generate_poams_from_findings @dataclass class FindingPoamMatch: @@ -21,6 +23,90 @@ class TrivyAlertsDiff: existing_matches: List[FindingPoamMatch] # Findings matched to existing POAMs closed_poams: List[PoamEntry] # POAMs without corresponding findings reopened_findings: List[FindingPoamMatch] # Findings that match previously closed POAMs + proposed_poams: List[Tuple[List[Finding], PoamEntry]] # Proposed new POAMs with their findings + + def to_json(self) -> Dict[str, Any]: + """ + Convert the diff results to a JSON-serializable dictionary. + + Returns: + Dictionary containing the diff results in a structured format + """ + def format_datetime(dt: datetime) -> str: + return dt.strftime("%Y-%m-%d") if dt else None + + def poam_to_full_dict(poam: PoamEntry) -> Dict[str, Any]: + """Convert a POAM to a complete dictionary with all fields.""" + return { + "poam_id": poam.poam_id, + "controls": poam.controls, + "weakness_name": poam.weakness_name, + "weakness_description": poam.weakness_description, + "weakness_detector_source": poam.weakness_detector_source, + "weakness_source_identifier": poam.weakness_source_identifier, + "asset_identifier": poam.asset_identifier, + "point_of_contact": poam.point_of_contact, + "resources_required": poam.resources_required, + "overall_remediation_plan": poam.overall_remediation_plan, + "original_detection_date": format_datetime(poam.original_detection_date), + "scheduled_completion_date": format_datetime(poam.scheduled_completion_date), + "planned_milestones": poam.planned_milestones, + "milestone_changes": poam.milestone_changes, + "status_date": format_datetime(poam.status_date), + "vendor_dependency": poam.vendor_dependency, + "last_vendor_check_in_date": format_datetime(poam.last_vendor_check_in_date), + "vendor_dependent_product_name": poam.vendor_dependent_product_name, + "original_risk_rating": poam.original_risk_rating, + "adjusted_risk_rating": poam.adjusted_risk_rating, + "risk_adjustment": poam.risk_adjustment, + "false_positive": poam.false_positive, + "operational_requirement": poam.operational_requirement, + "deviation_rationale": poam.deviation_rationale, + "supporting_documents": poam.supporting_documents, + "comments": poam.comments, + "auto_approve": poam.auto_approve, + "binding_operational_directive_22_01_tracking": poam.binding_operational_directive_22_01_tracking, + "binding_operational_directive_22_01_due_date": format_datetime(poam.binding_operational_directive_22_01_due_date), + "cve": poam.cve, + "service_name": poam.service_name + } + + def finding_to_dict(finding: Finding) -> Dict[str, Any]: + return { + "finding_id": finding.finding_id, + "weakness_name": finding.weakness_name, + "asset_identifier": finding.asset_identifier, + "original_detection_date": format_datetime(finding.original_detection_date), + "original_risk_rating": finding.original_risk_rating, + "cve": finding.cve, + "service_name": finding.service_name + } + + return { + "metadata": { + "new_findings_count": len(self.new_findings), + "existing_matches_count": len(self.existing_matches), + "closed_poams_count": len(self.closed_poams), + "reopened_findings_count": len(self.reopened_findings), + "proposed_poams_count": len(self.proposed_poams) + }, + "new_poams": [ + { + "poam": poam_to_full_dict(poam), + "findings": [finding_to_dict(f) for f in findings], + "finding_ids": [f.finding_id for f in findings] + } + for findings, poam in self.proposed_poams + ], + "reopen_poams": [ + { + "poam_id": match.poam.poam_id, + "finding_id": match.finding.finding_id + } + for match in self.reopened_findings + ], + "close_poams": [poam.poam_id for poam in self.closed_poams] + } def print_summary(self, max_preview: int = 10) -> None: """Print a human-readable summary of the diff.""" @@ -58,6 +144,28 @@ def print_summary(self, max_preview: int = 10) -> None: poam_ids = [poam.poam_id for poam in self.closed_poams] print(f"POAM IDs no longer active: {', '.join(poam_ids)}") + # Print proposed POAMs + print("\n=== Proposed POAMs ===") + print(f"Count: {len(self.proposed_poams)}") + if self.proposed_poams: + for findings, poam in self.proposed_poams[:max_preview]: + finding_ids = [f.finding_id for f in findings] + print(f"{', '.join(finding_ids)} => {poam.poam_id}") + if len(self.proposed_poams) > max_preview: + print(f"... and {len(self.proposed_poams) - max_preview} more") + + # Show sample of first proposed POAM + print("\nSample new POAM:") + sample_findings, sample_poam = self.proposed_poams[0] + print(f"POAM ID: {sample_poam.poam_id}") + print(f"Weakness Name: {sample_poam.weakness_name}") + print(f"Asset Identifiers: {sample_poam.asset_identifier}") + print(f"Finding IDs: {sample_poam.comments}") + print(f"Detection Date: {sample_poam.original_detection_date.strftime('%Y-%m-%d')}") + print(f"Risk Rating: {sample_poam.original_risk_rating}") + if sample_poam.cve: + print(f"CVE: {sample_poam.cve}") + def _is_exact_match(text1: str, text2: str) -> bool: """ Check if two strings match exactly (case-insensitive). @@ -113,12 +221,17 @@ def compare_findings_to_trivy_poams(findings: List[Finding], poam_file: Path) -> # Load Trivy POAMs poam_file_handler = PoamFile(poam_file) open_poams, closed_poams = poam_file_handler.get_trivy_poam_entries() - return compare_findings_to_poams(findings, open_poams, closed_poams) + + # Get all POAM IDs for generating new ones + all_poam_ids = [p.poam_id for p in [*open_poams, *closed_poams]] + + return compare_findings_to_poams(findings, open_poams, closed_poams, all_poam_ids) def compare_findings_to_poams(findings: List[Finding], open_poams: List[PoamEntry], - closed_poams: List[PoamEntry]) -> TrivyAlertsDiff: + closed_poams: List[PoamEntry], + existing_poam_ids: List[str]) -> TrivyAlertsDiff: """ Compare a list of findings against existing POAMs. @@ -126,6 +239,7 @@ def compare_findings_to_poams(findings: List[Finding], findings: List of current findings from Trivy open_poams: List of open POAMs closed_poams: List of closed POAMs + existing_poam_ids: List of all existing POAM IDs Returns: TrivyAlertsDiff containing new, existing, closed, and reopened findings @@ -153,9 +267,13 @@ def compare_findings_to_poams(findings: List[Finding], # Find closed POAMs (those without matches) closed_poams = [poam for poam in open_poams if poam not in matched_poams] + # Generate proposed POAMs for new findings + proposed_poams = generate_poams_from_findings(new_findings, existing_poam_ids) + return TrivyAlertsDiff( new_findings=new_findings, existing_matches=existing_matches, closed_poams=closed_poams, - reopened_findings=reopened_findings + reopened_findings=reopened_findings, + proposed_poams=proposed_poams ) \ No newline at end of file diff --git a/tools/trivy/poam_generator.py b/tools/trivy/poam_generator.py new file mode 100644 index 0000000..ecb1bdc --- /dev/null +++ b/tools/trivy/poam_generator.py @@ -0,0 +1,172 @@ +""" +Module for generating POAMs from findings. +""" +from datetime import datetime +from typing import List, Dict, Tuple +import re +from ..findings import Finding +from ..poam import PoamEntry + +def parse_trivy_id(poam_id: str) -> Tuple[int, int]: + """ + Parse a Trivy POAM ID into year and sequence components. + + Args: + poam_id: POAM ID in format YYYY-TRIVYXXXX + + Returns: + Tuple of (year, sequence_number) + + Raises: + ValueError: If the ID format is invalid + """ + match = re.match(r'^(\d{4})-TRIVY(\d{4,})$', poam_id) + if not match: + raise ValueError(f"Invalid Trivy POAM ID format: {poam_id}") + + year = int(match.group(1)) + sequence = int(match.group(2)) + return year, sequence + +def get_next_trivy_id(existing_poam_ids: List[str], current_year: int = None) -> str: + """ + Generate the next available Trivy POAM ID. + + Args: + existing_poam_ids: List of existing POAM IDs + current_year: Optional year to use (defaults to current year) + + Returns: + Next available POAM ID in YYYY-TRIVYXXXX format + """ + if current_year is None: + current_year = datetime.now().year + + # Find the highest sequence number for the current year + max_sequence = 0 + for poam_id in existing_poam_ids: + try: + year, sequence = parse_trivy_id(poam_id) + if year == current_year and sequence > max_sequence: + max_sequence = sequence + except ValueError: + continue # Skip invalid IDs + + return f"{current_year}-TRIVY{max_sequence + 1:04d}" + +def findings_to_poam(findings: List[Finding], poam_id: str) -> PoamEntry: + """ + Convert a list of findings with the same weakness into a single POAM. + + Args: + findings: List of findings with the same weakness + poam_id: POAM ID to use for the new POAM + + Returns: + PoamEntry combining all findings + + Raises: + ValueError: If findings have different weakness names + """ + if not findings: + raise ValueError("Cannot create POAM from empty findings list") + + # Verify all findings have the same weakness name + weakness_name = findings[0].weakness_name + if not all(f.weakness_name == weakness_name for f in findings): + raise ValueError("All findings must have the same weakness name") + + # Combine asset identifiers and finding IDs + asset_identifiers = [f.asset_identifier for f in findings] + finding_ids = [f.finding_id for f in findings] + + # Use the first finding as a template + first = findings[0] + return PoamEntry( + poam_id=poam_id, + controls=first.controls, + weakness_name=first.weakness_name, + weakness_description=first.weakness_description, + weakness_detector_source=first.weakness_detector_source, + weakness_source_identifier=first.weakness_source_identifier, + asset_identifier=", ".join(asset_identifiers), + point_of_contact=first.point_of_contact, + resources_required=first.resources_required, + overall_remediation_plan=first.overall_remediation_plan, + original_detection_date=first.original_detection_date, + scheduled_completion_date=first.scheduled_completion_date, + planned_milestones=first.planned_milestones, + milestone_changes=first.milestone_changes, + status_date=first.status_date, + vendor_dependency=first.vendor_dependency, + last_vendor_check_in_date=first.last_vendor_check_in_date, + vendor_dependent_product_name=first.vendor_dependent_product_name, + original_risk_rating=first.original_risk_rating, + adjusted_risk_rating=first.adjusted_risk_rating, + risk_adjustment=first.risk_adjustment, + false_positive=first.false_positive, + operational_requirement=first.operational_requirement, + deviation_rationale=first.deviation_rationale, + supporting_documents=first.supporting_documents, + comments=", ".join(finding_ids), # Store finding IDs in comments + auto_approve=first.auto_approve, + binding_operational_directive_22_01_tracking=first.binding_operational_directive_22_01_tracking, + binding_operational_directive_22_01_due_date=first.binding_operational_directive_22_01_due_date, + cve=first.cve, + service_name=first.service_name + ) + +def group_findings_by_weakness(findings: List[Finding]) -> Dict[str, List[Finding]]: + """ + Group findings by weakness name. + + Args: + findings: List of findings to group + + Returns: + Dictionary mapping weakness names to lists of findings, with findings sorted by ID + """ + groups: Dict[str, List[Finding]] = {} + for finding in findings: + groups.setdefault(finding.weakness_name, []).append(finding) + + # Sort each group by finding ID + for findings_list in groups.values(): + findings_list.sort(key=lambda f: f.finding_id) + + return groups + +def generate_poams_from_findings(findings: List[Finding], existing_poam_ids: List[str], current_year: int = None) -> List[Tuple[List[Finding], PoamEntry]]: + """ + Generate new POAMs from a list of findings. + + Args: + findings: List of findings to convert to POAMs + existing_poam_ids: List of existing POAM IDs + current_year: Optional year to use (defaults to current year) + + Returns: + List of tuples containing (findings_list, generated_poam), sorted by first finding ID + """ + # Group findings by weakness + grouped_findings = group_findings_by_weakness(findings) + + # Sort groups by the first finding ID in each group + sorted_groups = sorted( + grouped_findings.values(), + key=lambda findings_list: findings_list[0].finding_id if findings_list else "" + ) + + # Generate POAMs for each group + result = [] + current_year = datetime.now().year if current_year is None else current_year + next_id = get_next_trivy_id(existing_poam_ids, current_year) + + for findings_list in sorted_groups: + poam = findings_to_poam(findings_list, next_id) + result.append((findings_list, poam)) + + # Get next ID for the next group + next_id = get_next_trivy_id([*existing_poam_ids, next_id], current_year) + + return result \ No newline at end of file From 558b2227ac173693d1a141ece4858a1b5cbdb388 Mon Sep 17 00:00:00 2001 From: Chris Llanwarne Date: Fri, 16 May 2025 12:12:53 -0400 Subject: [PATCH 05/33] merging working --- cli/cli.py | 23 ++++++ tools/trivy/diff_apply.py | 152 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 175 insertions(+) create mode 100644 tools/trivy/diff_apply.py diff --git a/cli/cli.py b/cli/cli.py index a83a548..51f13d8 100755 --- a/cli/cli.py +++ b/cli/cli.py @@ -16,6 +16,7 @@ from tools.trivy.alerts import convert_alerts_to_poam from tools.trivy.importer import import_alerts_from_csv from tools.trivy.diff import compare_findings_to_trivy_poams +from tools.trivy.diff_apply import apply_diff_from_files @click.group() def cli(): @@ -172,5 +173,27 @@ def alerts_diff(poam_file: Path, alerts_csv: Path): click.echo(f"Error comparing alerts: {str(e)}", err=True) sys.exit(1) +@cli.command() +@click.argument('poam_file', type=click.Path(exists=True, path_type=Path)) +@click.argument('diff_file', type=click.Path(exists=True, path_type=Path)) +def apply_diff(poam_file: Path, diff_file: Path) -> None: + """Apply diff changes to a POAM Excel file. + + POAM_FILE: Excel file containing POAMs + DIFF_FILE: JSON file containing diff changes + + This command will: + - Add new POAMs to the Open POA&M Items sheet + - Move reopened POAMs from Closed to Open sheet + - Move closed POAMs from Open to Closed sheet + """ + try: + apply_diff_from_files(poam_file, diff_file) + click.echo(f"Successfully applied diff changes to {poam_file}") + except Exception as e: + click.echo(f"Error applying diff: {str(e)}", err=True) + raise e + sys.exit(1) + if __name__ == '__main__': cli() \ No newline at end of file diff --git a/tools/trivy/diff_apply.py b/tools/trivy/diff_apply.py new file mode 100644 index 0000000..0bf3950 --- /dev/null +++ b/tools/trivy/diff_apply.py @@ -0,0 +1,152 @@ +""" +Module for applying Trivy diff changes to POAM Excel files. +""" +from pathlib import Path +import json +from typing import Dict, Any, List +from datetime import datetime +import shutil +import openpyxl + +def create_updateable_copy(file_path: Path) -> Path: + """Create a timestamped backup copy of the Excel file.""" + timestamp = datetime.now().strftime('%Y%m%d-%H%M%S') + backup_path = file_path.parent / f"{file_path.stem}-diff-applied-{timestamp}{file_path.suffix}" + shutil.copy2(file_path, backup_path) + return backup_path + +def dict_to_row(data: Dict[str, Any]) -> Dict[str, Any]: + """Convert a dictionary to row format.""" + excel_mapping = { + "poam_id": "POAM ID", + "controls": "Controls", + "weakness_name": "Weakness Name", + "weakness_description": "Weakness Description", + "weakness_detector_source": "Weakness Detector Source", + "weakness_source_identifier": "Weakness Source Identifier", + "asset_identifier": "Asset Identifier", + "point_of_contact": "Point of Contact", + "resources_required": "Resources Required", + "overall_remediation_plan": "Overall Remediation Plan", + "original_detection_date": "Original Detection Date", + "scheduled_completion_date": "Scheduled Completion Date", + "planned_milestones": "Planned Milestones", + "milestone_changes": "Milestone Changes", + "status_date": "Status Date", + "vendor_dependency": "Vendor Dependency", + "last_vendor_check_in_date": "Last Vendor Check-in Date", + "vendor_dependent_product_name": "Vendor Dependent Product Name", + "original_risk_rating": "Original Risk Rating", + "adjusted_risk_rating": "Adjusted Risk Rating", + "risk_adjustment": "Risk Adjustment", + "false_positive": "False Positive", + "operational_requirement": "Operational Requirement", + "deviation_rationale": "Deviation Rationale", + "supporting_documents": "Supporting Documents", + "comments": "Comments", + "auto_approve": "Auto Approve", + "binding_operational_directive_22_01_tracking": "Binding Operational Directive 22-01 Tracking", + "binding_operational_directive_22_01_due_date": "Binding Operational Directive 22-01 Due Date", + "cve": "CVE", + "service_name": "Service Name" + } + return {excel_mapping[k]: v for k, v in data.items() if k in excel_mapping} + +def apply_diff(poam_file: Path, diff_json: Dict[str, Any]) -> None: + """ + Apply diff changes to a POAM Excel file. + + Args: + poam_file: Path to the POAM Excel file + diff_json: Dictionary containing diff changes + """ + # Create backup copy + backup_file = create_updateable_copy(poam_file) + + try: + # Load workbook from backup copy + wb = openpyxl.load_workbook(backup_file) + + # Get sheets + if "Open POA&M Items" not in wb.sheetnames: + raise ValueError('Excel file must contain "Open POA&M Items" sheet') + open_sheet = wb["Open POA&M Items"] + + # Get or create closed sheet + if "Closed POA&M Items" not in wb.sheetnames: + raise ValueError('Excel file must contain "Open POA&M Items" sheet') + else: + closed_sheet = wb["Closed POA&M Items"] + + # Get column indices from header row (row 5) + header_row = 5 + open_headers = {cell.value: cell.column for cell in open_sheet[header_row]} + + # Handle new POAMs - add to open sheet + if diff_json.get("new_poams"): + for new_poam in diff_json["new_poams"]: + row_data = dict_to_row(new_poam["poam"]) + # Add row at the end + next_row = open_sheet.max_row + 1 + for header, value in row_data.items(): + if header in open_headers: + open_sheet.cell(row=next_row, column=open_headers[header], value=value) + + # Handle reopened POAMs - move from closed to open + if diff_json.get("reopen_poams"): + reopen_ids = {p["poam_id"] for p in diff_json["reopen_poams"]} + poam_id_col = next(col for header, col in open_headers.items() if header == "POAM ID") + + # Find and move rows + rows_to_delete = [] + for row in range(header_row + 1, closed_sheet.max_row + 1): + poam_id = closed_sheet.cell(row=row, column=poam_id_col).value + if poam_id in reopen_ids: + # Copy row to open sheet + next_row = open_sheet.max_row + 1 + for col in range(1, closed_sheet.max_column + 1): + open_sheet.cell(row=next_row, column=col, value=closed_sheet.cell(row=row, column=col).value) + rows_to_delete.append(row) + + # Delete moved rows from closed sheet (in reverse order to maintain indices) + for row in sorted(rows_to_delete, reverse=True): + closed_sheet.delete_rows(row) + + # Handle closed POAMs - move from open to closed + if diff_json.get("close_poams"): + close_ids = set(diff_json["close_poams"]) + poam_id_col = next(col for header, col in open_headers.items() if header == "POAM ID") + + # Find and move rows + rows_to_delete = [] + for row in range(header_row + 1, open_sheet.max_row + 1): + poam_id = open_sheet.cell(row=row, column=poam_id_col).value + if poam_id in close_ids: + # Copy row to closed sheet + next_row = closed_sheet.max_row + 1 + for col in range(1, open_sheet.max_column + 1): + closed_sheet.cell(row=next_row, column=col, value=open_sheet.cell(row=row, column=col).value) + rows_to_delete.append(row) + + # Delete moved rows from open sheet (in reverse order to maintain indices) + for row in sorted(rows_to_delete, reverse=True): + open_sheet.delete_rows(row) + + # Save changes to the backup file + wb.save(backup_file) + + except Exception as e: + # If anything goes wrong, leave the backup file for inspection + raise type(e)(f"Error applying diff changes. Backup saved as {backup_file}. Error: {str(e)}") from e + +def apply_diff_from_files(poam_file: Path, diff_file: Path) -> None: + """ + Apply diff changes from a JSON file to a POAM Excel file. + + Args: + poam_file: Path to the POAM Excel file + diff_file: Path to the JSON diff file + """ + with open(diff_file, 'r') as f: + diff_json = json.load(f) + apply_diff(poam_file, diff_json) \ No newline at end of file From 3a07e574dd59e11d4f33d65e4bb16fc4620e68bc Mon Sep 17 00:00:00 2001 From: Chris Llanwarne Date: Fri, 16 May 2025 14:58:19 -0400 Subject: [PATCH 06/33] cli groups --- README.md | 41 +++++++++++++++++++++--- cli/README.md | 89 +++++++++++++++------------------------------------ cli/cli.py | 81 ++++++++++------------------------------------ 3 files changed, 78 insertions(+), 133 deletions(-) diff --git a/README.md b/README.md index cee093d..bb798b3 100644 --- a/README.md +++ b/README.md @@ -45,6 +45,42 @@ pip install -r requirements.txt The application includes command line tools for automation and data management. These tools are available through the `cli.py` script in the `cli` directory. +### Command Groups + +The CLI is organized into the following command groups: + +1. `poams` - Commands for working with POAMs: + ```bash + # Preview Trivy POAMs from an Excel file + ./cli/cli.py poams preview-trivy [--limit ] + + # Apply diff changes to a POAM Excel file + ./cli/cli.py poams apply-diff + ``` + +2. `trivy` - Commands for working with Trivy: + ```bash + # Download Trivy alerts from GitHub code scanning API + ./cli/cli.py trivy download-alerts + + # Convert GitHub Trivy alerts JSON to POAM CSV format + ./cli/cli.py trivy convert-alerts + + # Compare Trivy alerts against existing POAMs + ./cli/cli.py trivy alerts-diff + ``` + +To see all available commands and their options: +```bash +./cli/cli.py --help +``` + +For help on a specific command group: +```bash +./cli/cli.py poams --help +./cli/cli.py trivy --help +``` + ### Authentication The tools that interact with Google services use Application Default Credentials (ADC). To set up authentication: @@ -72,11 +108,6 @@ export GOOGLE_APPLICATION_CREDENTIALS="/path/to/service-account-key.json" The downloaded files will be saved to the `working` directory in the project root. -To see all available commands and their options: -```bash -./cli/cli.py --help -``` - ## Web Application Usage 1. Start the Streamlit application (make sure you're in the repository root directory): diff --git a/cli/README.md b/cli/README.md index 3851366..acbde4b 100644 --- a/cli/README.md +++ b/cli/README.md @@ -56,99 +56,60 @@ The test suite includes: ## Commands -### Download Alerts +The CLI is organized into command groups for better organization and usability. -Download Trivy alerts from GitHub's code scanning API: +### POAM Commands -```bash -./cli.py download-alerts -``` - -This command: -- Downloads Trivy alerts from GitHub's code scanning API -- Saves them as a JSON file in the working directory -- Requires either: - 1. GitHub CLI (`gh`) installed and authenticated via `gh auth login` - 2. GitHub token provided via `GITHUB_TOKEN` environment variable - -### Convert Alerts - -Convert downloaded GitHub Trivy alerts from JSON to CSV format: +Commands for working with POAMs are grouped under the `poams` command: ```bash -./cli.py convert-alerts -``` - -This command: -- Takes a JSON file containing GitHub code scanning alerts -- Converts the alerts to a CSV format suitable for findings tracking -- Saves the output as a CSV file in the working directory - -### Import and View Alerts +# Preview POAMs from an Excel file +./cli.py poams preview-trivy [--limit ] -Import alerts from CSV and display the first entry in YAML format: - -```bash -./cli.py import-alerts +# Apply diff changes to a POAM Excel file +./cli.py poams apply-diff ``` -This command: -- Reads a CSV file containing Trivy alerts -- Converts each row into a Finding object -- Displays the first finding in YAML format for review +### Trivy Commands -### Compare Alerts with POAMs - -Compare current Trivy alerts against existing POAMs: +Commands for working with Trivy alerts are grouped under the `trivy` command: ```bash -./cli.py alerts-diff -``` +# Download Trivy alerts from GitHub's code scanning API +./cli.py trivy download-alerts -This command: -- Reads existing POAMs from an Excel file -- Compares them against current findings from a CSV file -- Shows: - - New findings that need POAMs created - - Existing findings that already have POAMs (with confidence scores) - - Closed POAMs that no longer have corresponding findings -- Matching is done based on: - - Weakness name similarity - - Asset identifier matching +# Convert downloaded GitHub Trivy alerts from JSON to CSV format +./cli.py trivy convert-alerts -### Preview Trivy POAMs - -Preview POAMs from an Excel file: - -```bash -./cli.py preview-trivy [--limit ] +# Compare current Trivy alerts against existing POAMs +./cli.py trivy alerts-diff ``` -This command: -- Reads POAMs from an Excel file -- Displays a preview of the first n entries (default: 5) -- Requires an Excel file with an "Open POA&M Items" sheet and headers in row 5 +Each command includes error handling and will provide helpful error messages if something goes wrong. ## Example Workflow 1. Download alerts from GitHub: ```bash - ./cli.py download-alerts + ./cli.py trivy download-alerts ``` 2. Convert the downloaded JSON to CSV: ```bash - ./cli.py convert-alerts alerts_20240513.json + ./cli.py trivy convert-alerts alerts_20240513.json ``` 3. Compare new alerts against existing POAMs: ```bash - ./cli.py alerts-diff existing_poams.xlsx working/trivy_alerts_20240513_180947.csv + ./cli.py trivy alerts-diff existing_poams.xlsx working/trivy_alerts_20240513_180947.csv ``` -4. Import and verify specific alerts: +4. Preview POAMs in an Excel file: ```bash - ./cli.py import-alerts working/trivy_alerts_20240513_180947.csv + ./cli.py poams preview-trivy existing_poams.xlsx ``` -Each command includes error handling and will provide helpful error messages if something goes wrong. \ No newline at end of file +5. Apply diff changes to update POAMs: + ```bash + ./cli.py poams apply-diff existing_poams.xlsx alerts_20240513.diff.json + ``` \ No newline at end of file diff --git a/cli/cli.py b/cli/cli.py index 51f13d8..04830c4 100755 --- a/cli/cli.py +++ b/cli/cli.py @@ -17,13 +17,24 @@ from tools.trivy.importer import import_alerts_from_csv from tools.trivy.diff import compare_findings_to_trivy_poams from tools.trivy.diff_apply import apply_diff_from_files +from tools.findings import import_alerts_from_csv @click.group() def cli(): - """Security findings management CLI.""" + """Security tools CLI.""" pass -@cli.command() +@cli.group() +def poams(): + """Commands for working with POAMs.""" + pass + +@cli.group() +def trivy(): + """Commands for working with Trivy.""" + pass + +@poams.command('preview-trivy') @click.argument('file_path', type=click.Path(exists=True)) @click.option('--limit', '-n', default=5, help='Number of POAMs to preview') def preview_trivy(file_path, limit): @@ -40,7 +51,7 @@ def preview_trivy(file_path, limit): click.echo(f"Error: {str(e)}", err=True) sys.exit(1) -@cli.command() +@trivy.command('download-alerts') def download_alerts(): """Download Trivy alerts from GitHub code scanning API. @@ -60,7 +71,7 @@ def download_alerts(): click.echo(f"Error: {str(e)}", err=True) sys.exit(1) -@cli.command() +@trivy.command('convert-alerts') @click.argument('alerts_file', type=click.Path(exists=True)) def convert_alerts(alerts_file): """Convert GitHub Trivy alerts JSON to POAM CSV format. @@ -78,64 +89,7 @@ def convert_alerts(alerts_file): click.echo(f"Error: {str(e)}", err=True) sys.exit(1) -@cli.command() -@click.argument('csv_file', type=click.Path(exists=True, path_type=Path)) -def import_alerts(csv_file: Path): - """ - Import Trivy alerts from a CSV file and display the first entry in YAML format. - - CSV_FILE: Path to the CSV file containing Trivy alerts - """ - try: - findings = import_alerts_from_csv(csv_file) - if not findings: - click.echo("No findings found in CSV file", err=True) - sys.exit(1) - - # Get the first finding and convert to dict for YAML output - first_finding = findings[0] - finding_dict = { - 'finding_id': first_finding.finding_id, - 'controls': first_finding.controls, - 'weakness_name': first_finding.weakness_name, - 'weakness_description': first_finding.weakness_description, - 'weakness_detector_source': first_finding.weakness_detector_source, - 'weakness_source_identifier': first_finding.weakness_source_identifier, - 'asset_identifier': first_finding.asset_identifier, - 'point_of_contact': first_finding.point_of_contact, - 'resources_required': first_finding.resources_required, - 'overall_remediation_plan': first_finding.overall_remediation_plan, - 'original_detection_date': first_finding.original_detection_date.strftime("%Y-%m-%d"), - 'scheduled_completion_date': first_finding.scheduled_completion_date.strftime("%Y-%m-%d"), - 'planned_milestones': first_finding.planned_milestones, - 'milestone_changes': first_finding.milestone_changes, - 'status_date': first_finding.status_date.strftime("%Y-%m-%d"), - 'vendor_dependency': first_finding.vendor_dependency, - 'last_vendor_check_in_date': first_finding.last_vendor_check_in_date.strftime("%Y-%m-%d") if first_finding.last_vendor_check_in_date else None, - 'vendor_dependent_product_name': first_finding.vendor_dependent_product_name, - 'original_risk_rating': first_finding.original_risk_rating, - 'adjusted_risk_rating': first_finding.adjusted_risk_rating, - 'risk_adjustment': first_finding.risk_adjustment, - 'false_positive': first_finding.false_positive, - 'operational_requirement': first_finding.operational_requirement, - 'deviation_rationale': first_finding.deviation_rationale, - 'supporting_documents': first_finding.supporting_documents, - 'comments': first_finding.comments, - 'auto_approve': first_finding.auto_approve, - 'binding_operational_directive_22_01_tracking': first_finding.binding_operational_directive_22_01_tracking, - 'binding_operational_directive_22_01_due_date': first_finding.binding_operational_directive_22_01_due_date.strftime("%Y-%m-%d") if first_finding.binding_operational_directive_22_01_due_date else None, - 'cve': first_finding.cve, - 'service_name': first_finding.service_name - } - - # Output as YAML - click.echo(yaml.dump(finding_dict, sort_keys=False)) - - except Exception as e: - click.echo(f"Error importing alerts: {str(e)}", err=True) - sys.exit(1) - -@cli.command() +@trivy.command('alerts-diff') @click.argument('poam_file', type=click.Path(exists=True, path_type=Path)) @click.argument('alerts_csv', type=click.Path(exists=True, path_type=Path)) def alerts_diff(poam_file: Path, alerts_csv: Path): @@ -173,7 +127,7 @@ def alerts_diff(poam_file: Path, alerts_csv: Path): click.echo(f"Error comparing alerts: {str(e)}", err=True) sys.exit(1) -@cli.command() +@poams.command('apply-diff') @click.argument('poam_file', type=click.Path(exists=True, path_type=Path)) @click.argument('diff_file', type=click.Path(exists=True, path_type=Path)) def apply_diff(poam_file: Path, diff_file: Path) -> None: @@ -192,7 +146,6 @@ def apply_diff(poam_file: Path, diff_file: Path) -> None: click.echo(f"Successfully applied diff changes to {poam_file}") except Exception as e: click.echo(f"Error applying diff: {str(e)}", err=True) - raise e sys.exit(1) if __name__ == '__main__': From 3313a1c42cbbfcf3a0a2ff5eebc991b2c2536e94 Mon Sep 17 00:00:00 2001 From: Chris Llanwarne Date: Sat, 17 May 2025 09:48:36 -0400 Subject: [PATCH 07/33] zap alerts to findings --- cli/README.md | 20 ++++++ cli/cli.py | 33 +++++++++- tools/zap/__init__.py | 7 +++ tools/zap/alerts.py | 142 ++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 201 insertions(+), 1 deletion(-) create mode 100644 tools/zap/__init__.py create mode 100644 tools/zap/alerts.py diff --git a/cli/README.md b/cli/README.md index acbde4b..958fb2c 100644 --- a/cli/README.md +++ b/cli/README.md @@ -85,6 +85,26 @@ Commands for working with Trivy alerts are grouped under the `trivy` command: ./cli.py trivy alerts-diff ``` +### ZAP Commands + +Commands for working with ZAP scan reports are grouped under the `zap` command: + +```bash +# Convert ZAP XML alerts to findings JSON format +./cli.py zap alerts-to-findings +``` + +This command: +- Takes a ZAP XML report file as input +- Converts each alert to a finding object with: + - Finding ID (based on ZAP plugin ID) + - Weakness name and description + - Asset identifier (host) + - Risk rating and confidence level + - List of affected instances (URLs, methods, parameters) +- Saves the findings as a JSON file +- Displays the first finding and total count + Each command includes error handling and will provide helpful error messages if something goes wrong. ## Example Workflow diff --git a/cli/cli.py b/cli/cli.py index 04830c4..9ccbbe6 100755 --- a/cli/cli.py +++ b/cli/cli.py @@ -17,7 +17,7 @@ from tools.trivy.importer import import_alerts_from_csv from tools.trivy.diff import compare_findings_to_trivy_poams from tools.trivy.diff_apply import apply_diff_from_files -from tools.findings import import_alerts_from_csv +from tools.zap import convert_alerts_to_findings @click.group() def cli(): @@ -34,6 +34,11 @@ def trivy(): """Commands for working with Trivy.""" pass +@cli.group() +def zap(): + """Commands for working with ZAP scan reports.""" + pass + @poams.command('preview-trivy') @click.argument('file_path', type=click.Path(exists=True)) @click.option('--limit', '-n', default=5, help='Number of POAMs to preview') @@ -148,5 +153,31 @@ def apply_diff(poam_file: Path, diff_file: Path) -> None: click.echo(f"Error applying diff: {str(e)}", err=True) sys.exit(1) +@zap.command('alerts-to-findings') +@click.argument('xml_file', type=click.Path(exists=True)) +def alerts_to_findings(xml_file): + """Convert ZAP XML alerts to findings JSON format. + + XML_FILE should be a ZAP XML report file. + The findings will be saved as a JSON file and the first finding will be displayed. + """ + try: + # Convert alerts to findings + output_file = convert_alerts_to_findings(xml_file) + + # Load and display first finding + with open(output_file) as f: + findings = json.load(f) + if findings: + click.echo("\nFirst finding from the report:") + click.echo(json.dumps(findings[0], indent=2)) + click.echo(f"\nTotal findings: {len(findings)}") + click.echo(f"All findings saved to: {output_file}") + else: + click.echo("No findings found in the report.") + except Exception as e: + click.echo(f"Error converting alerts: {str(e)}", err=True) + sys.exit(1) + if __name__ == '__main__': cli() \ No newline at end of file diff --git a/tools/zap/__init__.py b/tools/zap/__init__.py new file mode 100644 index 0000000..f1121f6 --- /dev/null +++ b/tools/zap/__init__.py @@ -0,0 +1,7 @@ +""" +Package for handling ZAP scan reports and findings. +""" + +from .alerts import convert_alerts_to_findings + +__all__ = ['convert_alerts_to_findings'] \ No newline at end of file diff --git a/tools/zap/alerts.py b/tools/zap/alerts.py new file mode 100644 index 0000000..4fbf996 --- /dev/null +++ b/tools/zap/alerts.py @@ -0,0 +1,142 @@ +""" +Module for handling ZAP scan reports and converting alerts to findings. +""" +import xml.etree.ElementTree as ET +from datetime import datetime, timedelta +from typing import List +import json +from pathlib import Path +from ..findings import Finding + +def get_completion_date(severity: str, detection_date: datetime) -> datetime: + """Calculate completion date based on severity.""" + days_map = { + 'Critical': 15, + 'High': 30, + 'Medium': 90, # Using 90 days for medium + 'Low': 180, + 'Informational': 180 + } + days = days_map.get(severity, 180) # Default to 180 days if unknown severity + return detection_date + timedelta(days=days) + +def parse_zap_xml(xml_file: str) -> List[Finding]: + """ + Parse a ZAP XML report and extract alert findings. + + Args: + xml_file: Path to the ZAP XML report file + + Returns: + List of Finding objects + """ + tree = ET.parse(xml_file) + root = tree.getroot() + + findings = [] + + # Extract scan date from report + scan_date = datetime.strptime(root.get('generated'), '%a, %d %b %Y %H:%M:%S') + + # Process each alert + for site in root.findall('.//site'): + for alertitem in site.findall('.//alertitem'): + # Get basic alert info + alert_name = alertitem.find('alert').text + risk_code = int(alertitem.find('riskcode').text) + description = alertitem.find('desc').text + plugin_id = alertitem.find('pluginid').text + + # Map risk code to severity + severity_map = { + 0: 'Informational', + 1: 'Low', + 2: 'Medium', + 3: 'High' + } + severity = severity_map.get(risk_code, 'Unknown') + + # Calculate completion date based on severity + completion_date = get_completion_date(severity, scan_date) + + # Create a finding for each instance + for idx, instance in enumerate(alertitem.findall('.//instance')): + uri = instance.find('uri').text + evidence = instance.find('evidence').text if instance.find('evidence') is not None else None + other_info = instance.find('otherinfo').text if instance.find('otherinfo') is not None else None + + # Add evidence and other info to description if available + full_description = description + if evidence: + full_description += f"\n\nEvidence:\n{evidence}" + if other_info: + full_description += f"\n\nAdditional Information:\n{other_info}" + + finding = Finding( + finding_id=f"ZAP-{plugin_id}-{idx+1}", + controls="RA-5", + weakness_name=alert_name, + weakness_description=full_description, + weakness_detector_source="ZAP", + weakness_source_identifier=plugin_id, + asset_identifier=uri, + point_of_contact="Chris Llanwarne", + resources_required="None", + overall_remediation_plan="Perform necessary updates to resolve the vulnerability", + original_detection_date=scan_date, + scheduled_completion_date=completion_date, + planned_milestones=f"(1) {completion_date.strftime('%Y-%m-%d')}: Perform necessary updates to resolve the vulnerability", + milestone_changes="", + status_date=scan_date, + vendor_dependency="No", + last_vendor_check_in_date=None, + vendor_dependent_product_name="N/A", + original_risk_rating=severity, + adjusted_risk_rating=None, + risk_adjustment="", + false_positive="", + operational_requirement="", + deviation_rationale=None, + supporting_documents=None, + comments=None, + auto_approve="", + binding_operational_directive_22_01_tracking="", + binding_operational_directive_22_01_due_date=None, + cve=None, + service_name="Hail" + ) + findings.append(finding) + + return findings + +def convert_alerts_to_findings(xml_file: str) -> str: + """ + Convert ZAP XML alerts to findings JSON format. + + Args: + xml_file: Path to the ZAP XML report file + + Returns: + Path to the output JSON file + """ + findings = parse_zap_xml(xml_file) + + # Convert findings to dictionaries + findings_data = [] + for finding in findings: + finding_dict = vars(finding) + # Convert datetime objects to strings + for key, value in finding_dict.items(): + if isinstance(value, datetime): + finding_dict[key] = value.strftime('%Y-%m-%d %H:%M:%S') + findings_data.append(finding_dict) + + # Generate output filename + input_path = Path(xml_file) + output_file = input_path.with_suffix('.findings.json') + + # Write findings to JSON file + with open(output_file, 'w') as f: + json.dump(findings_data, f, indent=2) + + return str(output_file) \ No newline at end of file From 41da6c9848764061d140671e86582f61e0da0c56 Mon Sep 17 00:00:00 2001 From: Chris Llanwarne Date: Sat, 17 May 2025 17:37:23 -0400 Subject: [PATCH 08/33] zap diff working --- cli/cli.py | 34 ++++ tools/zap/diff.py | 300 ++++++++++++++++++++++++++++++++++++ tools/zap/poam_generator.py | 171 ++++++++++++++++++++ 3 files changed, 505 insertions(+) create mode 100644 tools/zap/diff.py create mode 100644 tools/zap/poam_generator.py diff --git a/cli/cli.py b/cli/cli.py index 9ccbbe6..328e658 100755 --- a/cli/cli.py +++ b/cli/cli.py @@ -7,10 +7,12 @@ from pathlib import Path import yaml from datetime import datetime +from typing import Optional # Add the project root to the Python path sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) +from tools.findings import Finding from tools.poam import PoamFile from tools.github import download_trivy_alerts from tools.trivy.alerts import convert_alerts_to_poam @@ -18,6 +20,7 @@ from tools.trivy.diff import compare_findings_to_trivy_poams from tools.trivy.diff_apply import apply_diff_from_files from tools.zap import convert_alerts_to_findings +from tools.zap.diff import compare_findings_to_zap_poams @click.group() def cli(): @@ -179,5 +182,36 @@ def alerts_to_findings(xml_file): click.echo(f"Error converting alerts: {str(e)}", err=True) sys.exit(1) +@zap.command('alerts-diff') +@click.argument('findings_file', type=click.Path(exists=True)) +@click.argument('poam_file', type=click.Path(exists=True)) +@click.option('--json-output', type=click.Path(), help='Path to save JSON output') +def alerts_diff(findings_file: str, poam_file: str, json_output: Optional[str]) -> None: + """Compare ZAP findings against existing POAMs.""" + try: + # Load findings from JSON file + with open(findings_file) as f: + findings_data = json.load(f) + findings = [Finding.from_dict(f) for f in findings_data] + + # Compare findings to POAMs + diff = compare_findings_to_zap_poams(findings, poam_file) + + # Print human readable summary + diff.print_summary() + + # Save JSON output if requested + if not json_output: + # Finding file as a path: + findings_path = Path(findings_file) + json_output = findings_path.with_suffix('.diff.json') + json_data = diff.to_json() + with open(json_output, 'w') as f: + json.dump(json_data, f, indent=2) + click.echo(f"JSON output saved to: {json_output}") + except Exception as e: + click.echo(f"Error comparing findings: {str(e)}", err=True) + sys.exit(1) + if __name__ == '__main__': cli() \ No newline at end of file diff --git a/tools/zap/diff.py b/tools/zap/diff.py new file mode 100644 index 0000000..dcdafcc --- /dev/null +++ b/tools/zap/diff.py @@ -0,0 +1,300 @@ +""" +Module for comparing ZAP findings against existing POAMs. +""" +from dataclasses import dataclass +from typing import List, Optional, Tuple, Dict, Any +from pathlib import Path +from datetime import datetime + +from ..findings import Finding +from ..poam import PoamFile, PoamEntry +from .poam_generator import generate_poams_from_findings + +@dataclass +class FindingPoamMatch: + """Represents a match between a finding and an existing POAM.""" + finding: Finding + poam: PoamEntry + +@dataclass +class ZapAlertsDiff: + """Represents the difference between current findings and existing POAMs.""" + new_findings: List[Finding] # Findings without corresponding POAMs + existing_matches: List[FindingPoamMatch] # Findings matched to existing POAMs + closed_poams: List[PoamEntry] # POAMs without corresponding findings + reopened_findings: List[FindingPoamMatch] # Findings that match previously closed POAMs + proposed_poams: List[Tuple[List[Finding], PoamEntry]] # Proposed new POAMs with their findings + + def to_json(self) -> Dict[str, Any]: + """ + Convert the diff results to a JSON-serializable dictionary. + + Returns: + Dictionary containing the diff results in a structured format + """ + def format_datetime(dt: datetime | str) -> str: + if isinstance(dt, str): + return dt + else: + return dt.strftime("%Y-%m-%d") if dt else None + + def poam_to_full_dict(poam: PoamEntry) -> Dict[str, Any]: + """Convert a POAM to a complete dictionary with all fields.""" + return { + "poam_id": poam.poam_id, + "controls": poam.controls, + "weakness_name": poam.weakness_name, + "weakness_description": poam.weakness_description, + "weakness_detector_source": poam.weakness_detector_source, + "weakness_source_identifier": poam.weakness_source_identifier, + "asset_identifier": poam.asset_identifier, + "point_of_contact": poam.point_of_contact, + "resources_required": poam.resources_required, + "overall_remediation_plan": poam.overall_remediation_plan, + "original_detection_date": format_datetime(poam.original_detection_date), + "scheduled_completion_date": format_datetime(poam.scheduled_completion_date), + "planned_milestones": poam.planned_milestones, + "milestone_changes": poam.milestone_changes, + "status_date": format_datetime(poam.status_date), + "vendor_dependency": poam.vendor_dependency, + "last_vendor_check_in_date": format_datetime(poam.last_vendor_check_in_date), + "vendor_dependent_product_name": poam.vendor_dependent_product_name, + "original_risk_rating": poam.original_risk_rating, + "adjusted_risk_rating": poam.adjusted_risk_rating, + "risk_adjustment": poam.risk_adjustment, + "false_positive": poam.false_positive, + "operational_requirement": poam.operational_requirement, + "deviation_rationale": poam.deviation_rationale, + "supporting_documents": poam.supporting_documents, + "comments": poam.comments, + "auto_approve": poam.auto_approve, + "binding_operational_directive_22_01_tracking": poam.binding_operational_directive_22_01_tracking, + "binding_operational_directive_22_01_due_date": format_datetime(poam.binding_operational_directive_22_01_due_date), + "cve": poam.cve, + "service_name": poam.service_name + } + + def finding_to_dict(finding: Finding) -> Dict[str, Any]: + return { + "finding_id": finding.finding_id, + "weakness_name": finding.weakness_name, + "asset_identifier": finding.asset_identifier, + "original_detection_date": format_datetime(finding.original_detection_date), + "original_risk_rating": finding.original_risk_rating, + "cve": finding.cve, + "service_name": finding.service_name + } + + return { + "metadata": { + "new_findings_count": len(self.new_findings), + "existing_matches_count": len(self.existing_matches), + "closed_poams_count": len(self.closed_poams), + "reopened_findings_count": len(self.reopened_findings), + "proposed_poams_count": len(self.proposed_poams) + }, + "new_poams": [ + { + "poam": poam_to_full_dict(poam), + "findings": [finding_to_dict(f) for f in findings], + "finding_ids": [f.finding_id for f in findings] + } + for findings, poam in self.proposed_poams + ], + "reopen_poams": [ + { + "poam_id": match.poam.poam_id, + "finding_id": match.finding.finding_id + } + for match in self.reopened_findings + ], + "close_poams": [poam.poam_id for poam in self.closed_poams] + } + + def print_summary(self, max_preview: int = 10) -> None: + """Print a human-readable summary of the diff.""" + # Print new findings + print("\n=== New Findings ===") + print(f"Count: {len(self.new_findings)}") + if self.new_findings: + finding_ids = [finding.finding_id for finding in self.new_findings] + print(f"Finding IDs: {', '.join(finding_ids)}") + + # Print existing matches + print("\n=== Existing Matches ===") + print(f"Count: {len(self.existing_matches)}") + if self.existing_matches: + matches = [f"{match.finding.finding_id} -> {match.poam.poam_id}" + for match in self.existing_matches[:max_preview]] + print(f"Preview of matches: {', '.join(matches)}") + if len(self.existing_matches) > max_preview: + print(f"... and {len(self.existing_matches) - max_preview} more") + + # Print reopened findings + print("\n=== Reopened Findings ===") + print(f"Count: {len(self.reopened_findings)}") + if self.reopened_findings: + matches = [f"{match.finding.finding_id} -> {match.poam.poam_id}" + for match in self.reopened_findings[:max_preview]] + print(f"Preview of matches: {', '.join(matches)}") + if len(self.reopened_findings) > max_preview: + print(f"... and {len(self.reopened_findings) - max_preview} more") + + # Print closed POAMs + print("\n=== Closed POAMs ===") + print(f"Count: {len(self.closed_poams)}") + if self.closed_poams: + poam_ids = [poam.poam_id for poam in self.closed_poams] + print(f"POAM IDs no longer active: {', '.join(poam_ids)}") + + # Print proposed POAMs + print("\n=== Proposed POAMs ===") + print(f"Count: {len(self.proposed_poams)}") + if self.proposed_poams: + for findings, poam in self.proposed_poams[:max_preview]: + finding_ids = [f.finding_id for f in findings] + print(f"{', '.join(finding_ids)} => {poam.poam_id}") + if len(self.proposed_poams) > max_preview: + print(f"... and {len(self.proposed_poams) - max_preview} more") + + # Show sample of first proposed POAM + print("\nSample new POAM:") + sample_findings, sample_poam = self.proposed_poams[0] + print(f"POAM ID: {sample_poam.poam_id}") + print(f"Weakness Name: {sample_poam.weakness_name}") + print(f"Asset Identifiers: {sample_poam.asset_identifier}") + print(f"Finding IDs: {sample_poam.comments}") + # Handle case where date might already be a string + detection_date = sample_poam.original_detection_date + if isinstance(detection_date, datetime): + detection_date = detection_date.strftime('%Y-%m-%d') + print(f"Detection Date: {detection_date}") + print(f"Risk Rating: {sample_poam.original_risk_rating}") + if sample_poam.cve: + print(f"CVE: {sample_poam.cve}") + +def _is_exact_match(str1: str, str2: str) -> bool: + """Check if two strings match exactly, ignoring case.""" + if not str1 or not str2: + return False + return str1.lower().strip() == str2.lower().strip() + +def _is_asset_covered(finding_asset: str, poam_assets: str) -> bool: + """ + Check if the finding's asset is included in the POAM's asset list. + + Args: + finding_asset: Asset identifier from the finding + poam_assets: Asset identifier field from the POAM (may contain multiple assets) + + Returns: + True if the finding's asset is contained within the POAM's asset list + """ + if not finding_asset or not poam_assets: + return False + return finding_asset.lower().strip() in poam_assets.lower().strip() + +def _find_matching_poam(finding: Finding, poams: List[PoamEntry]) -> Optional[FindingPoamMatch]: + """Find a matching POAM for a given finding based on exact weakness name match and asset coverage.""" + for poam in poams: + # Weakness name must match exactly, and the finding's asset must be included in the POAM's assets + if (_is_exact_match(finding.weakness_name, poam.weakness_name) and + _is_asset_covered(finding.asset_identifier, poam.asset_identifier)): + return FindingPoamMatch(finding=finding, poam=poam) + + return None + +def compare_findings_to_zap_poams(findings: List[Finding], poam_file: Path) -> ZapAlertsDiff: + """ + Compare a list of findings against ZAP POAMs. + + Args: + findings: List of current findings from ZAP + poam_file: Path to Excel file containing ZAP POAMs + + Returns: + ZapAlertsDiff containing new, existing, closed, and reopened findings + """ + # Load ZAP POAMs + poam_file_handler = PoamFile(poam_file) + open_poams, closed_poams = get_zap_poam_entries(poam_file_handler) + + # Get all POAM IDs for generating new ones + all_poam_ids = [p.poam_id for p in [*open_poams, *closed_poams]] + + return compare_findings_to_poams(findings, open_poams, closed_poams, all_poam_ids) + +def get_zap_poam_entries(poam_file: PoamFile) -> Tuple[List[PoamEntry], List[PoamEntry]]: + """ + Get ZAP POAMs from a POAM file. + + Args: + poam_file: PoamFile instance + + Returns: + Tuple of (open_poams, closed_poams) + """ + # Pattern matches YYYY-ZAPXXXX where XXXX is 4 or more digits + zap_pattern = r'^\d{4}-ZAP\d{4,}$' + + # Get open POAMs + open_df = poam_file.df[poam_file.df['POAM ID'].str.match(zap_pattern, na=False)] + open_poams = [PoamEntry.from_dict(row) for _, row in open_df.iterrows()] + + # Get closed POAMs if available + closed_poams = [] + if poam_file.closed_df is not None: + closed_df = poam_file.closed_df[poam_file.closed_df['POAM ID'].str.match(zap_pattern, na=False)] + closed_poams = [PoamEntry.from_dict(row) for _, row in closed_df.iterrows()] + + return open_poams, closed_poams + +def compare_findings_to_poams(findings: List[Finding], + open_poams: List[PoamEntry], + closed_poams: List[PoamEntry], + existing_poam_ids: List[str]) -> ZapAlertsDiff: + """ + Compare a list of findings against existing POAMs. + + Args: + findings: List of current findings from ZAP + open_poams: List of open POAMs + closed_poams: List of closed POAMs + existing_poam_ids: List of all existing POAM IDs + + Returns: + ZapAlertsDiff containing new, existing, closed, and reopened findings + """ + # Track which POAMs are matched + matched_poams = set() + new_findings = [] + existing_matches = [] + reopened_findings = [] + + # First check for matches against open POAMs + for finding in findings: + match = _find_matching_poam(finding, open_poams) + if match: + existing_matches.append(match) + matched_poams.add(match.poam) + else: + # If no match in open POAMs, check closed POAMs + closed_match = _find_matching_poam(finding, closed_poams) + if closed_match: + reopened_findings.append(closed_match) + else: + new_findings.append(finding) + + # Find closed POAMs (those without matches) + closed_poams = [poam for poam in open_poams if poam not in matched_poams] + + # Generate proposed POAMs for new findings + proposed_poams = generate_poams_from_findings(new_findings, existing_poam_ids) + + return ZapAlertsDiff( + new_findings=new_findings, + existing_matches=existing_matches, + closed_poams=closed_poams, + reopened_findings=reopened_findings, + proposed_poams=proposed_poams + ) \ No newline at end of file diff --git a/tools/zap/poam_generator.py b/tools/zap/poam_generator.py new file mode 100644 index 0000000..e5bbc57 --- /dev/null +++ b/tools/zap/poam_generator.py @@ -0,0 +1,171 @@ +""" +Module for generating POAMs from ZAP findings. +""" +from datetime import datetime +from typing import List, Dict, Tuple +import re +from ..findings import Finding +from ..poam import PoamEntry + +def parse_zap_id(poam_id: str) -> Tuple[int, int]: + """ + Parse a ZAP POAM ID into year and sequence components. + + Args: + poam_id: POAM ID in format YYYY-ZAPXXXX + + Returns: + Tuple of (year, sequence_number) + + Raises: + ValueError: If the ID format is invalid + """ + match = re.match(r'^(\d{4})-ZAP(\d{4,})$', poam_id) + if not match: + raise ValueError(f"Invalid ZAP POAM ID format: {poam_id}") + + year = int(match.group(1)) + sequence = int(match.group(2)) + return year, sequence + +def get_next_zap_id(existing_poam_ids: List[str], current_year: int = None) -> str: + """ + Generate the next available ZAP POAM ID. + + Args: + existing_poam_ids: List of existing POAM IDs + current_year: Optional year to use (defaults to current year) + + Returns: + Next available POAM ID in format YYYY-ZAPXXXX + """ + current_year = datetime.now().year if current_year is None else current_year + + # Find highest sequence number for the current year + max_sequence = 0 + for poam_id in existing_poam_ids: + try: + year, sequence = parse_zap_id(poam_id) + if year == current_year: + max_sequence = max(max_sequence, sequence) + except ValueError: + continue # Skip non-ZAP IDs + + return f"{current_year}-ZAP{max_sequence + 1:04d}" + +def findings_to_poam(findings: List[Finding], poam_id: str) -> PoamEntry: + """ + Convert a list of findings with the same weakness into a single POAM. + + Args: + findings: List of findings with the same weakness + poam_id: POAM ID to use for the new POAM + + Returns: + PoamEntry combining all findings + + Raises: + ValueError: If findings have different weakness names + """ + if not findings: + raise ValueError("Cannot create POAM from empty findings list") + + # Verify all findings have the same weakness name + weakness_name = findings[0].weakness_name + if not all(f.weakness_name == weakness_name for f in findings): + raise ValueError("All findings must have the same weakness name") + + # Combine asset identifiers and finding IDs + asset_identifiers = [f.asset_identifier for f in findings] + finding_ids = [f.finding_id for f in findings] + + # Use the first finding as a template + first = findings[0] + return PoamEntry( + poam_id=poam_id, + controls=first.controls, + weakness_name=first.weakness_name, + weakness_description=first.weakness_description, + weakness_detector_source=first.weakness_detector_source, + weakness_source_identifier=first.weakness_source_identifier, + asset_identifier=", ".join(asset_identifiers), + point_of_contact=first.point_of_contact, + resources_required=first.resources_required, + overall_remediation_plan=first.overall_remediation_plan, + original_detection_date=first.original_detection_date, + scheduled_completion_date=first.scheduled_completion_date, + planned_milestones=first.planned_milestones, + milestone_changes=first.milestone_changes, + status_date=first.status_date, + vendor_dependency=first.vendor_dependency, + last_vendor_check_in_date=first.last_vendor_check_in_date, + vendor_dependent_product_name=first.vendor_dependent_product_name, + original_risk_rating=first.original_risk_rating, + adjusted_risk_rating=first.adjusted_risk_rating, + risk_adjustment=first.risk_adjustment, + false_positive=first.false_positive, + operational_requirement=first.operational_requirement, + deviation_rationale=first.deviation_rationale, + supporting_documents=first.supporting_documents, + comments=", ".join(finding_ids), # Store finding IDs in comments + auto_approve=first.auto_approve, + binding_operational_directive_22_01_tracking=first.binding_operational_directive_22_01_tracking, + binding_operational_directive_22_01_due_date=first.binding_operational_directive_22_01_due_date, + cve=first.cve, + service_name=first.service_name + ) + +def group_findings_by_weakness(findings: List[Finding]) -> Dict[str, List[Finding]]: + """ + Group findings by weakness name. + + Args: + findings: List of findings to group + + Returns: + Dictionary mapping weakness names to lists of findings, with findings sorted by ID + """ + groups: Dict[str, List[Finding]] = {} + for finding in findings: + groups.setdefault(finding.weakness_name, []).append(finding) + + # Sort each group by finding ID + for findings_list in groups.values(): + findings_list.sort(key=lambda f: f.finding_id) + + return groups + +def generate_poams_from_findings(findings: List[Finding], existing_poam_ids: List[str], current_year: int = None) -> List[Tuple[List[Finding], PoamEntry]]: + """ + Generate new POAMs from a list of findings. + + Args: + findings: List of findings to convert to POAMs + existing_poam_ids: List of existing POAM IDs + current_year: Optional year to use (defaults to current year) + + Returns: + List of tuples containing (findings_list, generated_poam), sorted by first finding ID + """ + # Group findings by weakness + grouped_findings = group_findings_by_weakness(findings) + + # Sort groups by the first finding ID in each group + sorted_groups = sorted( + grouped_findings.values(), + key=lambda findings_list: findings_list[0].finding_id if findings_list else "" + ) + + # Generate POAMs for each group + result = [] + current_year = datetime.now().year if current_year is None else current_year + next_id = get_next_zap_id(existing_poam_ids, current_year) + + for findings_list in sorted_groups: + poam = findings_to_poam(findings_list, next_id) + result.append((findings_list, poam)) + + # Get next ID for the next group + next_id = get_next_zap_id([*existing_poam_ids, next_id], current_year) + + return result \ No newline at end of file From 4b3653a8e8e16d867d3161452ee019657d306b92 Mon Sep 17 00:00:00 2001 From: Chris Llanwarne Date: Mon, 19 May 2025 16:45:24 -0400 Subject: [PATCH 09/33] first two CIS tools --- cli/cli.py | 62 ++++++++++++++++++ tools/cis/converter.py | 139 +++++++++++++++++++++++++++++++++++++++++ tools/cis/splitter.py | 64 +++++++++++++++++++ 3 files changed, 265 insertions(+) create mode 100644 tools/cis/converter.py create mode 100644 tools/cis/splitter.py diff --git a/cli/cli.py b/cli/cli.py index 328e658..458267c 100755 --- a/cli/cli.py +++ b/cli/cli.py @@ -21,6 +21,8 @@ from tools.trivy.diff_apply import apply_diff_from_files from tools.zap import convert_alerts_to_findings from tools.zap.diff import compare_findings_to_zap_poams +from tools.cis.splitter import split_connected_sheet +from tools.cis.converter import convert_to_findings_file @click.group() def cli(): @@ -42,6 +44,11 @@ def zap(): """Commands for working with ZAP scan reports.""" pass +@cli.group() +def cis(): + """Commands for working with CIS scan reports.""" + pass + @poams.command('preview-trivy') @click.argument('file_path', type=click.Path(exists=True)) @click.option('--limit', '-n', default=5, help='Number of POAMs to preview') @@ -213,5 +220,60 @@ def alerts_diff(findings_file: str, poam_file: str, json_output: Optional[str]) click.echo(f"Error comparing findings: {str(e)}", err=True) sys.exit(1) +@cis.command('split-connected-sheet') +@click.argument('xlsx_file', type=click.Path(exists=True, path_type=Path)) +def split_connected_sheet_cmd(xlsx_file: Path) -> None: + """Split a CIS connected sheet into separate CSV files by date. + + XLSX_FILE should be a CIS connected sheet Excel file. + + The command will: + - Create a "Divided CIS Scans" directory if it doesn't exist + - Split the file into multiple CSVs based on the Date field + - Name each file as " - YYYY-MM-DD.csv" + - Skip writing if a file for a particular date already exists + """ + try: + output_files = split_connected_sheet(xlsx_file) + if output_files: + click.echo(f"Successfully split {xlsx_file.name} into {len(output_files)} files:") + for f in output_files: + click.echo(f" - {f.name}") + else: + click.echo("No new files created (all dates already exist)") + except Exception as e: + click.echo(f"Error splitting connected sheet: {str(e)}", err=True) + sys.exit(1) + +@cis.command('csv-to-findings') +@click.argument('csv_file', type=click.Path(exists=True, path_type=Path)) +def csv_to_findings_cmd(csv_file: Path) -> None: + """Convert a CIS CSV file to findings JSON format. + + CSV_FILE should be a CIS CSV file (typically from split-connected-sheet). + + The command will: + - Convert each row into one or more findings based on the Failures field + - Generate finding IDs in the format CIS--XXXX + - Save the findings as .findings.json + """ + try: + output_file = convert_to_findings_file(csv_file) + + # Load and display summary + with open(output_file) as f: + findings = json.load(f) + click.echo(f"\nSuccessfully converted {csv_file.name} to findings:") + click.echo(f"- Total findings: {len(findings)}") + if findings: + unique_rules = len({f['weakness_name'] for f in findings}) + click.echo(f"- Unique CIS rules: {unique_rules}") + click.echo("\nSample finding:") + click.echo(json.dumps(findings[0], indent=2)) + click.echo(f"\nOutput saved to: {output_file}") + except Exception as e: + click.echo(f"Error converting CSV to findings: {str(e)}", err=True) + sys.exit(1) + if __name__ == '__main__': cli() \ No newline at end of file diff --git a/tools/cis/converter.py b/tools/cis/converter.py new file mode 100644 index 0000000..27b96f2 --- /dev/null +++ b/tools/cis/converter.py @@ -0,0 +1,139 @@ +""" +Module for converting CIS scan reports to findings. +""" +from pathlib import Path +import pandas as pd +from datetime import datetime, timedelta +from typing import List +import json + +from ..findings import Finding + +def get_cvss_range(cvss: str) -> str: + """Convert CVSS score to range category.""" + try: + score = float(cvss) + if score >= 9.0: + return "Critical" + elif score >= 7.0: + return "High" + elif score >= 4.0: + return "Medium" + elif score > 0: + return "Low" + else: + return "Info" + except (ValueError, TypeError): + if not cvss: + return "Info" + else: + return "Unknown" + +def calculate_due_date(cvss: str, detection_date: datetime) -> datetime: + """Calculate due date based on severity level.""" + severity_mapping = { + 'Critical': 15, + 'High': 30, + 'Medium': 90, + 'Low': 180, + 'Info': 180 + } + severity = get_cvss_range(cvss) + days = severity_mapping.get(severity, 180) # Default to 180 days for unknown + return detection_date + timedelta(days=days) + +def convert_csv_to_findings(input_file: Path) -> List[Finding]: + """ + Convert a CIS CSV file to a list of findings. + + Args: + input_file: Path to the input CSV file + + Returns: + List of Finding objects + """ + # Read the CSV file + df = pd.read_csv(input_file) + + # Extract date from filename + date_str = input_file.stem.split(" - ")[-1] + detection_date = datetime.strptime(date_str, "%Y-%m-%d") + + findings = [] + + # Process each row + for _, row in df.iterrows(): + # Split failures into individual asset identifiers + failures = row['Failures'].strip().split('\n') + + # Calculate completion date based on CVSS + completion_date = calculate_due_date(row['CVSS'], detection_date) + + # Create a finding for each failure + for failure in failures: + finding = Finding( + finding_id=f"CIS-{row['CIS_ID']}-{len(findings)+1:04d}", + controls="CM-6", + weakness_name=row['Title'], + weakness_description=row['Description'], + weakness_detector_source=input_file.name, + weakness_source_identifier="CIS", + asset_identifier=failure.strip(), + point_of_contact="Chris Llanwarne", + resources_required=None, + overall_remediation_plan="Perform necessary updates to resolve the vulnerability", + original_detection_date=detection_date, + scheduled_completion_date=completion_date, + planned_milestones=f"(1) {completion_date.strftime('%Y-%m-%d')} Perform necessary updates to resolve the vulnerability", + milestone_changes="", + status_date=detection_date, + vendor_dependency="No", + last_vendor_check_in_date=None, + vendor_dependent_product_name="", + original_risk_rating=get_cvss_range(row['CVSS']), + adjusted_risk_rating="N/A", + risk_adjustment="", + false_positive="No", + operational_requirement="No", + deviation_rationale=None, + supporting_documents=None, + comments=None, + auto_approve="No", + binding_operational_directive_22_01_tracking="", + binding_operational_directive_22_01_due_date=None, + cve=None, + service_name="Hail" + ) + findings.append(finding) + + return findings + +def convert_to_findings_file(input_file: Path) -> Path: + """ + Convert a CIS CSV file to a findings JSON file. + + Args: + input_file: Path to the input CSV file + + Returns: + Path to the output JSON file + """ + # Generate findings + findings = convert_csv_to_findings(input_file) + + # Convert findings to dictionaries + findings_data = [] + for finding in findings: + finding_dict = vars(finding) + # Convert datetime objects to strings + for key, value in finding_dict.items(): + if isinstance(value, datetime): + finding_dict[key] = value.strftime("%Y-%m-%d") + findings_data.append(finding_dict) + + # Write to JSON file + output_file = input_file.with_suffix('.findings.json') + with open(output_file, 'w') as f: + json.dump(findings_data, f, indent=2) + + return output_file \ No newline at end of file diff --git a/tools/cis/splitter.py b/tools/cis/splitter.py new file mode 100644 index 0000000..798bfad --- /dev/null +++ b/tools/cis/splitter.py @@ -0,0 +1,64 @@ +""" +Module for handling CIS scan reports. +""" +from pathlib import Path +import pandas as pd +from datetime import datetime +import os + +def split_connected_sheet(input_file: Path) -> list[Path]: + """ + Split a CIS connected sheet Excel file into multiple CSV files by date. + + Args: + input_file: Path to the input Excel file + + Returns: + List of paths to the generated CSV files + + Notes: + - Creates files in a "Divided CIS Scans" subdirectory + - Names files as " - YYYY-MM-DD.csv" + - Preserves original row order + - Skips writing if file for a date already exists + """ + # Read the Excel file + df = pd.read_excel(input_file) + + # Output directory is input directory with a "Divided CIS Scans" subdirectory + output_dir = input_file.parent / "Divided CIS Scans" + + # Ensure output directory exists + output_dir.mkdir(exist_ok=True) + + # Get base filename without "(Connected Sheet)" suffix + base_name = input_file.stem + if base_name.endswith("(Connected Sheet)"): + base_name = base_name[:-len("(Connected Sheet)")].strip() + + # Group by date and write separate files + output_files = [] + for date, group in df.groupby("Date"): + # Parse date and format filename + try: + # Try to parse date if it's not already a datetime + if not isinstance(date, datetime): + date = pd.to_datetime(date) + date_str = date.strftime("%Y-%m-%d") + except: + # If date parsing fails, use the raw value + date_str = str(date) + + # Generate output path + output_file = output_dir / f"{base_name} - {date_str}.csv" + + # Skip if file already exists + if not output_file.exists(): + # Sort by original index to preserve row order + group = group.sort_index() + + # Write to CSV + group.to_csv(output_file, index=False) + output_files.append(output_file) + + return output_files \ No newline at end of file From 0718f59915ea378a64f969a34f0e3f987cc06bd3 Mon Sep 17 00:00:00 2001 From: Chris Llanwarne Date: Tue, 20 May 2025 14:39:15 -0400 Subject: [PATCH 10/33] CIS diff --- cli/cli.py | 39 ++++- tools/cis/diff.py | 66 ++++++++ tools/cis/poam_generator.py | 144 ++++++++++++++++ tools/diff.py | 319 ++++++++++++++++++++++++++++++++++++ tools/trivy/diff.py | 7 +- tools/zap/diff.py | 71 ++------ 6 files changed, 582 insertions(+), 64 deletions(-) create mode 100644 tools/cis/diff.py create mode 100644 tools/cis/poam_generator.py create mode 100644 tools/diff.py diff --git a/cli/cli.py b/cli/cli.py index 458267c..17bbde1 100755 --- a/cli/cli.py +++ b/cli/cli.py @@ -23,6 +23,7 @@ from tools.zap.diff import compare_findings_to_zap_poams from tools.cis.splitter import split_connected_sheet from tools.cis.converter import convert_to_findings_file +from tools.cis.diff import compare_findings_to_cis_poams @click.group() def cli(): @@ -46,7 +47,7 @@ def zap(): @cli.group() def cis(): - """Commands for working with CIS scan reports.""" + """Commands for working with CIS findings.""" pass @poams.command('preview-trivy') @@ -275,5 +276,41 @@ def csv_to_findings_cmd(csv_file: Path) -> None: click.echo(f"Error converting CSV to findings: {str(e)}", err=True) sys.exit(1) +@cis.command('alerts-diff') +@click.argument('findings_file', type=click.Path(exists=True)) +@click.argument('poam_file', type=click.Path(exists=True)) +@click.option('--json-output', type=click.Path(), help='Path to save JSON output') +def alerts_diff(findings_file: str, poam_file: str, json_output: Optional[str]) -> None: + """ + Compare CIS findings against existing configuration findings. + + FINDINGS_FILE: JSON file containing CIS findings + POAM_FILE: Excel file containing configuration findings + """ + try: + # Load findings from JSON file + with open(findings_file) as f: + findings_data = json.load(f) + findings = [Finding.from_dict(f) for f in findings_data] + + # Compare findings to configuration findings + diff = compare_findings_to_cis_poams(findings, poam_file) + + # Print human readable summary + diff.print_summary() + + # Save JSON output if requested + if not json_output: + # Finding file as a path: + findings_path = Path(findings_file) + json_output = findings_path.with_suffix('.diff.json') + json_data = diff.to_json() + with open(json_output, 'w') as f: + json.dump(json_data, f, indent=2) + click.echo(f"JSON output saved to: {json_output}") + except Exception as e: + click.echo(f"Error comparing findings: {str(e)}", err=True) + sys.exit(1) + if __name__ == '__main__': cli() \ No newline at end of file diff --git a/tools/cis/diff.py b/tools/cis/diff.py new file mode 100644 index 0000000..3f94f3e --- /dev/null +++ b/tools/cis/diff.py @@ -0,0 +1,66 @@ +""" +Module for comparing CIS findings against existing configuration findings. +""" +from pathlib import Path +from typing import List, Tuple +import pandas as pd +import re + +from ..findings import Finding +from ..poam import PoamFile, PoamEntry +from ..diff import PoamFileDiff, compare_findings_to_poams +from .poam_generator import generate_poams_from_findings + +def get_cis_configuration_findings(poam_file: PoamFile) -> List[PoamEntry]: + """ + Get CIS configuration findings from a POAM file. + + Args: + poam_file: PoamFile instance + + Returns: + List of configuration findings from the Configuration Findings sheet + """ + # Pattern matches YYYY-CISXXXX where XXXX is 4 or more digits + cis_pattern = r'^\d{4}-CIS\d{4,}$' + + # Get configuration findings from the Configuration Findings sheet + config_df = poam_file.workbook.parse( + sheet_name="Configuration Findings", + header=4 # 0-based index for row 5 + ) + config_findings = [ + PoamEntry.from_dict(row) + for _, row in config_df.iterrows() + if pd.notna(row.get('POAM ID')) and re.match(cis_pattern, str(row['POAM ID'])) + ] + + return config_findings + +def compare_findings_to_cis_poams(findings: List[Finding], poam_file: Path) -> PoamFileDiff: + """ + Compare a list of findings against CIS configuration findings. + + Args: + findings: List of current findings from CIS + poam_file: Path to Excel file containing CIS configuration findings + + Returns: + PoamFileDiff containing new, existing, and closed configuration findings + """ + # Load CIS configuration findings + poam_file_handler = PoamFile(poam_file) + config_findings = get_cis_configuration_findings(poam_file_handler) + + # Get all POAM IDs for generating new ones + all_poam_ids = [p.poam_id for p in config_findings] + + # Compare findings using shared function, with empty closed_poams list and store as configuration findings + return compare_findings_to_poams( + findings=findings, + open_poams=config_findings, + closed_poams=[], # No closed POAMs for CIS + existing_poam_ids=all_poam_ids, + poam_generator=generate_poams_from_findings, + store_as_configuration_findings=True + ) diff --git a/tools/cis/poam_generator.py b/tools/cis/poam_generator.py new file mode 100644 index 0000000..562a06e --- /dev/null +++ b/tools/cis/poam_generator.py @@ -0,0 +1,144 @@ +""" +Module for generating POAMs from CIS findings. +""" +from datetime import datetime, timedelta +from typing import List, Tuple, Dict +from collections import defaultdict + +from ..findings import Finding +from ..poam import PoamEntry + +def _get_next_poam_id(existing_poam_ids: List[str], current_year: int = None) -> str: + """ + Generate the next available POAM ID for CIS findings. + + Args: + existing_poam_ids: List of existing POAM IDs + current_year: Year to use for POAM ID (defaults to current year) + + Returns: + Next available POAM ID in format YYYY-CISXXXX + """ + if current_year is None: + current_year = datetime.now().year + + # Filter to just this year's CIS POAMs + year_prefix = f"{current_year}-CIS" + year_poams = [p for p in existing_poam_ids if p.startswith(year_prefix)] + + if not year_poams: + # First POAM for this year + return f"{year_prefix}0001" + + # Get highest number used + highest = max(int(p[-4:]) for p in year_poams) + + # Return next number + return f"{year_prefix}{highest + 1:04d}" + +def _group_findings_by_weakness(findings: List[Finding]) -> Dict[str, List[Finding]]: + """ + Group findings by weakness name and asset identifier. + + Args: + findings: List of findings to group + + Returns: + Dictionary mapping (weakness_name, asset_identifier) to list of findings + """ + groups = defaultdict(list) + for finding in findings: + key = (finding.weakness_name, finding.asset_identifier) + groups[key].append(finding) + return dict(groups) + +def _get_completion_date(risk_rating: str) -> datetime: + """ + Calculate completion date based on risk rating. + + Args: + risk_rating: Risk rating of the finding + + Returns: + Datetime object for completion date + """ + today = datetime.now() + + if risk_rating.lower() == "critical": + return today + timedelta(days=15) + elif risk_rating.lower() == "high": + return today + timedelta(days=30) + elif risk_rating.lower() == "moderate": + return today + timedelta(days=90) + else: # Low + return today + timedelta(days=180) + +def generate_poams_from_findings(findings: List[Finding], existing_poam_ids: List[str], current_year: int = None) -> List[Tuple[List[Finding], PoamEntry]]: + """ + Generate POAMs from CIS findings. + + Args: + findings: List of findings to generate POAMs for + existing_poam_ids: List of existing POAM IDs + current_year: Year to use for POAM IDs (defaults to current year) + + Returns: + List of tuples containing (findings, generated_poam) + """ + result = [] + + # Group findings by weakness name and asset + grouped_findings = _group_findings_by_weakness(findings) + + for (weakness_name, asset_id), group in grouped_findings.items(): + # Get earliest detection date from group + detection_date = min(f.original_detection_date for f in group) + + # Get highest risk rating from group + risk_rating = max(f.original_risk_rating for f in group) + + # Generate POAM ID + poam_id = _get_next_poam_id(existing_poam_ids, current_year) + existing_poam_ids.append(poam_id) # Add to list so next ID will be different + + # Get completion date based on risk + completion_date = _get_completion_date(risk_rating) + + # Create POAM entry + poam = PoamEntry( + poam_id=poam_id, + controls="", # CIS findings don't map to specific controls + weakness_name=weakness_name, + weakness_description=f"CIS configuration finding: {weakness_name}", + weakness_detector_source="CIS", + weakness_source_identifier="", # No specific identifier for CIS findings + asset_identifier=asset_id, + point_of_contact="Security Team", + resources_required="Security Team time", + overall_remediation_plan=f"Remediate {weakness_name} configuration finding", + original_detection_date=detection_date, + scheduled_completion_date=completion_date, + planned_milestones="", + milestone_changes="", + status_date=datetime.now(), + vendor_dependency="No", + last_vendor_check_in_date=None, + vendor_dependent_product_name="", + original_risk_rating=risk_rating, + adjusted_risk_rating="", + risk_adjustment="", + false_positive="No", + operational_requirement="No", + deviation_rationale="", + supporting_documents="", + comments=", ".join(f.finding_id for f in group), # Store finding IDs in comments + auto_approve="No", + binding_operational_directive_22_01_tracking="No", + binding_operational_directive_22_01_due_date=None, + cve="", # CIS findings don't have CVEs + service_name="" # CIS findings don't have service names + ) + + result.append((group, poam)) + + return result \ No newline at end of file diff --git a/tools/diff.py b/tools/diff.py new file mode 100644 index 0000000..64c48c2 --- /dev/null +++ b/tools/diff.py @@ -0,0 +1,319 @@ +""" +Module for comparing findings against existing POAMs. +""" +from dataclasses import dataclass +from typing import List, Optional, Tuple, Dict, Any +from pathlib import Path +from datetime import datetime + +from .findings import Finding +from .poam import PoamFile, PoamEntry + +@dataclass +class FindingPoamMatch: + """Represents a match between a finding and an existing POAM.""" + finding: Finding + poam: PoamEntry + +@dataclass +class PoamFileDiff: + """Represents the difference between current findings and existing POAMs.""" + new_findings: List[Finding] # Findings without corresponding POAMs + existing_matches: List[FindingPoamMatch] # Findings matched to existing POAMs + closed_poams: List[PoamEntry] # POAMs without corresponding findings + reopened_findings: List[FindingPoamMatch] # Findings that match previously closed POAMs + proposed_poams: List[Tuple[List[Finding], PoamEntry]] # Proposed new POAMs with their findings + proposed_configuration_findings: List[Tuple[List[Finding], PoamEntry]] # Proposed new configuration findings + closed_configuration_findings: List[PoamEntry] # Configuration findings without matches + + def to_json(self) -> Dict[str, Any]: + """ + Convert the diff results to a JSON-serializable dictionary. + + Returns: + Dictionary containing the diff results in a structured format + """ + def format_datetime(dt: datetime | str) -> str: + if isinstance(dt, str): + return dt + else: + return dt.strftime("%Y-%m-%d") if dt else None + + def poam_to_full_dict(poam: PoamEntry) -> Dict[str, Any]: + """Convert a POAM to a complete dictionary with all fields.""" + return { + "poam_id": poam.poam_id, + "controls": poam.controls, + "weakness_name": poam.weakness_name, + "weakness_description": poam.weakness_description, + "weakness_detector_source": poam.weakness_detector_source, + "weakness_source_identifier": poam.weakness_source_identifier, + "asset_identifier": poam.asset_identifier, + "point_of_contact": poam.point_of_contact, + "resources_required": poam.resources_required, + "overall_remediation_plan": poam.overall_remediation_plan, + "original_detection_date": format_datetime(poam.original_detection_date), + "scheduled_completion_date": format_datetime(poam.scheduled_completion_date), + "planned_milestones": poam.planned_milestones, + "milestone_changes": poam.milestone_changes, + "status_date": format_datetime(poam.status_date), + "vendor_dependency": poam.vendor_dependency, + "last_vendor_check_in_date": format_datetime(poam.last_vendor_check_in_date), + "vendor_dependent_product_name": poam.vendor_dependent_product_name, + "original_risk_rating": poam.original_risk_rating, + "adjusted_risk_rating": poam.adjusted_risk_rating, + "risk_adjustment": poam.risk_adjustment, + "false_positive": poam.false_positive, + "operational_requirement": poam.operational_requirement, + "deviation_rationale": poam.deviation_rationale, + "supporting_documents": poam.supporting_documents, + "comments": poam.comments, + "auto_approve": poam.auto_approve, + "binding_operational_directive_22_01_tracking": poam.binding_operational_directive_22_01_tracking, + "binding_operational_directive_22_01_due_date": format_datetime(poam.binding_operational_directive_22_01_due_date), + "cve": poam.cve, + "service_name": poam.service_name + } + + def finding_to_dict(finding: Finding) -> Dict[str, Any]: + return { + "finding_id": finding.finding_id, + "weakness_name": finding.weakness_name, + "asset_identifier": finding.asset_identifier, + "original_detection_date": format_datetime(finding.original_detection_date), + "original_risk_rating": finding.original_risk_rating, + "cve": finding.cve, + "service_name": finding.service_name + } + + result = { + "metadata": { + "new_findings_count": len(self.new_findings), + "existing_matches_count": len(self.existing_matches), + "closed_poams_count": len(self.closed_poams), + "reopened_findings_count": len(self.reopened_findings), + "proposed_poams_count": len(self.proposed_poams) + }, + "new_poams": [ + { + "poam": poam_to_full_dict(poam), + "findings": [finding_to_dict(f) for f in findings], + "finding_ids": [f.finding_id for f in findings] + } + for findings, poam in self.proposed_poams + ], + "reopen_poams": [ + { + "poam_id": match.poam.poam_id, + "finding_id": match.finding.finding_id + } + for match in self.reopened_findings + ], + "close_poams": [poam.poam_id for poam in self.closed_poams] + } + + # Add configuration findings if present + if self.proposed_configuration_findings is not None: + result["metadata"]["proposed_configuration_findings_count"] = len(self.proposed_configuration_findings) + result["proposed_configuration_findings"] = [ + { + "poam": poam_to_full_dict(poam), + "findings": [finding_to_dict(f) for f in findings], + "finding_ids": [f.finding_id for f in findings] + } + for findings, poam in self.proposed_configuration_findings + ] + + if self.closed_configuration_findings is not None: + result["metadata"]["closed_configuration_findings_count"] = len(self.closed_configuration_findings) + result["closed_configuration_findings"] = [poam.poam_id for poam in self.closed_configuration_findings] + + return result + + def print_summary(self, max_preview: int = 10) -> None: + """Print a human-readable summary of the diff.""" + # Print new findings + print("\n=== New Findings ===") + print(f"Count: {len(self.new_findings)}") + if self.new_findings: + finding_ids = [finding.finding_id for finding in self.new_findings] + print(f"Finding IDs: {', '.join(finding_ids)}") + + # Print existing matches + print("\n=== Existing Matches ===") + print(f"Count: {len(self.existing_matches)}") + if self.existing_matches: + matches = [f"{match.finding.finding_id} -> {match.poam.poam_id}" + for match in self.existing_matches[:max_preview]] + print(f"Preview of matches: {', '.join(matches)}") + if len(self.existing_matches) > max_preview: + print(f"... and {len(self.existing_matches) - max_preview} more") + + # Print reopened findings + print("\n=== Reopened Findings ===") + print(f"Count: {len(self.reopened_findings)}") + if self.reopened_findings: + matches = [f"{match.finding.finding_id} -> {match.poam.poam_id}" + for match in self.reopened_findings[:max_preview]] + print(f"Preview of matches: {', '.join(matches)}") + if len(self.reopened_findings) > max_preview: + print(f"... and {len(self.reopened_findings) - max_preview} more") + + # Print closed POAMs + print("\n=== Closed POAMs ===") + print(f"Count: {len(self.closed_poams)}") + if self.closed_poams: + poam_ids = [poam.poam_id for poam in self.closed_poams] + print(f"POAM IDs no longer active: {', '.join(poam_ids)}") + + # Print proposed POAMs + print("\n=== Proposed POAMs ===") + print(f"Count: {len(self.proposed_poams)}") + if self.proposed_poams: + for findings, poam in self.proposed_poams[:max_preview]: + finding_ids = [f.finding_id for f in findings] + print(f"{', '.join(finding_ids)} => {poam.poam_id}") + if len(self.proposed_poams) > max_preview: + print(f"... and {len(self.proposed_poams) - max_preview} more") + + # Show sample of first proposed POAM + print("\nSample new POAM:") + sample_findings, sample_poam = self.proposed_poams[0] + print(f"POAM ID: {sample_poam.poam_id}") + print(f"Weakness Name: {sample_poam.weakness_name}") + print(f"Asset Identifiers: {sample_poam.asset_identifier}") + print(f"Finding IDs: {sample_poam.comments}") + # Handle case where date might already be a string + detection_date = sample_poam.original_detection_date + if isinstance(detection_date, datetime): + detection_date = detection_date.strftime('%Y-%m-%d') + print(f"Detection Date: {detection_date}") + print(f"Risk Rating: {sample_poam.original_risk_rating}") + if sample_poam.cve: + print(f"CVE: {sample_poam.cve}") + + # Print configuration findings if present + print("\n=== Proposed Configuration Findings ===") + print(f"Count: {len(self.proposed_configuration_findings)}") + if self.proposed_configuration_findings: + for findings, poam in self.proposed_configuration_findings[:max_preview]: + finding_ids = [f.finding_id for f in findings] + print(f"{', '.join(finding_ids)} => {poam.poam_id}") + if len(self.proposed_configuration_findings) > max_preview: + print(f"... and {len(self.proposed_configuration_findings) - max_preview} more") + + # Show sample of first proposed configuration finding + print("\nSample new Configuration Finding:") + sample_findings, sample_poam = self.proposed_configuration_findings[0] + print(f"POAM ID: {sample_poam.poam_id}") + print(f"Weakness Name: {sample_poam.weakness_name}") + print(f"Asset Identifiers: {sample_poam.asset_identifier}") + print(f"Finding IDs: {sample_poam.comments}") + detection_date = sample_poam.original_detection_date + if isinstance(detection_date, datetime): + detection_date = detection_date.strftime('%Y-%m-%d') + print(f"Detection Date: {detection_date}") + print(f"Risk Rating: {sample_poam.original_risk_rating}") + if sample_poam.cve: + print(f"CVE: {sample_poam.cve}") + + print("\n=== Closed Configuration Findings ===") + print(f"Count: {len(self.closed_configuration_findings)}") + if self.closed_configuration_findings: + poam_ids = [poam.poam_id for poam in self.closed_configuration_findings] + print(f"Configuration Finding IDs no longer active: {', '.join(poam_ids)}") + +def _is_exact_match(str1: str, str2: str) -> bool: + """Check if two strings match exactly, ignoring case.""" + if not str1 or not str2: + return False + return str1.lower().strip() == str2.lower().strip() + +def _is_asset_covered(finding_asset: str, poam_assets: str) -> bool: + """ + Check if the finding's asset is included in the POAM's asset list. + + Args: + finding_asset: Asset identifier from the finding + poam_assets: Asset identifier field from the POAM (may contain multiple assets) + + Returns: + True if the finding's asset is contained within the POAM's asset list + """ + if not finding_asset or not poam_assets: + return False + return finding_asset.lower().strip() in poam_assets.lower().strip() + +def _find_matching_poam(finding: Finding, poams: List[PoamEntry]) -> Optional[FindingPoamMatch]: + """ + Find a matching POAM for a given finding. + + Args: + finding: Finding to match + poams: List of POAMs to search + + Returns: + FindingPoamMatch if a match is found, None otherwise + """ + for poam in poams: + # Match based on exact weakness name match and asset coverage + if _is_exact_match(finding.weakness_name, poam.weakness_name) and \ + _is_asset_covered(finding.asset_identifier, poam.asset_identifier): + return FindingPoamMatch(finding=finding, poam=poam) + return None + +def compare_findings_to_poams(findings: List[Finding], + open_poams: List[PoamEntry], + closed_poams: List[PoamEntry], + existing_poam_ids: List[str], + poam_generator, + store_as_configuration_findings: bool = False) -> PoamFileDiff: + """ + Compare a list of findings against existing POAMs. + + Args: + findings: List of current findings + open_poams: List of open POAMs + closed_poams: List of closed POAMs + existing_poam_ids: List of all existing POAM IDs + poam_generator: Function to generate POAMs from findings + store_as_configuration_findings: Whether to store results in configuration findings fields + + Returns: + PoamFileDiff containing new, existing, closed, and reopened findings + """ + # Track which POAMs are matched + matched_poams = set() + new_findings = [] + existing_matches = [] + reopened_findings = [] + + # First check for matches against open POAMs + for finding in findings: + match = _find_matching_poam(finding, open_poams) + if match: + existing_matches.append(match) + matched_poams.add(match.poam) + else: + # If no match in open POAMs, check closed POAMs + closed_match = _find_matching_poam(finding, closed_poams) + if closed_match: + reopened_findings.append(closed_match) + else: + new_findings.append(finding) + + # Find closed POAMs (those without matches) + closed_poams = [poam for poam in open_poams if poam not in matched_poams] + + # Generate proposed POAMs for new findings + proposed_poams = poam_generator(new_findings, existing_poam_ids) + + return PoamFileDiff( + new_findings=new_findings, + existing_matches=existing_matches, + closed_poams=closed_poams if not store_as_configuration_findings else [], + reopened_findings=reopened_findings, + proposed_poams=proposed_poams if not store_as_configuration_findings else [], + proposed_configuration_findings=proposed_poams if store_as_configuration_findings else [], + closed_configuration_findings=closed_poams if store_as_configuration_findings else [] + ) diff --git a/tools/trivy/diff.py b/tools/trivy/diff.py index c397557..3a23b3b 100644 --- a/tools/trivy/diff.py +++ b/tools/trivy/diff.py @@ -8,6 +8,7 @@ from ..findings import Finding from ..poam import PoamFile, PoamEntry +from ..diff import PoamFileDiff, compare_findings_to_poams from .poam_generator import generate_poams_from_findings @dataclass @@ -207,7 +208,7 @@ def _find_matching_poam(finding: Finding, poams: List[PoamEntry]) -> Optional[Fi return None -def compare_findings_to_trivy_poams(findings: List[Finding], poam_file: Path) -> TrivyAlertsDiff: +def compare_findings_to_trivy_poams(findings: List[Finding], poam_file: Path) -> PoamFileDiff: """ Compare a list of findings against Trivy POAMs. @@ -216,7 +217,7 @@ def compare_findings_to_trivy_poams(findings: List[Finding], poam_file: Path) -> poam_file: Path to Excel file containing Trivy POAMs Returns: - TrivyAlertsDiff containing new, existing, closed, and reopened findings + PoamFileDiff containing new, existing, closed, and reopened findings """ # Load Trivy POAMs poam_file_handler = PoamFile(poam_file) @@ -225,7 +226,7 @@ def compare_findings_to_trivy_poams(findings: List[Finding], poam_file: Path) -> # Get all POAM IDs for generating new ones all_poam_ids = [p.poam_id for p in [*open_poams, *closed_poams]] - return compare_findings_to_poams(findings, open_poams, closed_poams, all_poam_ids) + return compare_findings_to_poams(findings, open_poams, closed_poams, all_poam_ids, generate_poams_from_findings, store_as_configuration_findings=False) def compare_findings_to_poams(findings: List[Finding], diff --git a/tools/zap/diff.py b/tools/zap/diff.py index dcdafcc..e3e1cda 100644 --- a/tools/zap/diff.py +++ b/tools/zap/diff.py @@ -8,6 +8,7 @@ from ..findings import Finding from ..poam import PoamFile, PoamEntry +from ..diff import PoamFileDiff, compare_findings_to_poams from .poam_generator import generate_poams_from_findings @dataclass @@ -204,26 +205,6 @@ def _find_matching_poam(finding: Finding, poams: List[PoamEntry]) -> Optional[Fi return None -def compare_findings_to_zap_poams(findings: List[Finding], poam_file: Path) -> ZapAlertsDiff: - """ - Compare a list of findings against ZAP POAMs. - - Args: - findings: List of current findings from ZAP - poam_file: Path to Excel file containing ZAP POAMs - - Returns: - ZapAlertsDiff containing new, existing, closed, and reopened findings - """ - # Load ZAP POAMs - poam_file_handler = PoamFile(poam_file) - open_poams, closed_poams = get_zap_poam_entries(poam_file_handler) - - # Get all POAM IDs for generating new ones - all_poam_ids = [p.poam_id for p in [*open_poams, *closed_poams]] - - return compare_findings_to_poams(findings, open_poams, closed_poams, all_poam_ids) - def get_zap_poam_entries(poam_file: PoamFile) -> Tuple[List[PoamEntry], List[PoamEntry]]: """ Get ZAP POAMs from a POAM file. @@ -249,52 +230,22 @@ def get_zap_poam_entries(poam_file: PoamFile) -> Tuple[List[PoamEntry], List[Poa return open_poams, closed_poams -def compare_findings_to_poams(findings: List[Finding], - open_poams: List[PoamEntry], - closed_poams: List[PoamEntry], - existing_poam_ids: List[str]) -> ZapAlertsDiff: +def compare_findings_to_zap_poams(findings: List[Finding], poam_file: Path) -> PoamFileDiff: """ - Compare a list of findings against existing POAMs. + Compare a list of findings against ZAP POAMs. Args: findings: List of current findings from ZAP - open_poams: List of open POAMs - closed_poams: List of closed POAMs - existing_poam_ids: List of all existing POAM IDs + poam_file: Path to Excel file containing ZAP POAMs Returns: - ZapAlertsDiff containing new, existing, closed, and reopened findings + PoamFileDiff containing new, existing, closed, and reopened findings """ - # Track which POAMs are matched - matched_poams = set() - new_findings = [] - existing_matches = [] - reopened_findings = [] - - # First check for matches against open POAMs - for finding in findings: - match = _find_matching_poam(finding, open_poams) - if match: - existing_matches.append(match) - matched_poams.add(match.poam) - else: - # If no match in open POAMs, check closed POAMs - closed_match = _find_matching_poam(finding, closed_poams) - if closed_match: - reopened_findings.append(closed_match) - else: - new_findings.append(finding) - - # Find closed POAMs (those without matches) - closed_poams = [poam for poam in open_poams if poam not in matched_poams] + # Load ZAP POAMs + poam_file_handler = PoamFile(poam_file) + open_poams, closed_poams = get_zap_poam_entries(poam_file_handler) - # Generate proposed POAMs for new findings - proposed_poams = generate_poams_from_findings(new_findings, existing_poam_ids) + # Get all POAM IDs for generating new ones + all_poam_ids = [p.poam_id for p in [*open_poams, *closed_poams]] - return ZapAlertsDiff( - new_findings=new_findings, - existing_matches=existing_matches, - closed_poams=closed_poams, - reopened_findings=reopened_findings, - proposed_poams=proposed_poams - ) \ No newline at end of file + return compare_findings_to_poams(findings, open_poams, closed_poams, all_poam_ids, generate_poams_from_findings, store_as_configuration_findings=False) From 524d12302a6c4700cc1faf336e1590ab84a77317 Mon Sep 17 00:00:00 2001 From: Chris Llanwarne Date: Tue, 20 May 2025 14:49:38 -0400 Subject: [PATCH 11/33] updated single diff apply --- cli/cli.py | 2 +- tools/{trivy => }/diff_apply.py | 60 ++++++++++++++++++++++++++------- 2 files changed, 49 insertions(+), 13 deletions(-) rename tools/{trivy => }/diff_apply.py (68%) diff --git a/cli/cli.py b/cli/cli.py index 17bbde1..394bc6b 100755 --- a/cli/cli.py +++ b/cli/cli.py @@ -18,7 +18,7 @@ from tools.trivy.alerts import convert_alerts_to_poam from tools.trivy.importer import import_alerts_from_csv from tools.trivy.diff import compare_findings_to_trivy_poams -from tools.trivy.diff_apply import apply_diff_from_files +from tools.diff_apply import apply_diff_from_files from tools.zap import convert_alerts_to_findings from tools.zap.diff import compare_findings_to_zap_poams from tools.cis.splitter import split_connected_sheet diff --git a/tools/trivy/diff_apply.py b/tools/diff_apply.py similarity index 68% rename from tools/trivy/diff_apply.py rename to tools/diff_apply.py index 0bf3950..6209bd1 100644 --- a/tools/trivy/diff_apply.py +++ b/tools/diff_apply.py @@ -1,5 +1,5 @@ """ -Module for applying Trivy diff changes to POAM Excel files. +Module for applying diff changes to POAM Excel files. """ from pathlib import Path import json @@ -60,12 +60,12 @@ def apply_diff(poam_file: Path, diff_json: Dict[str, Any]) -> None: poam_file: Path to the POAM Excel file diff_json: Dictionary containing diff changes """ - # Create backup copy - backup_file = create_updateable_copy(poam_file) + # Create editable copy + editable_copy = create_updateable_copy(poam_file) try: - # Load workbook from backup copy - wb = openpyxl.load_workbook(backup_file) + # Load workbook from editable copy + wb = openpyxl.load_workbook(editable_copy) # Get sheets if "Open POA&M Items" not in wb.sheetnames: @@ -74,13 +74,19 @@ def apply_diff(poam_file: Path, diff_json: Dict[str, Any]) -> None: # Get or create closed sheet if "Closed POA&M Items" not in wb.sheetnames: - raise ValueError('Excel file must contain "Open POA&M Items" sheet') - else: - closed_sheet = wb["Closed POA&M Items"] + raise ValueError('Excel file must contain "Closed POA&M Items" sheet') + closed_sheet = wb["Closed POA&M Items"] + + # Get or validate Configuration Findings sheet + if "Configuration Findings" not in wb.sheetnames: + raise ValueError('Excel file must contain "Configuration Findings" sheet') + config_sheet = wb["Configuration Findings"] # Get column indices from header row (row 5) header_row = 5 open_headers = {cell.value: cell.column for cell in open_sheet[header_row]} + closed_headers = {cell.value: cell.column for cell in closed_sheet[header_row]} + config_headers = {cell.value: cell.column for cell in config_sheet[header_row]} # Handle new POAMs - add to open sheet if diff_json.get("new_poams"): @@ -92,6 +98,16 @@ def apply_diff(poam_file: Path, diff_json: Dict[str, Any]) -> None: if header in open_headers: open_sheet.cell(row=next_row, column=open_headers[header], value=value) + # Handle new configuration findings - add to Configuration Findings sheet + if diff_json.get("proposed_configuration_findings"): + for new_finding in diff_json["proposed_configuration_findings"]: + row_data = dict_to_row(new_finding["poam"]) + # Add row at the top (after header) + config_sheet.insert_rows(header_row + 1) + for header, value in row_data.items(): + if header in config_headers: + config_sheet.cell(row=header_row + 1, column=config_headers[header], value=value) + # Handle reopened POAMs - move from closed to open if diff_json.get("reopen_poams"): reopen_ids = {p["poam_id"] for p in diff_json["reopen_poams"]} @@ -131,13 +147,33 @@ def apply_diff(poam_file: Path, diff_json: Dict[str, Any]) -> None: # Delete moved rows from open sheet (in reverse order to maintain indices) for row in sorted(rows_to_delete, reverse=True): open_sheet.delete_rows(row) + + # Handle closed configuration findings - move to Closed POA&M Items and delete from Configuration Findings + if diff_json.get("closed_configuration_findings"): + close_ids = set(diff_json["closed_configuration_findings"]) + poam_id_col = next(col for header, col in config_headers.items() if header == "POAM ID") + + # Find and move rows + rows_to_delete = [] + for row in range(header_row + 1, config_sheet.max_row + 1): + poam_id = config_sheet.cell(row=row, column=poam_id_col).value + if poam_id in close_ids: + # Copy row to closed sheet + next_row = closed_sheet.max_row + 1 + for col in range(1, config_sheet.max_column + 1): + closed_sheet.cell(row=next_row, column=col, value=config_sheet.cell(row=row, column=col).value) + rows_to_delete.append(row) + + # Delete moved rows from config sheet (in reverse order to maintain indices) + for row in sorted(rows_to_delete, reverse=True): + config_sheet.delete_rows(row) - # Save changes to the backup file - wb.save(backup_file) + # Save changes to the editable copy + wb.save(editable_copy) except Exception as e: - # If anything goes wrong, leave the backup file for inspection - raise type(e)(f"Error applying diff changes. Backup saved as {backup_file}. Error: {str(e)}") from e + # If anything goes wrong, leave the half-edited copy for inspection + raise type(e)(f"Error applying diff changes. Incomplete edit saved as {editable_copy}. Error: {str(e)}") from e def apply_diff_from_files(poam_file: Path, diff_file: Path) -> None: """ From 8ec0c5267018fe0d78f7c9138043105b5ec643ad Mon Sep 17 00:00:00 2001 From: Chris Llanwarne Date: Tue, 20 May 2025 14:55:34 -0400 Subject: [PATCH 12/33] group CIS poams correctly --- tools/cis/poam_generator.py | 48 +++++++++++++++++++------------------ 1 file changed, 25 insertions(+), 23 deletions(-) diff --git a/tools/cis/poam_generator.py b/tools/cis/poam_generator.py index 562a06e..7ed68e2 100644 --- a/tools/cis/poam_generator.py +++ b/tools/cis/poam_generator.py @@ -36,22 +36,6 @@ def _get_next_poam_id(existing_poam_ids: List[str], current_year: int = None) -> # Return next number return f"{year_prefix}{highest + 1:04d}" -def _group_findings_by_weakness(findings: List[Finding]) -> Dict[str, List[Finding]]: - """ - Group findings by weakness name and asset identifier. - - Args: - findings: List of findings to group - - Returns: - Dictionary mapping (weakness_name, asset_identifier) to list of findings - """ - groups = defaultdict(list) - for finding in findings: - key = (finding.weakness_name, finding.asset_identifier) - groups[key].append(finding) - return dict(groups) - def _get_completion_date(risk_rating: str) -> datetime: """ Calculate completion date based on risk rating. @@ -73,6 +57,23 @@ def _get_completion_date(risk_rating: str) -> datetime: else: # Low return today + timedelta(days=180) +def _group_findings_by_weakness_and_date(findings: List[Finding]) -> Dict[tuple, List[Finding]]: + """ + Group findings by weakness name and completion date. + + Args: + findings: List of findings to group + + Returns: + Dictionary mapping (weakness_name, completion_date) to list of findings + """ + groups = defaultdict(list) + for finding in findings: + completion_date = _get_completion_date(finding.original_risk_rating) + key = (finding.weakness_name, completion_date) + groups[key].append(finding) + return dict(groups) + def generate_poams_from_findings(findings: List[Finding], existing_poam_ids: List[str], current_year: int = None) -> List[Tuple[List[Finding], PoamEntry]]: """ Generate POAMs from CIS findings. @@ -87,23 +88,24 @@ def generate_poams_from_findings(findings: List[Finding], existing_poam_ids: Lis """ result = [] - # Group findings by weakness name and asset - grouped_findings = _group_findings_by_weakness(findings) + # Group findings by weakness name and completion date + grouped_findings = _group_findings_by_weakness_and_date(findings) - for (weakness_name, asset_id), group in grouped_findings.items(): + for (weakness_name, completion_date), group in grouped_findings.items(): # Get earliest detection date from group detection_date = min(f.original_detection_date for f in group) # Get highest risk rating from group risk_rating = max(f.original_risk_rating for f in group) + # Combine asset identifiers + asset_ids = sorted(set(f.asset_identifier for f in group)) + combined_asset_id = ", ".join(asset_ids) + # Generate POAM ID poam_id = _get_next_poam_id(existing_poam_ids, current_year) existing_poam_ids.append(poam_id) # Add to list so next ID will be different - # Get completion date based on risk - completion_date = _get_completion_date(risk_rating) - # Create POAM entry poam = PoamEntry( poam_id=poam_id, @@ -112,7 +114,7 @@ def generate_poams_from_findings(findings: List[Finding], existing_poam_ids: Lis weakness_description=f"CIS configuration finding: {weakness_name}", weakness_detector_source="CIS", weakness_source_identifier="", # No specific identifier for CIS findings - asset_identifier=asset_id, + asset_identifier=combined_asset_id, point_of_contact="Security Team", resources_required="Security Team time", overall_remediation_plan=f"Remediate {weakness_name} configuration finding", From ebce0a5b5d8c7c112b202f97a9949b859ed15fa3 Mon Sep 17 00:00:00 2001 From: Chris Llanwarne Date: Tue, 20 May 2025 15:13:29 -0400 Subject: [PATCH 13/33] fix groupings --- cli/cli.py | 2 - tests/cis/__init__.py | 3 + tests/cis/test_poam_generator.py | 281 +++++++++++++++++++++++++++++++ tools/cis/poam_generator.py | 29 ++-- 4 files changed, 301 insertions(+), 14 deletions(-) create mode 100644 tests/cis/__init__.py create mode 100644 tests/cis/test_poam_generator.py diff --git a/cli/cli.py b/cli/cli.py index 394bc6b..f835570 100755 --- a/cli/cli.py +++ b/cli/cli.py @@ -5,8 +5,6 @@ import sys import os from pathlib import Path -import yaml -from datetime import datetime from typing import Optional # Add the project root to the Python path diff --git a/tests/cis/__init__.py b/tests/cis/__init__.py new file mode 100644 index 0000000..9e47485 --- /dev/null +++ b/tests/cis/__init__.py @@ -0,0 +1,3 @@ +""" +Tests for CIS-related functionality. +""" \ No newline at end of file diff --git a/tests/cis/test_poam_generator.py b/tests/cis/test_poam_generator.py new file mode 100644 index 0000000..a469e3f --- /dev/null +++ b/tests/cis/test_poam_generator.py @@ -0,0 +1,281 @@ +""" +Tests for CIS POAM generator. +""" +from datetime import datetime, timedelta, timezone +from tools.findings import Finding +from tools.cis.poam_generator import generate_poams_from_findings + +def test_findings_with_same_weakness_are_grouped(): + """Test that findings with the same weakness name and risk rating are grouped into one POAM.""" + detection_date = datetime.now(timezone.utc) + completion_date = detection_date + timedelta(days=30) + status_date = detection_date + + # Create two findings with same weakness name and risk rating + findings = [ + Finding( + finding_id="finding1", + controls="", + weakness_name="[VMS] Ensure that instances are not configured to use the default service account", + weakness_description="Instance 1 is using default service account", + weakness_detector_source="CIS", + weakness_source_identifier="", + asset_identifier="instance-1", + point_of_contact="Security Team", + resources_required="Security Team time", + overall_remediation_plan="Remediate default service account usage", + original_detection_date=detection_date, + scheduled_completion_date=completion_date, + planned_milestones="", + milestone_changes="", + status_date=status_date, + vendor_dependency="No", + last_vendor_check_in_date=None, + vendor_dependent_product_name="", + original_risk_rating="High", + adjusted_risk_rating=None, + risk_adjustment="", + false_positive="No", + operational_requirement="No", + deviation_rationale=None, + supporting_documents=None, + comments=None, + auto_approve="No", + binding_operational_directive_22_01_tracking="No", + binding_operational_directive_22_01_due_date=None, + cve=None, + service_name="" + ), + Finding( + finding_id="finding2", + controls="", + weakness_name="[VMS] Ensure that instances are not configured to use the default service account", + weakness_description="Instance 2 is using default service account", + weakness_detector_source="CIS", + weakness_source_identifier="", + asset_identifier="instance-2", + point_of_contact="Security Team", + resources_required="Security Team time", + overall_remediation_plan="Remediate default service account usage", + original_detection_date=detection_date, + scheduled_completion_date=completion_date, + planned_milestones="", + milestone_changes="", + status_date=status_date, + vendor_dependency="No", + last_vendor_check_in_date=None, + vendor_dependent_product_name="", + original_risk_rating="High", + adjusted_risk_rating=None, + risk_adjustment="", + false_positive="No", + operational_requirement="No", + deviation_rationale=None, + supporting_documents=None, + comments=None, + auto_approve="No", + binding_operational_directive_22_01_tracking="No", + binding_operational_directive_22_01_due_date=None, + cve=None, + service_name="" + ) + ] + + # Generate POAMs + result = generate_poams_from_findings(findings, existing_poam_ids=[]) + + # Verify we only got one POAM + assert len(result) == 1, "Expected findings to be grouped into one POAM" + + # Get the findings and POAM from the result + grouped_findings, poam = result[0] + + # Verify both findings are in the group + assert len(grouped_findings) == 2, "Expected both findings in the group" + assert {f.finding_id for f in grouped_findings} == {"finding1", "finding2"} + + # Verify POAM has combined asset identifiers + assert "instance-1" in poam.asset_identifier + assert "instance-2" in poam.asset_identifier + assert "," in poam.asset_identifier # Should be comma-separated + + # Verify other POAM fields + assert poam.weakness_name == "[VMS] Ensure that instances are not configured to use the default service account" + assert poam.original_risk_rating == "High" + # Due date should be 30 days from now (for High risk) + expected_due_date = datetime.now(timezone.utc) + timedelta(days=30) + assert abs((poam.scheduled_completion_date - expected_due_date).days) <= 1 # Allow 1 day difference due to timing + +def test_findings_with_different_weakness_not_grouped(): + """Test that findings with different weakness names are not grouped.""" + detection_date = datetime.now(timezone.utc) + completion_date = detection_date + timedelta(days=30) + status_date = detection_date + + # Create two findings with different weakness names + findings = [ + Finding( + finding_id="finding1", + controls="", + weakness_name="[VMS] Ensure that instances are not configured to use the default service account", + weakness_description="Instance 1 is using default service account", + weakness_detector_source="CIS", + weakness_source_identifier="", + asset_identifier="instance-1", + point_of_contact="Security Team", + resources_required="Security Team time", + overall_remediation_plan="Remediate default service account usage", + original_detection_date=detection_date, + scheduled_completion_date=completion_date, + planned_milestones="", + milestone_changes="", + status_date=status_date, + vendor_dependency="No", + last_vendor_check_in_date=None, + vendor_dependent_product_name="", + original_risk_rating="High", + adjusted_risk_rating=None, + risk_adjustment="", + false_positive="No", + operational_requirement="No", + deviation_rationale=None, + supporting_documents=None, + comments=None, + auto_approve="No", + binding_operational_directive_22_01_tracking="No", + binding_operational_directive_22_01_due_date=None, + cve=None, + service_name="" + ), + Finding( + finding_id="finding2", + controls="", + weakness_name="[VMS] Ensure that instances do not have public IP addresses", + weakness_description="Instance 2 has public IP", + weakness_detector_source="CIS", + weakness_source_identifier="", + asset_identifier="instance-2", + point_of_contact="Security Team", + resources_required="Security Team time", + overall_remediation_plan="Remove public IP addresses", + original_detection_date=detection_date, + scheduled_completion_date=completion_date, + planned_milestones="", + milestone_changes="", + status_date=status_date, + vendor_dependency="No", + last_vendor_check_in_date=None, + vendor_dependent_product_name="", + original_risk_rating="High", + adjusted_risk_rating=None, + risk_adjustment="", + false_positive="No", + operational_requirement="No", + deviation_rationale=None, + supporting_documents=None, + comments=None, + auto_approve="No", + binding_operational_directive_22_01_tracking="No", + binding_operational_directive_22_01_due_date=None, + cve=None, + service_name="" + ) + ] + + # Generate POAMs + result = generate_poams_from_findings(findings, existing_poam_ids=[]) + + # Verify we got two POAMs + assert len(result) == 2, "Expected findings to generate separate POAMs" + + # Verify each POAM has one finding + for findings_group, poam in result: + assert len(findings_group) == 1, "Expected one finding per POAM" + assert poam.asset_identifier in ["instance-1", "instance-2"] + assert "," not in poam.asset_identifier # Should not be comma-separated + +def test_findings_with_different_risk_ratings_not_grouped(): + """Test that findings with same weakness but different risk ratings are not grouped.""" + detection_date = datetime.now(timezone.utc) + completion_date = detection_date + timedelta(days=30) + status_date = detection_date + + # Create two findings with same weakness but different risk ratings + findings = [ + Finding( + finding_id="finding1", + controls="", + weakness_name="[VMS] Ensure that instances are not configured to use the default service account", + weakness_description="Instance 1 is using default service account", + weakness_detector_source="CIS", + weakness_source_identifier="", + asset_identifier="instance-1", + point_of_contact="Security Team", + resources_required="Security Team time", + overall_remediation_plan="Remediate default service account usage", + original_detection_date=detection_date, + scheduled_completion_date=completion_date, + planned_milestones="", + milestone_changes="", + status_date=status_date, + vendor_dependency="No", + last_vendor_check_in_date=None, + vendor_dependent_product_name="", + original_risk_rating="High", + adjusted_risk_rating=None, + risk_adjustment="", + false_positive="No", + operational_requirement="No", + deviation_rationale=None, + supporting_documents=None, + comments=None, + auto_approve="No", + binding_operational_directive_22_01_tracking="No", + binding_operational_directive_22_01_due_date=None, + cve=None, + service_name="" + ), + Finding( + finding_id="finding2", + controls="", + weakness_name="[VMS] Ensure that instances are not configured to use the default service account", + weakness_description="Instance 2 is using default service account", + weakness_detector_source="CIS", + weakness_source_identifier="", + asset_identifier="instance-2", + point_of_contact="Security Team", + resources_required="Security Team time", + overall_remediation_plan="Remediate default service account usage", + original_detection_date=detection_date, + scheduled_completion_date=completion_date, + planned_milestones="", + milestone_changes="", + status_date=status_date, + vendor_dependency="No", + last_vendor_check_in_date=None, + vendor_dependent_product_name="", + original_risk_rating="Low", # Different risk rating + adjusted_risk_rating=None, + risk_adjustment="", + false_positive="No", + operational_requirement="No", + deviation_rationale=None, + supporting_documents=None, + comments=None, + auto_approve="No", + binding_operational_directive_22_01_tracking="No", + binding_operational_directive_22_01_due_date=None, + cve=None, + service_name="" + ) + ] + + # Generate POAMs + result = generate_poams_from_findings(findings, existing_poam_ids=[]) + + # Verify we got two POAMs (due to different completion dates) + assert len(result) == 2, "Expected findings to generate separate POAMs due to different risk ratings" + + # Verify completion dates are different + completion_dates = {poam.scheduled_completion_date for _, poam in result} + assert len(completion_dates) == 2, "Expected different completion dates for different risk ratings" \ No newline at end of file diff --git a/tools/cis/poam_generator.py b/tools/cis/poam_generator.py index 7ed68e2..256d8a5 100644 --- a/tools/cis/poam_generator.py +++ b/tools/cis/poam_generator.py @@ -1,7 +1,7 @@ """ Module for generating POAMs from CIS findings. """ -from datetime import datetime, timedelta +from datetime import datetime, timedelta, timezone from typing import List, Tuple, Dict from collections import defaultdict @@ -20,7 +20,7 @@ def _get_next_poam_id(existing_poam_ids: List[str], current_year: int = None) -> Next available POAM ID in format YYYY-CISXXXX """ if current_year is None: - current_year = datetime.now().year + current_year = datetime.now(timezone.utc).year # Filter to just this year's CIS POAMs year_prefix = f"{current_year}-CIS" @@ -46,7 +46,7 @@ def _get_completion_date(risk_rating: str) -> datetime: Returns: Datetime object for completion date """ - today = datetime.now() + today = datetime.now(timezone.utc) if risk_rating.lower() == "critical": return today + timedelta(days=15) @@ -70,8 +70,9 @@ def _group_findings_by_weakness_and_date(findings: List[Finding]) -> Dict[tuple, groups = defaultdict(list) for finding in findings: completion_date = _get_completion_date(finding.original_risk_rating) - key = (finding.weakness_name, completion_date) - groups[key].append(finding) + # Use only the date part for grouping key, but store the full datetime with the finding + key = (finding.weakness_name, completion_date.date()) + groups[key].append((finding, completion_date)) return dict(groups) def generate_poams_from_findings(findings: List[Finding], existing_poam_ids: List[str], current_year: int = None) -> List[Tuple[List[Finding], PoamEntry]]: @@ -91,15 +92,19 @@ def generate_poams_from_findings(findings: List[Finding], existing_poam_ids: Lis # Group findings by weakness name and completion date grouped_findings = _group_findings_by_weakness_and_date(findings) - for (weakness_name, completion_date), group in grouped_findings.items(): + for (weakness_name, _), group in grouped_findings.items(): + # Unpack findings and their completion dates + findings_list = [f for f, _ in group] + completion_date = group[0][1] # Use the completion date from the first finding (they're all the same) + # Get earliest detection date from group - detection_date = min(f.original_detection_date for f in group) + detection_date = min(f.original_detection_date for f in findings_list) # Get highest risk rating from group - risk_rating = max(f.original_risk_rating for f in group) + risk_rating = max(f.original_risk_rating for f in findings_list) # Combine asset identifiers - asset_ids = sorted(set(f.asset_identifier for f in group)) + asset_ids = sorted(set(f.asset_identifier for f in findings_list)) combined_asset_id = ", ".join(asset_ids) # Generate POAM ID @@ -122,7 +127,7 @@ def generate_poams_from_findings(findings: List[Finding], existing_poam_ids: Lis scheduled_completion_date=completion_date, planned_milestones="", milestone_changes="", - status_date=datetime.now(), + status_date=datetime.now(timezone.utc), vendor_dependency="No", last_vendor_check_in_date=None, vendor_dependent_product_name="", @@ -133,7 +138,7 @@ def generate_poams_from_findings(findings: List[Finding], existing_poam_ids: Lis operational_requirement="No", deviation_rationale="", supporting_documents="", - comments=", ".join(f.finding_id for f in group), # Store finding IDs in comments + comments=", ".join(f.finding_id for f in findings_list), # Store finding IDs in comments auto_approve="No", binding_operational_directive_22_01_tracking="No", binding_operational_directive_22_01_due_date=None, @@ -141,6 +146,6 @@ def generate_poams_from_findings(findings: List[Finding], existing_poam_ids: Lis service_name="" # CIS findings don't have service names ) - result.append((group, poam)) + result.append((findings_list, poam)) return result \ No newline at end of file From 0fc52639a08c60ad637d3b02ce208e492b8c66ec Mon Sep 17 00:00:00 2001 From: Chris Llanwarne Date: Tue, 20 May 2025 17:03:25 -0400 Subject: [PATCH 14/33] fix CIS fields --- tools/cis/poam_generator.py | 53 +++++++++++++++++++------------------ 1 file changed, 27 insertions(+), 26 deletions(-) diff --git a/tools/cis/poam_generator.py b/tools/cis/poam_generator.py index 256d8a5..fce2170 100644 --- a/tools/cis/poam_generator.py +++ b/tools/cis/poam_generator.py @@ -95,7 +95,8 @@ def generate_poams_from_findings(findings: List[Finding], existing_poam_ids: Lis for (weakness_name, _), group in grouped_findings.items(): # Unpack findings and their completion dates findings_list = [f for f, _ in group] - completion_date = group[0][1] # Use the completion date from the first finding (they're all the same) + first_finding = findings_list[0] + completion_date = first_finding.scheduled_completion_date # Get earliest detection date from group detection_date = min(f.original_detection_date for f in findings_list) @@ -114,36 +115,36 @@ def generate_poams_from_findings(findings: List[Finding], existing_poam_ids: Lis # Create POAM entry poam = PoamEntry( poam_id=poam_id, - controls="", # CIS findings don't map to specific controls - weakness_name=weakness_name, - weakness_description=f"CIS configuration finding: {weakness_name}", - weakness_detector_source="CIS", - weakness_source_identifier="", # No specific identifier for CIS findings + controls=first_finding.controls, + weakness_name=first_finding.weakness_name, + weakness_description=first_finding.weakness_description, + weakness_detector_source=first_finding.weakness_detector_source, + weakness_source_identifier=first_finding.weakness_source_identifier, asset_identifier=combined_asset_id, - point_of_contact="Security Team", - resources_required="Security Team time", - overall_remediation_plan=f"Remediate {weakness_name} configuration finding", + point_of_contact=first_finding.point_of_contact, + resources_required=first_finding.resources_required, + overall_remediation_plan=first_finding.overall_remediation_plan, original_detection_date=detection_date, scheduled_completion_date=completion_date, - planned_milestones="", - milestone_changes="", + planned_milestones=first_finding.planned_milestones, + milestone_changes=first_finding.milestone_changes, status_date=datetime.now(timezone.utc), - vendor_dependency="No", - last_vendor_check_in_date=None, - vendor_dependent_product_name="", + vendor_dependency=first_finding.vendor_dependency, + last_vendor_check_in_date=first_finding.last_vendor_check_in_date, + vendor_dependent_product_name=first_finding.vendor_dependent_product_name, original_risk_rating=risk_rating, - adjusted_risk_rating="", - risk_adjustment="", - false_positive="No", - operational_requirement="No", - deviation_rationale="", - supporting_documents="", - comments=", ".join(f.finding_id for f in findings_list), # Store finding IDs in comments - auto_approve="No", - binding_operational_directive_22_01_tracking="No", - binding_operational_directive_22_01_due_date=None, - cve="", # CIS findings don't have CVEs - service_name="" # CIS findings don't have service names + adjusted_risk_rating=first_finding.adjusted_risk_rating, + risk_adjustment=first_finding.risk_adjustment, + false_positive=first_finding.false_positive, + operational_requirement=first_finding.operational_requirement, + deviation_rationale=first_finding.deviation_rationale, + supporting_documents=first_finding.supporting_documents, + comments="", # No finding IDs for CIS findings + auto_approve=first_finding.auto_approve, + binding_operational_directive_22_01_tracking=first_finding.binding_operational_directive_22_01_tracking, + binding_operational_directive_22_01_due_date=first_finding.binding_operational_directive_22_01_due_date, + cve=first_finding.cve, + service_name=first_finding.service_name ) result.append((findings_list, poam)) From 4554cd0a23e9f04a7991ae7328cb9cb7c29d7afb Mon Sep 17 00:00:00 2001 From: Chris Llanwarne Date: Tue, 20 May 2025 17:38:51 -0400 Subject: [PATCH 15/33] update zap input format --- cli/README.md | 8 +-- cli/cli.py | 10 +-- tools/zap/alerts.py | 144 +++++++++++++++++++------------------------- 3 files changed, 71 insertions(+), 91 deletions(-) diff --git a/cli/README.md b/cli/README.md index 958fb2c..b19ff90 100644 --- a/cli/README.md +++ b/cli/README.md @@ -90,14 +90,14 @@ Commands for working with Trivy alerts are grouped under the `trivy` command: Commands for working with ZAP scan reports are grouped under the `zap` command: ```bash -# Convert ZAP XML alerts to findings JSON format -./cli.py zap alerts-to-findings +# Convert ZAP CSV alerts to findings JSON format +./cli.py zap alerts-to-findings ``` This command: -- Takes a ZAP XML report file as input +- Takes a ZAP CSV report file as input - Converts each alert to a finding object with: - - Finding ID (based on ZAP plugin ID) + - Finding ID (based on ZAP alert ID) - Weakness name and description - Asset identifier (host) - Risk rating and confidence level diff --git a/cli/cli.py b/cli/cli.py index f835570..865cdaf 100755 --- a/cli/cli.py +++ b/cli/cli.py @@ -163,16 +163,16 @@ def apply_diff(poam_file: Path, diff_file: Path) -> None: sys.exit(1) @zap.command('alerts-to-findings') -@click.argument('xml_file', type=click.Path(exists=True)) -def alerts_to_findings(xml_file): - """Convert ZAP XML alerts to findings JSON format. +@click.argument('csv_file', type=click.Path(exists=True)) +def alerts_to_findings(csv_file): + """Convert ZAP CSV alerts to findings JSON format. - XML_FILE should be a ZAP XML report file. + CSV_FILE should be a ZAP CSV report file. The findings will be saved as a JSON file and the first finding will be displayed. """ try: # Convert alerts to findings - output_file = convert_alerts_to_findings(xml_file) + output_file = convert_alerts_to_findings(csv_file) # Load and display first finding with open(output_file) as f: diff --git a/tools/zap/alerts.py b/tools/zap/alerts.py index 4fbf996..9e14245 100644 --- a/tools/zap/alerts.py +++ b/tools/zap/alerts.py @@ -1,8 +1,8 @@ """ Module for handling ZAP scan reports and converting alerts to findings. """ -import xml.etree.ElementTree as ET -from datetime import datetime, timedelta +import csv +from datetime import datetime, timedelta, timezone from typing import List import json from pathlib import Path @@ -20,106 +20,86 @@ def get_completion_date(severity: str, detection_date: datetime) -> datetime: days = days_map.get(severity, 180) # Default to 180 days if unknown severity return detection_date + timedelta(days=days) -def parse_zap_xml(xml_file: str) -> List[Finding]: +def parse_zap_csv(csv_file: str) -> List[Finding]: """ - Parse a ZAP XML report and extract alert findings. + Parse a ZAP CSV report and extract alert findings. Args: - xml_file: Path to the ZAP XML report file + csv_file: Path to the ZAP CSV report file Returns: List of Finding objects """ - tree = ET.parse(xml_file) - root = tree.getroot() - findings = [] - # Extract scan date from report - scan_date = datetime.strptime(root.get('generated'), '%a, %d %b %Y %H:%M:%S') - - # Process each alert - for site in root.findall('.//site'): - for alertitem in site.findall('.//alertitem'): - # Get basic alert info - alert_name = alertitem.find('alert').text - risk_code = int(alertitem.find('riskcode').text) - description = alertitem.find('desc').text - plugin_id = alertitem.find('pluginid').text - - # Map risk code to severity - severity_map = { - 0: 'Informational', - 1: 'Low', - 2: 'Medium', - 3: 'High' - } - severity = severity_map.get(risk_code, 'Unknown') + with open(csv_file, 'r') as f: + reader = csv.DictReader(f) + for row in reader: + # Parse dates + detection_date = datetime.strptime(row['Original Detection Date'], '%m/%d/%Y') + detection_date = detection_date.replace(tzinfo=timezone.utc) - # Calculate completion date based on severity - completion_date = get_completion_date(severity, scan_date) + # Parse completion date if available + completion_date = None + if row['Scheduled Completion Date']: + try: + completion_date = datetime.strptime(row['Scheduled Completion Date'], '%Y-%m-%d %H:%M:%S') + completion_date = completion_date.replace(tzinfo=timezone.utc) + except ValueError: + # If parsing fails, calculate based on risk rating + completion_date = get_completion_date(row['Original Risk Rating'], detection_date) + else: + completion_date = get_completion_date(row['Original Risk Rating'], detection_date) - # Create a finding for each instance - for idx, instance in enumerate(alertitem.findall('.//instance')): - uri = instance.find('uri').text - evidence = instance.find('evidence').text if instance.find('evidence') is not None else None - other_info = instance.find('otherinfo').text if instance.find('otherinfo') is not None else None - - # Add evidence and other info to description if available - full_description = description - if evidence: - full_description += f"\n\nEvidence:\n{evidence}" - if other_info: - full_description += f"\n\nAdditional Information:\n{other_info}" - - finding = Finding( - finding_id=f"ZAP-{plugin_id}-{idx+1}", - controls="RA-5", - weakness_name=alert_name, - weakness_description=full_description, - weakness_detector_source="ZAP", - weakness_source_identifier=plugin_id, - asset_identifier=uri, - point_of_contact="Chris Llanwarne", - resources_required="None", - overall_remediation_plan="Perform necessary updates to resolve the vulnerability", - original_detection_date=scan_date, - scheduled_completion_date=completion_date, - planned_milestones=f"(1) {completion_date.strftime('%Y-%m-%d')}: Perform necessary updates to resolve the vulnerability", - milestone_changes="", - status_date=scan_date, - vendor_dependency="No", - last_vendor_check_in_date=None, - vendor_dependent_product_name="N/A", - original_risk_rating=severity, - adjusted_risk_rating=None, - risk_adjustment="", - false_positive="", - operational_requirement="", - deviation_rationale=None, - supporting_documents=None, - comments=None, - auto_approve="", - binding_operational_directive_22_01_tracking="", - binding_operational_directive_22_01_due_date=None, - cve=None, - service_name="Hail" - ) - findings.append(finding) + # Create finding + finding = Finding( + finding_id=f"{row['ids']}", + controls="RA-5", + weakness_name=row['Weakness Name'], + weakness_description=row['Weakness Description'], + weakness_detector_source=row['Weakness Detector Source'], + weakness_source_identifier=row['Weakness Source Identifier'], + asset_identifier=row['Asset Identifier'], + point_of_contact="Chris Llanwarne", + resources_required="None", + overall_remediation_plan="Perform necessary updates to resolve the vulnerability", + original_detection_date=detection_date, + scheduled_completion_date=completion_date, + planned_milestones=f"(1) {completion_date.strftime('%Y-%m-%d')}: Perform necessary updates to resolve the vulnerability", + milestone_changes="", + status_date=datetime.now(timezone.utc), + vendor_dependency="No", + last_vendor_check_in_date=None, + vendor_dependent_product_name="N/A", + original_risk_rating=row['Original Risk Rating'], + adjusted_risk_rating=None, + risk_adjustment="", + false_positive="", + operational_requirement="", + deviation_rationale=None, + supporting_documents=None, + comments=None, + auto_approve="", + binding_operational_directive_22_01_tracking="", + binding_operational_directive_22_01_due_date=None, + cve=None, + service_name="Hail" + ) + findings.append(finding) return findings -def convert_alerts_to_findings(xml_file: str) -> str: +def convert_alerts_to_findings(csv_file: str) -> str: """ - Convert ZAP XML alerts to findings JSON format. + Convert ZAP CSV alerts to findings JSON format. Args: - xml_file: Path to the ZAP XML report file + csv_file: Path to the ZAP CSV report file Returns: Path to the output JSON file """ - findings = parse_zap_xml(xml_file) + findings = parse_zap_csv(csv_file) # Convert findings to dictionaries findings_data = [] @@ -132,7 +112,7 @@ def convert_alerts_to_findings(xml_file: str) -> str: findings_data.append(finding_dict) # Generate output filename - input_path = Path(xml_file) + input_path = Path(csv_file) output_file = input_path.with_suffix('.findings.json') # Write findings to JSON file From 710906249354ae7849096071186baa315db926d2 Mon Sep 17 00:00:00 2001 From: Chris Llanwarne Date: Tue, 27 May 2025 15:11:19 -0400 Subject: [PATCH 16/33] progress --- cli/cli.py | 3 ++ tools/trivy/diff.py | 86 ++++++++++++++++++++++----------------------- 2 files changed, 46 insertions(+), 43 deletions(-) diff --git a/cli/cli.py b/cli/cli.py index 865cdaf..11a3666 100755 --- a/cli/cli.py +++ b/cli/cli.py @@ -4,6 +4,7 @@ import click import sys import os +import traceback from pathlib import Path from typing import Optional @@ -160,6 +161,8 @@ def apply_diff(poam_file: Path, diff_file: Path) -> None: click.echo(f"Successfully applied diff changes to {poam_file}") except Exception as e: click.echo(f"Error applying diff: {str(e)}", err=True) + click.echo("\nFull traceback:", err=True) + click.echo(traceback.format_exc(), err=True) sys.exit(1) @zap.command('alerts-to-findings') diff --git a/tools/trivy/diff.py b/tools/trivy/diff.py index 3a23b3b..c396a7a 100644 --- a/tools/trivy/diff.py +++ b/tools/trivy/diff.py @@ -229,52 +229,52 @@ def compare_findings_to_trivy_poams(findings: List[Finding], poam_file: Path) -> return compare_findings_to_poams(findings, open_poams, closed_poams, all_poam_ids, generate_poams_from_findings, store_as_configuration_findings=False) -def compare_findings_to_poams(findings: List[Finding], - open_poams: List[PoamEntry], - closed_poams: List[PoamEntry], - existing_poam_ids: List[str]) -> TrivyAlertsDiff: - """ - Compare a list of findings against existing POAMs. +# def compare_findings_to_poams(findings: List[Finding], +# open_poams: List[PoamEntry], +# closed_poams: List[PoamEntry], +# existing_poam_ids: List[str]) -> TrivyAlertsDiff: +# """ +# Compare a list of findings against existing POAMs. - Args: - findings: List of current findings from Trivy - open_poams: List of open POAMs - closed_poams: List of closed POAMs - existing_poam_ids: List of all existing POAM IDs +# Args: +# findings: List of current findings from Trivy +# open_poams: List of open POAMs +# closed_poams: List of closed POAMs +# existing_poam_ids: List of all existing POAM IDs - Returns: - TrivyAlertsDiff containing new, existing, closed, and reopened findings - """ - # Track which POAMs are matched - matched_poams = set() - new_findings = [] - existing_matches = [] - reopened_findings = [] +# Returns: +# TrivyAlertsDiff containing new, existing, closed, and reopened findings +# """ +# # Track which POAMs are matched +# matched_poams = set() +# new_findings = [] +# existing_matches = [] +# reopened_findings = [] - # First check for matches against open POAMs - for finding in findings: - match = _find_matching_poam(finding, open_poams) - if match: - existing_matches.append(match) - matched_poams.add(match.poam) - else: - # If no match in open POAMs, check closed POAMs - closed_match = _find_matching_poam(finding, closed_poams) - if closed_match: - reopened_findings.append(closed_match) - else: - new_findings.append(finding) +# # First check for matches against open POAMs +# for finding in findings: +# match = _find_matching_poam(finding, open_poams) +# if match: +# existing_matches.append(match) +# matched_poams.add(match.poam) +# else: +# # If no match in open POAMs, check closed POAMs +# closed_match = _find_matching_poam(finding, closed_poams) +# if closed_match: +# reopened_findings.append(closed_match) +# else: +# new_findings.append(finding) - # Find closed POAMs (those without matches) - closed_poams = [poam for poam in open_poams if poam not in matched_poams] +# # Find closed POAMs (those without matches) +# closed_poams = [poam for poam in open_poams if poam not in matched_poams] - # Generate proposed POAMs for new findings - proposed_poams = generate_poams_from_findings(new_findings, existing_poam_ids) +# # Generate proposed POAMs for new findings +# proposed_poams = generate_poams_from_findings(new_findings, existing_poam_ids) - return TrivyAlertsDiff( - new_findings=new_findings, - existing_matches=existing_matches, - closed_poams=closed_poams, - reopened_findings=reopened_findings, - proposed_poams=proposed_poams - ) \ No newline at end of file +# return TrivyAlertsDiff( +# new_findings=new_findings, +# existing_matches=existing_matches, +# closed_poams=closed_poams, +# reopened_findings=reopened_findings, +# proposed_poams=proposed_poams +# ) \ No newline at end of file From 34168b365aa0b78176352f72958a7d250e9b9021 Mon Sep 17 00:00:00 2001 From: Chris Llanwarne Date: Tue, 27 May 2025 17:09:39 -0400 Subject: [PATCH 17/33] remove anything after connected sheet too --- tools/cis/splitter.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tools/cis/splitter.py b/tools/cis/splitter.py index 798bfad..5429b2f 100644 --- a/tools/cis/splitter.py +++ b/tools/cis/splitter.py @@ -33,8 +33,10 @@ def split_connected_sheet(input_file: Path) -> list[Path]: # Get base filename without "(Connected Sheet)" suffix base_name = input_file.stem - if base_name.endswith("(Connected Sheet)"): - base_name = base_name[:-len("(Connected Sheet)")].strip() + + # Remove "(Connected Sheet)" (and anything after it) + if "(Connected Sheet)" in base_name: + base_name = base_name.split("(Connected Sheet)")[0].strip() # Group by date and write separate files output_files = [] From 14a4645dc3734cb2eae1f938c6d814bdc44fc064 Mon Sep 17 00:00:00 2001 From: Chris Llanwarne Date: Thu, 25 Sep 2025 11:15:26 -0400 Subject: [PATCH 18/33] exclude info findings from poam diffs --- cli/cli.py | 50 +++++++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 45 insertions(+), 5 deletions(-) diff --git a/cli/cli.py b/cli/cli.py index 11a3666..fd65112 100755 --- a/cli/cli.py +++ b/cli/cli.py @@ -118,13 +118,26 @@ def alerts_diff(poam_file: Path, alerts_csv: Path): - New findings that need POAMs created - Existing findings that already have POAMs - Closed POAMs that no longer have corresponding findings + + Note: Findings with Info severity are automatically excluded. """ try: # Import findings from CSV - findings = import_alerts_from_csv(alerts_csv) - if not findings: + all_findings = import_alerts_from_csv(alerts_csv) + if not all_findings: click.echo("No findings found in CSV file", err=True) sys.exit(1) + + # Filter out Info severity findings + findings = [f for f in all_findings if f.original_risk_rating.lower() != 'info'] + info_count = len(all_findings) - len(findings) + + if info_count > 0: + click.echo(f"Excluded {info_count} findings with Info severity") + + if not findings: + click.echo("No findings remaining after filtering out Info severity", err=True) + sys.exit(1) # Compare findings against POAMs diff = compare_findings_to_trivy_poams(findings, poam_file) @@ -196,12 +209,26 @@ def alerts_to_findings(csv_file): @click.argument('poam_file', type=click.Path(exists=True)) @click.option('--json-output', type=click.Path(), help='Path to save JSON output') def alerts_diff(findings_file: str, poam_file: str, json_output: Optional[str]) -> None: - """Compare ZAP findings against existing POAMs.""" + """Compare ZAP findings against existing POAMs. + + Note: Findings with Info severity are automatically excluded. + """ try: # Load findings from JSON file with open(findings_file) as f: findings_data = json.load(f) - findings = [Finding.from_dict(f) for f in findings_data] + all_findings = [Finding.from_dict(f) for f in findings_data] + + # Filter out Info severity findings + findings = [f for f in all_findings if f.original_risk_rating.lower() != 'info'] + info_count = len(all_findings) - len(findings) + + if info_count > 0: + click.echo(f"Excluded {info_count} findings with Info severity") + + if not findings: + click.echo("No findings remaining after filtering out Info severity", err=True) + sys.exit(1) # Compare findings to POAMs diff = compare_findings_to_zap_poams(findings, poam_file) @@ -287,12 +314,25 @@ def alerts_diff(findings_file: str, poam_file: str, json_output: Optional[str]) FINDINGS_FILE: JSON file containing CIS findings POAM_FILE: Excel file containing configuration findings + + Note: Findings with Info severity are automatically excluded. """ try: # Load findings from JSON file with open(findings_file) as f: findings_data = json.load(f) - findings = [Finding.from_dict(f) for f in findings_data] + all_findings = [Finding.from_dict(f) for f in findings_data] + + # Filter out Info severity findings + findings = [f for f in all_findings if f.original_risk_rating.lower() != 'info'] + info_count = len(all_findings) - len(findings) + + if info_count > 0: + click.echo(f"Excluded {info_count} findings with Info severity") + + if not findings: + click.echo("No findings remaining after filtering out Info severity", err=True) + sys.exit(1) # Compare findings to configuration findings diff = compare_findings_to_cis_poams(findings, poam_file) From b55ed18a9a80110338530a8fa70ce5a3f4b83cef Mon Sep 17 00:00:00 2001 From: Chris Llanwarne Date: Thu, 25 Sep 2025 11:19:14 -0400 Subject: [PATCH 19/33] make param order consistent across scan types --- cli/cli.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/cli/cli.py b/cli/cli.py index fd65112..0c02bae 100755 --- a/cli/cli.py +++ b/cli/cli.py @@ -205,10 +205,10 @@ def alerts_to_findings(csv_file): sys.exit(1) @zap.command('alerts-diff') -@click.argument('findings_file', type=click.Path(exists=True)) @click.argument('poam_file', type=click.Path(exists=True)) +@click.argument('findings_file', type=click.Path(exists=True)) @click.option('--json-output', type=click.Path(), help='Path to save JSON output') -def alerts_diff(findings_file: str, poam_file: str, json_output: Optional[str]) -> None: +def alerts_diff(poam_file: str, findings_file: str, json_output: Optional[str]) -> None: """Compare ZAP findings against existing POAMs. Note: Findings with Info severity are automatically excluded. @@ -305,10 +305,10 @@ def csv_to_findings_cmd(csv_file: Path) -> None: sys.exit(1) @cis.command('alerts-diff') -@click.argument('findings_file', type=click.Path(exists=True)) @click.argument('poam_file', type=click.Path(exists=True)) +@click.argument('findings_file', type=click.Path(exists=True)) @click.option('--json-output', type=click.Path(), help='Path to save JSON output') -def alerts_diff(findings_file: str, poam_file: str, json_output: Optional[str]) -> None: +def alerts_diff(poam_file: str, findings_file: str, json_output: Optional[str]) -> None: """ Compare CIS findings against existing configuration findings. From 11dac042049d517a582266f9c867f7068ef56ced Mon Sep 17 00:00:00 2001 From: Chris Llanwarne Date: Thu, 25 Sep 2025 11:56:56 -0400 Subject: [PATCH 20/33] merge diffs when applying --- cli/README.md | 10 ++- cli/cli.py | 63 +++++++++++++-- tests/test_diff_apply.py | 163 +++++++++++++++++++++++++++++++++++++++ tools/diff_apply.py | 50 ++++++++++-- 4 files changed, 275 insertions(+), 11 deletions(-) create mode 100644 tests/test_diff_apply.py diff --git a/cli/README.md b/cli/README.md index b19ff90..9e9d253 100644 --- a/cli/README.md +++ b/cli/README.md @@ -131,5 +131,13 @@ Each command includes error handling and will provide helpful error messages if 5. Apply diff changes to update POAMs: ```bash + # Apply single diff file ./cli.py poams apply-diff existing_poams.xlsx alerts_20240513.diff.json - ``` \ No newline at end of file + + # Apply multiple diff files + ./cli.py poams apply-diff existing_poams.xlsx alerts1.diff.json alerts2.diff.json alerts3.diff.json + ``` + + > [!NOTE] + > We can merge the diff files before applying with `./cli.py poams merge-diffs` before applying, but the apply command also + > accepts multiple diff files natively. diff --git a/cli/cli.py b/cli/cli.py index 0c02bae..e722a92 100755 --- a/cli/cli.py +++ b/cli/cli.py @@ -17,7 +17,7 @@ from tools.trivy.alerts import convert_alerts_to_poam from tools.trivy.importer import import_alerts_from_csv from tools.trivy.diff import compare_findings_to_trivy_poams -from tools.diff_apply import apply_diff_from_files +from tools.diff_apply import apply_diff_from_files, merge_diffs from tools.zap import convert_alerts_to_findings from tools.zap.diff import compare_findings_to_zap_poams from tools.cis.splitter import split_connected_sheet @@ -157,20 +157,26 @@ def alerts_diff(poam_file: Path, alerts_csv: Path): @poams.command('apply-diff') @click.argument('poam_file', type=click.Path(exists=True, path_type=Path)) -@click.argument('diff_file', type=click.Path(exists=True, path_type=Path)) -def apply_diff(poam_file: Path, diff_file: Path) -> None: +@click.argument('diff_files', nargs=-1, type=click.Path(exists=True, path_type=Path)) +def apply_diff(poam_file: Path, diff_files: tuple) -> None: """Apply diff changes to a POAM Excel file. POAM_FILE: Excel file containing POAMs - DIFF_FILE: JSON file containing diff changes + DIFF_FILES: One or more JSON files containing diff changes This command will: - Add new POAMs to the Open POA&M Items sheet - Move reopened POAMs from Closed to Open sheet - Move closed POAMs from Open to Closed sheet + + If multiple diff files are provided, they will be merged before applying. """ try: - apply_diff_from_files(poam_file, diff_file) + if not diff_files: + click.echo("Error: At least one diff file must be provided", err=True) + sys.exit(1) + + apply_diff_from_files(poam_file, list(diff_files)) click.echo(f"Successfully applied diff changes to {poam_file}") except Exception as e: click.echo(f"Error applying diff: {str(e)}", err=True) @@ -178,6 +184,53 @@ def apply_diff(poam_file: Path, diff_file: Path) -> None: click.echo(traceback.format_exc(), err=True) sys.exit(1) +@poams.command('merge-diffs') +@click.argument('diff_files', nargs=-1, type=click.Path(exists=True, path_type=Path)) +@click.option('--output', '-o', type=click.Path(path_type=Path), help='Output file path (default: merged_diff.json)') +def merge_diffs_cmd(diff_files: tuple, output: Optional[Path]) -> None: + """Merge multiple diff JSON files into a single diff file. + + DIFF_FILES: One or more JSON diff files to merge + --output: Output file path (default: merged_diff.json) + + This command combines all the changes from multiple diff files into a single + diff file that can be applied to a POAM Excel file. + """ + try: + if not diff_files: + click.echo("Error: At least one diff file must be provided", err=True) + sys.exit(1) + + if not output: + output = Path("merged_diff.json") + + merged_diff = merge_diffs(list(diff_files)) + + with open(output, 'w') as f: + json.dump(merged_diff, f, indent=2) + + click.echo(f"Successfully merged {len(diff_files)} diff files into {output}") + + # Print summary of merged content + total_new = len(merged_diff.get("new_poams", [])) + total_reopen = len(merged_diff.get("reopen_poams", [])) + total_close = len(merged_diff.get("close_poams", [])) + total_config_new = len(merged_diff.get("proposed_configuration_findings", [])) + total_config_close = len(merged_diff.get("closed_configuration_findings", [])) + + click.echo(f"Merged content:") + click.echo(f" - New POAMs: {total_new}") + click.echo(f" - Reopen POAMs: {total_reopen}") + click.echo(f" - Close POAMs: {total_close}") + click.echo(f" - New Configuration Findings: {total_config_new}") + click.echo(f" - Close Configuration Findings: {total_config_close}") + + except Exception as e: + click.echo(f"Error merging diffs: {str(e)}", err=True) + click.echo("\nFull traceback:", err=True) + click.echo(traceback.format_exc(), err=True) + sys.exit(1) + @zap.command('alerts-to-findings') @click.argument('csv_file', type=click.Path(exists=True)) def alerts_to_findings(csv_file): diff --git a/tests/test_diff_apply.py b/tests/test_diff_apply.py new file mode 100644 index 0000000..1df67c4 --- /dev/null +++ b/tests/test_diff_apply.py @@ -0,0 +1,163 @@ +""" +Tests for diff_apply module. +""" +import json +import tempfile +from pathlib import Path +import pytest +from tools.diff_apply import merge_diffs + + +def test_merge_diffs_single_file(): + """Test merging a single diff file.""" + # Create a temporary diff file + diff_data = { + "new_poams": [{"poam_id": "2025-TEST001", "weakness_name": "Test Weakness"}], + "reopen_poams": [], + "close_poams": ["2025-OLD001"], + "proposed_configuration_findings": [], + "closed_configuration_findings": [] + } + + with tempfile.NamedTemporaryFile(mode='w', suffix='.json', delete=False) as f: + json.dump(diff_data, f) + temp_file = Path(f.name) + + try: + result = merge_diffs([temp_file]) + + assert result["new_poams"] == [{"poam_id": "2025-TEST001", "weakness_name": "Test Weakness"}] + assert result["reopen_poams"] == [] + assert result["close_poams"] == ["2025-OLD001"] + assert result["proposed_configuration_findings"] == [] + assert result["closed_configuration_findings"] == [] + finally: + temp_file.unlink() + + +def test_merge_diffs_multiple_files(): + """Test merging multiple diff files.""" + # Create first diff file + diff1_data = { + "new_poams": [{"poam_id": "2025-TEST001", "weakness_name": "Test Weakness 1"}], + "reopen_poams": [{"poam_id": "2025-REOPEN001"}], + "close_poams": ["2025-OLD001"], + "proposed_configuration_findings": [], + "closed_configuration_findings": [] + } + + # Create second diff file + diff2_data = { + "new_poams": [{"poam_id": "2025-TEST002", "weakness_name": "Test Weakness 2"}], + "reopen_poams": [], + "close_poams": ["2025-OLD002"], + "proposed_configuration_findings": [{"poam_id": "2025-CIS001", "weakness_name": "Config Issue"}], + "closed_configuration_findings": [] + } + + with tempfile.NamedTemporaryFile(mode='w', suffix='.json', delete=False) as f1: + json.dump(diff1_data, f1) + temp_file1 = Path(f1.name) + + with tempfile.NamedTemporaryFile(mode='w', suffix='.json', delete=False) as f2: + json.dump(diff2_data, f2) + temp_file2 = Path(f2.name) + + try: + result = merge_diffs([temp_file1, temp_file2]) + + # Check that all items from both files are merged + assert len(result["new_poams"]) == 2 + assert {"poam_id": "2025-TEST001", "weakness_name": "Test Weakness 1"} in result["new_poams"] + assert {"poam_id": "2025-TEST002", "weakness_name": "Test Weakness 2"} in result["new_poams"] + + assert len(result["reopen_poams"]) == 1 + assert {"poam_id": "2025-REOPEN001"} in result["reopen_poams"] + + assert len(result["close_poams"]) == 2 + assert "2025-OLD001" in result["close_poams"] + assert "2025-OLD002" in result["close_poams"] + + assert len(result["proposed_configuration_findings"]) == 1 + assert {"poam_id": "2025-CIS001", "weakness_name": "Config Issue"} in result["proposed_configuration_findings"] + + assert result["closed_configuration_findings"] == [] + finally: + temp_file1.unlink() + temp_file2.unlink() + + +def test_merge_diffs_empty_lists(): + """Test merging files with empty lists.""" + diff_data = { + "new_poams": [], + "reopen_poams": [], + "close_poams": [], + "proposed_configuration_findings": [], + "closed_configuration_findings": [] + } + + with tempfile.NamedTemporaryFile(mode='w', suffix='.json', delete=False) as f: + json.dump(diff_data, f) + temp_file = Path(f.name) + + try: + result = merge_diffs([temp_file]) + + assert result["new_poams"] == [] + assert result["reopen_poams"] == [] + assert result["close_poams"] == [] + assert result["proposed_configuration_findings"] == [] + assert result["closed_configuration_findings"] == [] + finally: + temp_file.unlink() + + +def test_merge_diffs_no_files(): + """Test that merge_diffs raises error with no files.""" + with pytest.raises(ValueError, match="No diff files provided"): + merge_diffs([]) + + +def test_merge_diffs_nonexistent_file(): + """Test that merge_diffs raises error with nonexistent file.""" + with pytest.raises(ValueError, match="Diff file does not exist"): + merge_diffs([Path("nonexistent.json")]) + + +def test_merge_diffs_invalid_json(): + """Test that merge_diffs raises error with invalid JSON.""" + with tempfile.NamedTemporaryFile(mode='w', suffix='.json', delete=False) as f: + f.write("invalid json content") + temp_file = Path(f.name) + + try: + with pytest.raises(ValueError, match="Error reading diff file"): + merge_diffs([temp_file]) + finally: + temp_file.unlink() + + +def test_merge_diffs_missing_keys(): + """Test merging files with missing keys (should be handled gracefully).""" + # File with only some keys + diff_data = { + "new_poams": [{"poam_id": "2025-TEST001"}], + "close_poams": ["2025-OLD001"] + # Missing other keys + } + + with tempfile.NamedTemporaryFile(mode='w', suffix='.json', delete=False) as f: + json.dump(diff_data, f) + temp_file = Path(f.name) + + try: + result = merge_diffs([temp_file]) + + assert result["new_poams"] == [{"poam_id": "2025-TEST001"}] + assert result["close_poams"] == ["2025-OLD001"] + assert result["reopen_poams"] == [] + assert result["proposed_configuration_findings"] == [] + assert result["closed_configuration_findings"] == [] + finally: + temp_file.unlink() diff --git a/tools/diff_apply.py b/tools/diff_apply.py index 6209bd1..03d5d29 100644 --- a/tools/diff_apply.py +++ b/tools/diff_apply.py @@ -175,14 +175,54 @@ def apply_diff(poam_file: Path, diff_json: Dict[str, Any]) -> None: # If anything goes wrong, leave the half-edited copy for inspection raise type(e)(f"Error applying diff changes. Incomplete edit saved as {editable_copy}. Error: {str(e)}") from e -def apply_diff_from_files(poam_file: Path, diff_file: Path) -> None: +def merge_diffs(diff_files: List[Path]) -> Dict[str, Any]: + """ + Merge multiple diff JSON files into a single diff. + + Args: + diff_files: List of paths to diff JSON files + + Returns: + Merged diff dictionary + + Raises: + ValueError: If no diff files provided or if files are invalid + """ + if not diff_files: + raise ValueError("No diff files provided") + + merged_diff = { + "new_poams": [], + "reopen_poams": [], + "close_poams": [], + "proposed_configuration_findings": [], + "closed_configuration_findings": [] + } + + for diff_file in diff_files: + if not diff_file.exists(): + raise ValueError(f"Diff file does not exist: {diff_file}") + + try: + with open(diff_file, 'r') as f: + diff_data = json.load(f) + except (json.JSONDecodeError, IOError) as e: + raise ValueError(f"Error reading diff file {diff_file}: {e}") from e + + # Merge each section + for key in merged_diff.keys(): + if key in diff_data and isinstance(diff_data[key], list): + merged_diff[key].extend(diff_data[key]) + + return merged_diff + +def apply_diff_from_files(poam_file: Path, diff_files: List[Path]) -> None: """ Apply diff changes from a JSON file to a POAM Excel file. Args: poam_file: Path to the POAM Excel file - diff_file: Path to the JSON diff file + diff_files: List of paths to JSON diff files """ - with open(diff_file, 'r') as f: - diff_json = json.load(f) - apply_diff(poam_file, diff_json) \ No newline at end of file + merged_diff = merge_diffs(diff_files) + apply_diff(poam_file, merged_diff) \ No newline at end of file From 09031b7cbc3c16b89a8659268cb1376f5f8712da Mon Sep 17 00:00:00 2001 From: Chris Llanwarne Date: Thu, 25 Sep 2025 15:30:44 -0400 Subject: [PATCH 21/33] diff tests --- tests/test_diff.py | 73 +++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 72 insertions(+), 1 deletion(-) diff --git a/tests/test_diff.py b/tests/test_diff.py index a03a0ef..01939ec 100644 --- a/tests/test_diff.py +++ b/tests/test_diff.py @@ -10,8 +10,8 @@ _is_exact_match, _is_asset_covered, _find_matching_poam, - compare_findings_to_poams, ) +from tools.diff import compare_findings_to_poams def create_test_finding(finding_id: str, weakness_name: str, asset_identifier: str) -> Finding: """Helper to create a test Finding with minimal required fields.""" @@ -212,3 +212,74 @@ def test_compare_findings_to_poams(): # Verify closed POAMs assert {poam.poam_id for poam in diff.closed_poams} == {"POAM-002", "POAM-003"} + +def test_cis_configuration_findings_closed_when_no_matches(): + """Test that CIS configuration findings are marked as closed when no findings match them.""" + # Create empty findings list (no current findings) + findings = [] + + # Create test configuration findings (simulating your real case) + config_findings = [ + create_test_poam("2025-CIS0005-A", "Some Configuration Issue", "asset-1, asset-2, asset-3, asset-4, asset-5"), + create_test_poam("2025-CIS0006-B", "Another Configuration Issue", "asset-10, asset-11"), + ] + + # Mock poam_generator function + def mock_poam_generator(new_findings, existing_poam_ids): + return [] + + # Compare findings to POAMs with store_as_configuration_findings=True + diff = compare_findings_to_poams( + findings=findings, + open_poams=config_findings, + closed_poams=[], + existing_poam_ids=["2025-CIS0005-A", "2025-CIS0006-B"], + poam_generator=mock_poam_generator, + store_as_configuration_findings=True + ) + + # Verify that all configuration findings are marked as closed + assert len(diff.closed_configuration_findings) == 2 + assert {poam.poam_id for poam in diff.closed_configuration_findings} == {"2025-CIS0005-A", "2025-CIS0006-B"} + + # Verify no new findings or matches + assert len(diff.new_findings) == 0 + assert len(diff.existing_matches) == 0 + assert len(diff.reopened_findings) == 0 + assert len(diff.proposed_configuration_findings) == 0 + +def test_cis_configuration_findings_partial_matches(): + """Test that only unmatched CIS configuration findings are marked as closed.""" + # Create findings that match some but not all configuration findings + findings = [ + create_test_finding("CIS-001", "Some Configuration Issue", "asset-1"), # Should match first config finding + ] + + # Create test configuration findings + config_findings = [ + create_test_poam("2025-CIS0005-A", "Some Configuration Issue", "asset-1, asset-2, asset-3"), + create_test_poam("2025-CIS0006-B", "Another Configuration Issue", "asset-10, asset-11"), + ] + + # Mock poam_generator function + def mock_poam_generator(new_findings, existing_poam_ids): + return [] + + # Compare findings to POAMs with store_as_configuration_findings=True + diff = compare_findings_to_poams( + findings=findings, + open_poams=config_findings, + closed_poams=[], + existing_poam_ids=["2025-CIS0005-A", "2025-CIS0006-B"], + poam_generator=mock_poam_generator, + store_as_configuration_findings=True + ) + + # Verify that only the unmatched configuration finding is marked as closed + assert len(diff.closed_configuration_findings) == 1 + assert diff.closed_configuration_findings[0].poam_id == "2025-CIS0006-B" + + # Verify existing match + assert len(diff.existing_matches) == 1 + assert diff.existing_matches[0].poam.poam_id == "2025-CIS0005-A" + assert diff.existing_matches[0].finding.finding_id == "CIS-001" From 15ee743f55ab575c8de73d1717cc3183a0921e6a Mon Sep 17 00:00:00 2001 From: Chris Llanwarne Date: Tue, 14 Oct 2025 13:15:23 -0400 Subject: [PATCH 22/33] Use WORKING for trivy commands --- README.md | 4 ++-- cli/README.md | 8 ++++++++ cli/cli.py | 31 ++++++++++++++++++++++--------- tools/github.py | 12 ++++++++---- tools/trivy/alerts.py | 12 ++++++++---- tools/utils.py | 11 ++++++++++- 6 files changed, 58 insertions(+), 20 deletions(-) diff --git a/README.md b/README.md index bb798b3..6a8152d 100644 --- a/README.md +++ b/README.md @@ -61,10 +61,10 @@ The CLI is organized into the following command groups: 2. `trivy` - Commands for working with Trivy: ```bash # Download Trivy alerts from GitHub code scanning API - ./cli/cli.py trivy download-alerts + ./cli/cli.py trivy download-alerts [--destination ] # Convert GitHub Trivy alerts JSON to POAM CSV format - ./cli/cli.py trivy convert-alerts + ./cli/cli.py trivy convert-alerts [--output ] # Compare Trivy alerts against existing POAMs ./cli/cli.py trivy alerts-diff diff --git a/cli/README.md b/cli/README.md index 9e9d253..40b0881 100644 --- a/cli/README.md +++ b/cli/README.md @@ -111,12 +111,20 @@ Each command includes error handling and will provide helpful error messages if 1. Download alerts from GitHub: ```bash + # Download to default location (WORKING env var or pwd/working) ./cli.py trivy download-alerts + + # Download to specific file + ./cli.py trivy download-alerts --destination /path/to/alerts.json ``` 2. Convert the downloaded JSON to CSV: ```bash + # Convert with default .findings.csv extension in same directory ./cli.py trivy convert-alerts alerts_20240513.json + + # Convert to specific output file + ./cli.py trivy convert-alerts alerts_20240513.json --output /path/to/output.csv ``` 3. Compare new alerts against existing POAMs: diff --git a/cli/cli.py b/cli/cli.py index e722a92..d3f130e 100755 --- a/cli/cli.py +++ b/cli/cli.py @@ -67,20 +67,31 @@ def preview_trivy(file_path, limit): sys.exit(1) @trivy.command('download-alerts') -def download_alerts(): +@click.option('--destination', '-d', type=click.Path(), help='Destination file path for the alerts JSON file') +def download_alerts(destination): """Download Trivy alerts from GitHub code scanning API. - REPO: Optional GitHub repository in owner/name format (e.g. 'owner/repo') - If not provided, defaults to configured repository + If destination is not specified, uses WORKING environment variable or pwd/working + and sets filename to trivy-alerts-.json Requires one of: 1. GitHub CLI (gh) to be installed and authenticated via 'gh auth login' 2. GitHub token provided via --token option or GITHUB_TOKEN environment variable - - The alerts will be saved as a JSON file in the working directory. """ try: - output_file = download_trivy_alerts() + if destination: + # User provided a specific file path + output_file = Path(destination) + output_dir = output_file.parent + output_file = download_trivy_alerts(output_dir) + # Rename to the exact file the user requested + if output_file.name != Path(destination).name: + final_output = Path(destination) + output_file.rename(final_output) + output_file = final_output + else: + # Use default behavior with WORKING env var or pwd/working + output_file = download_trivy_alerts() click.echo(f"Successfully downloaded alerts to: {output_file}") except Exception as e: click.echo(f"Error: {str(e)}", err=True) @@ -88,17 +99,19 @@ def download_alerts(): @trivy.command('convert-alerts') @click.argument('alerts_file', type=click.Path(exists=True)) -def convert_alerts(alerts_file): +@click.option('--output', '-o', type=click.Path(), help='Output file path (default: same directory as input with .findings.csv extension)') +def convert_alerts(alerts_file, output): """Convert GitHub Trivy alerts JSON to POAM CSV format. ALERTS_FILE should be a JSON file containing GitHub code scanning alerts. The file can be obtained using the download-alerts command. - The converted POAM data will be saved as a CSV file in the working directory. + The converted POAM data will be saved as a CSV file with .findings.csv extension. """ try: alerts_path = Path(alerts_file) - output_file = convert_alerts_to_poam(alerts_path) + output_path = Path(output) if output else None + output_file = convert_alerts_to_poam(alerts_path, output_path) click.echo(f"Successfully converted alerts to POAM format: {output_file}") except Exception as e: click.echo(f"Error: {str(e)}", err=True) diff --git a/tools/github.py b/tools/github.py index f981e0d..88da878 100644 --- a/tools/github.py +++ b/tools/github.py @@ -8,19 +8,23 @@ from .utils import ensure_working_dir -def download_trivy_alerts() -> Path: +def download_trivy_alerts(output_dir: Path = None) -> Path: """ Download Trivy alerts from GitHub code scanning API. Uses gh CLI tool to handle authentication and pagination. + Args: + output_dir: Directory to save the alerts file. If None, uses working directory. + Returns: Path to the downloaded JSON file """ - working_dir = ensure_working_dir() + if output_dir is None: + output_dir = ensure_working_dir() # Generate timestamp for the filename - timestamp = datetime.now().strftime("%Y%m%d_%H%M%S") - output_file = working_dir / f"trivy_alerts_{timestamp}.json" + timestamp = datetime.now().strftime("%Y%m%d") + output_file = output_dir / f"trivy_alerts_{timestamp}.json" try: # Run gh command and capture output diff --git a/tools/trivy/alerts.py b/tools/trivy/alerts.py index 2190b82..7c613fd 100644 --- a/tools/trivy/alerts.py +++ b/tools/trivy/alerts.py @@ -80,12 +80,13 @@ def date_plus(iso_date_string: str, days_to_add: int) -> str: except ValueError as e: raise ValueError(f"Invalid ISO date string format: {iso_date_string}") from e -def convert_alerts_to_poam(alerts_file: Path) -> Path: +def convert_alerts_to_poam(alerts_file: Path, output_path: Path = None) -> Path: """ Convert GitHub Trivy alerts JSON to POAM CSV format. Args: alerts_file: Path to the JSON file containing GitHub alerts + output_path: Optional path for the output file. If None, uses same parent directory as input file. Returns: Path to the generated CSV file @@ -136,9 +137,12 @@ def convert_alerts_to_poam(alerts_file: Path) -> Path: rows.append(row) - # Generate output filename with timestamp - timestamp = datetime.now().strftime("%Y%m%d_%H%M%S") - output_file = ensure_working_dir() / f"trivy_alerts_{timestamp}.csv" + # Determine output file path + if output_path is None: + # Use same parent directory as input file, change extension to .findings.csv + output_file = alerts_file.parent / f"{alerts_file.stem}.findings.csv" + else: + output_file = output_path # Write CSV file with output_file.open('w', newline='') as csvfile: diff --git a/tools/utils.py b/tools/utils.py index 1979816..f34980a 100644 --- a/tools/utils.py +++ b/tools/utils.py @@ -9,9 +9,18 @@ def ensure_working_dir() -> Path: Ensure the working directory exists and return its path. The working directory is used for temporary files and downloads. + Checks for WORKING environment variable first, then falls back to pwd/working. + Returns: Path object for the working directory """ - working_dir = Path(os.getcwd()) / 'working' + # Check WORKING environment variable first + working_env = os.getenv('WORKING') + if working_env: + working_dir = Path(working_env) + else: + # Fall back to pwd/working + working_dir = Path(os.getcwd()) / 'working' + working_dir.mkdir(exist_ok=True) return working_dir \ No newline at end of file From ff386cd9f04dcde3970c9669ae70185a05ffee57 Mon Sep 17 00:00:00 2001 From: Chris Llanwarne Date: Tue, 14 Oct 2025 14:13:54 -0400 Subject: [PATCH 23/33] poams weekly-update process --- README.md | 3 + cli/README.md | 24 ++++++++ cli/cli.py | 137 +++++++++++++++++++++++++++++++++++++++-- tools/cis/converter.py | 8 ++- tools/cis/splitter.py | 6 +- tools/zap/alerts.py | 12 ++-- 6 files changed, 175 insertions(+), 15 deletions(-) diff --git a/README.md b/README.md index 6a8152d..897abae 100644 --- a/README.md +++ b/README.md @@ -51,6 +51,9 @@ The CLI is organized into the following command groups: 1. `poams` - Commands for working with POAMs: ```bash + # Interactive weekly update process + ./cli/cli.py poams weekly-update + # Preview Trivy POAMs from an Excel file ./cli/cli.py poams preview-trivy [--limit ] diff --git a/cli/README.md b/cli/README.md index 40b0881..d01d0da 100644 --- a/cli/README.md +++ b/cli/README.md @@ -63,6 +63,9 @@ The CLI is organized into command groups for better organization and usability. Commands for working with POAMs are grouped under the `poams` command: ```bash +# Interactive weekly update process +./cli.py poams weekly-update + # Preview POAMs from an Excel file ./cli.py poams preview-trivy [--limit ] @@ -70,6 +73,27 @@ Commands for working with POAMs are grouped under the `poams` command: ./cli.py poams apply-diff ``` +#### Weekly Update Process + +The `weekly-update` command provides an interactive workflow for processing weekly security findings: + +1. **Working Directory Setup**: Prompts for a working directory (default: `working/YYYY-MM-DD`) +2. **Directory Contents**: Shows current directory contents with relative paths +3. **Input Files**: Prompts for paths to: + - Continuous CIS findings sheet (suggests files with "CIS" in name if found) + - Most recent ZAP scan (suggests files starting with "hail_report" if found) + - Current POAMs file (suggests files with "POAM" in name if found) +4. **Trivy Processing**: Interactive prompts for: + - Downloading Trivy alerts + - Converting alerts to findings CSV +5. **CIS Processing**: Interactive prompts for: + - Splitting connected sheet + - Converting most recent findings to JSON +6. **ZAP Processing**: Interactive prompts for: + - Converting ZAP scan to findings JSON + +All prompts show suggested defaults and accept empty input to use the default value. + ### Trivy Commands Commands for working with Trivy alerts are grouped under the `trivy` command: diff --git a/cli/cli.py b/cli/cli.py index d3f130e..7030541 100755 --- a/cli/cli.py +++ b/cli/cli.py @@ -24,6 +24,7 @@ from tools.cis.converter import convert_to_findings_file from tools.cis.diff import compare_findings_to_cis_poams + @click.group() def cli(): """Security tools CLI.""" @@ -244,9 +245,131 @@ def merge_diffs_cmd(diff_files: tuple, output: Optional[Path]) -> None: click.echo(traceback.format_exc(), err=True) sys.exit(1) +@poams.command('weekly-update') +def weekly_update(): + """Interactive weekly update process for POAMs. + + This command guides you through the weekly update process including: + - Setting up working directory + - Processing Trivy alerts + - Processing CIS findings + - Processing ZAP scans + """ + from datetime import datetime + + try: + # 1. Prompt for working directory + today = datetime.now().strftime("%Y-%m-%d") + default_working_dir = f"working/{today}" + working_dir = click.prompt("Working directory", default=default_working_dir) + working_path = Path(working_dir) + working_path.mkdir(parents=True, exist_ok=True) + + # 2. Print current directory contents + click.echo(f"\nCurrent directory contents:") + items = list(sorted(working_path.iterdir())) + if len(items) > 0: + for item in items: + click.echo(f" {item}") + else: + click.echo(" (empty)") + + # 3. Ask for input file paths + click.echo(f"\n--- Input Files ---") + + # Find files in working directory + cis_files = list(working_path.glob("*CIS*")) + hail_files = list(working_path.glob("hail_report*")) + poam_files = list(working_path.glob("*POAM*")) + + # Use the first found file or empty string if none found + cis_default = str(cis_files[0]) if cis_files else "" + zap_default = str(hail_files[0]) if hail_files else "" + poams_default = str(poam_files[0]) if poam_files else "" + + cis_findings = click.prompt("Path to continuous CIS findings sheet", default=cis_default) + zap_scan = click.prompt("Path to most recent ZAP scan", default=zap_default) + poams_file = click.prompt("Path to current POAMs file", default=poams_default) + + # 4. Trivy actions + click.echo(f"\n--- Trivy Actions ---") + trivy_alerts_file = working_path / f"trivy-alerts-{today}.json" + trivy_findings_file = working_path / f"trivy-findings-{today}.findings.csv" + + skip_trivy = False + if trivy_findings_file.exists(): + skip_trivy = click.confirm(f"Trivy findings file {trivy_findings_file} already exists. Skip Trivy actions?") + + if not skip_trivy: + if click.confirm(f"Download Trivy alerts to {trivy_alerts_file}?"): + click.echo(f"> trivy download-alerts -d {trivy_alerts_file}") + download_trivy_alerts(trivy_alerts_file.parent) + # Rename to the exact file we want + if trivy_alerts_file.exists(): + trivy_alerts_file.unlink() # Remove if it exists with different name + # Find the actual downloaded file and rename it + for file in working_path.glob("trivy_alerts_*.json"): + file.rename(trivy_alerts_file) + break + + if click.confirm(f"Convert Trivy alerts to findings CSV?"): + click.echo(f"> trivy convert-alerts {trivy_alerts_file} -o {trivy_findings_file}") + convert_alerts_to_poam(trivy_alerts_file, trivy_findings_file) + + # 5. CIS actions + click.echo(f"\n--- CIS Actions ---") + split_output_dir = working_path / "Divided CIS Scans" + cis_findings_file = working_path / f"cis-findings-{today}.findings.json" + + skip_cis = False + if cis_findings_file.exists(): + skip_cis = click.confirm(f"CIS findings file {cis_findings_file} already exists. Skip CIS actions?") + + if not skip_cis: + if click.confirm(f"Split CIS connected sheet?"): + click.echo(f"> cis split-connected-sheet {cis_findings} -o {split_output_dir}") + split_connected_sheet(Path(cis_findings), split_output_dir) + + # Find the most recent findings file + if split_output_dir.exists(): + csv_files = list(split_output_dir.glob("*.csv")) + if csv_files: + most_recent_file = max(csv_files, key=lambda f: f.stat().st_mtime) + click.echo(f"Most recent findings file: {most_recent_file}") + + if click.confirm(f"Convert CIS CSV to findings?"): + click.echo(f"> cis csv-to-findings {most_recent_file} -o {cis_findings_file}") + convert_to_findings_file(most_recent_file, cis_findings_file) + + # 6. ZAP actions + click.echo(f"\n--- ZAP Actions ---") + zap_findings_file = working_path / f"zap-findings-{today}.findings.json" + + skip_zap = False + if zap_findings_file.exists(): + skip_zap = click.confirm(f"ZAP findings file {zap_findings_file} already exists. Skip ZAP actions?") + + if not skip_zap: + if click.confirm(f"Convert ZAP scan to findings?"): + click.echo(f"> zap alerts-to-findings {zap_scan} -o {zap_findings_file}") + convert_alerts_to_findings(zap_scan, str(zap_findings_file)) + + click.echo(f"\n--- Weekly Update Complete ---") + click.echo(f"Working directory: {working_path}") + click.echo(f"Generated files:") + click.echo(f" trivy findings: {trivy_findings_file}") + click.echo(f" cis findings: {cis_findings_file}") + click.echo(f" zap findings: {zap_findings_file}") + + except Exception as e: + click.echo(f"Error during weekly update: {str(e)}", err=True) + click.echo(traceback.format_exc(), err=True) + sys.exit(1) + @zap.command('alerts-to-findings') @click.argument('csv_file', type=click.Path(exists=True)) -def alerts_to_findings(csv_file): +@click.option('--output', '-o', type=click.Path(), help='Output file path (default: input file with .findings.json extension)') +def alerts_to_findings(csv_file, output): """Convert ZAP CSV alerts to findings JSON format. CSV_FILE should be a ZAP CSV report file. @@ -254,7 +377,7 @@ def alerts_to_findings(csv_file): """ try: # Convert alerts to findings - output_file = convert_alerts_to_findings(csv_file) + output_file = convert_alerts_to_findings(csv_file, output) # Load and display first finding with open(output_file) as f: @@ -317,7 +440,8 @@ def alerts_diff(poam_file: str, findings_file: str, json_output: Optional[str]) @cis.command('split-connected-sheet') @click.argument('xlsx_file', type=click.Path(exists=True, path_type=Path)) -def split_connected_sheet_cmd(xlsx_file: Path) -> None: +@click.option('--output', '-o', type=click.Path(path_type=Path), help='Output directory for split CSV files (default: input directory/Divided CIS Scans)') +def split_connected_sheet_cmd(xlsx_file: Path, output: Optional[Path]) -> None: """Split a CIS connected sheet into separate CSV files by date. XLSX_FILE should be a CIS connected sheet Excel file. @@ -329,7 +453,7 @@ def split_connected_sheet_cmd(xlsx_file: Path) -> None: - Skip writing if a file for a particular date already exists """ try: - output_files = split_connected_sheet(xlsx_file) + output_files = split_connected_sheet(xlsx_file, output) if output_files: click.echo(f"Successfully split {xlsx_file.name} into {len(output_files)} files:") for f in output_files: @@ -342,7 +466,8 @@ def split_connected_sheet_cmd(xlsx_file: Path) -> None: @cis.command('csv-to-findings') @click.argument('csv_file', type=click.Path(exists=True, path_type=Path)) -def csv_to_findings_cmd(csv_file: Path) -> None: +@click.option('--output', '-o', type=click.Path(path_type=Path), help='Output file path (default: input file with .findings.json extension)') +def csv_to_findings_cmd(csv_file: Path, output: Optional[Path]) -> None: """Convert a CIS CSV file to findings JSON format. CSV_FILE should be a CIS CSV file (typically from split-connected-sheet). @@ -353,7 +478,7 @@ def csv_to_findings_cmd(csv_file: Path) -> None: - Save the findings as .findings.json """ try: - output_file = convert_to_findings_file(csv_file) + output_file = convert_to_findings_file(csv_file, output) # Load and display summary with open(output_file) as f: diff --git a/tools/cis/converter.py b/tools/cis/converter.py index 27b96f2..be56912 100644 --- a/tools/cis/converter.py +++ b/tools/cis/converter.py @@ -108,12 +108,13 @@ def convert_csv_to_findings(input_file: Path) -> List[Finding]: return findings -def convert_to_findings_file(input_file: Path) -> Path: +def convert_to_findings_file(input_file: Path, output_file: Path = None) -> Path: """ Convert a CIS CSV file to a findings JSON file. Args: input_file: Path to the input CSV file + output_file: Optional output file path. If None, uses input file with .findings.json extension Returns: Path to the output JSON file @@ -131,8 +132,11 @@ def convert_to_findings_file(input_file: Path) -> Path: finding_dict[key] = value.strftime("%Y-%m-%d") findings_data.append(finding_dict) + # Determine output file path + if output_file is None: + output_file = input_file.with_suffix('.findings.json') + # Write to JSON file - output_file = input_file.with_suffix('.findings.json') with open(output_file, 'w') as f: json.dump(findings_data, f, indent=2) diff --git a/tools/cis/splitter.py b/tools/cis/splitter.py index 5429b2f..36c9d20 100644 --- a/tools/cis/splitter.py +++ b/tools/cis/splitter.py @@ -6,12 +6,13 @@ from datetime import datetime import os -def split_connected_sheet(input_file: Path) -> list[Path]: +def split_connected_sheet(input_file: Path, output_dir: Path = None) -> list[Path]: """ Split a CIS connected sheet Excel file into multiple CSV files by date. Args: input_file: Path to the input Excel file + output_dir: Optional output directory. If None, uses input directory/Divided CIS Scans Returns: List of paths to the generated CSV files @@ -26,7 +27,8 @@ def split_connected_sheet(input_file: Path) -> list[Path]: df = pd.read_excel(input_file) # Output directory is input directory with a "Divided CIS Scans" subdirectory - output_dir = input_file.parent / "Divided CIS Scans" + if output_dir is None: + output_dir = input_file.parent / "Divided CIS Scans" # Ensure output directory exists output_dir.mkdir(exist_ok=True) diff --git a/tools/zap/alerts.py b/tools/zap/alerts.py index 9e14245..dc82553 100644 --- a/tools/zap/alerts.py +++ b/tools/zap/alerts.py @@ -89,12 +89,13 @@ def parse_zap_csv(csv_file: str) -> List[Finding]: return findings -def convert_alerts_to_findings(csv_file: str) -> str: +def convert_alerts_to_findings(csv_file: str, output_file: str = None) -> str: """ Convert ZAP CSV alerts to findings JSON format. Args: csv_file: Path to the ZAP CSV report file + output_file: Optional output file path. If None, uses input file with .findings.json extension Returns: Path to the output JSON file @@ -111,12 +112,13 @@ def convert_alerts_to_findings(csv_file: str) -> str: finding_dict[key] = value.strftime('%Y-%m-%d %H:%M:%S') findings_data.append(finding_dict) - # Generate output filename - input_path = Path(csv_file) - output_file = input_path.with_suffix('.findings.json') + # Determine output filename + if output_file is None: + input_path = Path(csv_file) + output_file = str(input_path.with_suffix('.findings.json')) # Write findings to JSON file with open(output_file, 'w') as f: json.dump(findings_data, f, indent=2) - return str(output_file) \ No newline at end of file + return output_file \ No newline at end of file From 1009358781242015a516aa34fa203c3f807bdb00 Mon Sep 17 00:00:00 2001 From: Chris Llanwarne Date: Tue, 14 Oct 2025 14:44:05 -0400 Subject: [PATCH 24/33] Complete weekly update process command --- cli/cli.py | 317 +++++++++++++++++++++++++++++++++++++---------------- 1 file changed, 221 insertions(+), 96 deletions(-) diff --git a/cli/cli.py b/cli/cli.py index 7030541..1540dd7 100755 --- a/cli/cli.py +++ b/cli/cli.py @@ -24,6 +24,117 @@ from tools.cis.converter import convert_to_findings_file from tools.cis.diff import compare_findings_to_cis_poams +def generate_alerts_diff(findings_file: Path, poams_file: Path, diff_file: Path, + findings_loader, diff_generator, file_type: str) -> None: + """Generate alerts diff for a findings file. + + Args: + findings_file: Path to the findings file + poams_file: Path to the POAMs file + diff_file: Path where the diff file should be saved + findings_loader: Function to load findings from the file + diff_generator: Function to generate the diff + file_type: Type of findings (for display purposes) + + Raises: + FileNotFoundError: If findings file doesn't exist + """ + if not findings_file.exists(): + raise FileNotFoundError(f"{file_type} findings file not found: {findings_file}") + + click.echo(f"> {file_type.lower()} alerts-diff {poams_file} {findings_file}") + + # Load findings + all_findings = findings_loader(findings_file) + if not all_findings: + click.echo(f"No findings found in {file_type} file") + return + + # Filter out Info severity findings + findings = [f for f in all_findings if f.original_risk_rating.lower() != 'info'] + info_count = len(all_findings) - len(findings) + + if info_count > 0: + click.echo(f"Excluded {info_count} findings with Info severity") + + if not findings: + click.echo("No findings remaining after filtering out Info severity") + return + + # Generate diff + diff = diff_generator(findings, poams_file) + + # Print results + diff.print_summary() + + # Save JSON output + with open(diff_file, 'w') as f: + json.dump(diff.to_json(), f, indent=2) + click.echo(f"{file_type} diff saved to: {diff_file}") + +def load_trivy_findings(csv_file: Path): + """Load Trivy findings from CSV file.""" + return import_alerts_from_csv(csv_file) + +def load_json_findings(json_file: Path): + """Load findings from JSON file.""" + with open(json_file) as f: + findings_data = json.load(f) + return [Finding.from_dict(f) for f in findings_data] + +def apply_poam_diffs(poam_file: Path, diff_files: list[Path], output_file: Path) -> None: + """Apply diff changes to a POAM Excel file. + + Args: + poam_file: Path to the original POAM file + diff_files: List of diff files to apply + output_file: Path where the updated POAM file should be saved + + Raises: + FileNotFoundError: If POAM file or diff files don't exist + """ + if not poam_file.exists(): + raise FileNotFoundError(f"POAM file not found: {poam_file}") + + for diff_file in diff_files: + if not diff_file.exists(): + raise FileNotFoundError(f"Diff file not found: {diff_file}") + + click.echo(f"> poams apply-diff {poam_file} {' '.join(str(f) for f in diff_files)}") + + # Copy the original file to the output location + import shutil + shutil.copy2(poam_file, output_file) + + # Apply the diffs + apply_diff_from_files(output_file, diff_files) + click.echo(f"Successfully applied diff changes to {output_file}") + +def generate_updated_poam_filename(original_poam_file: str, today: str) -> str: + """Generate updated POAM filename by replacing date in original filename with today's date. + + Args: + original_poam_file: Original POAM file path + today: Today's date in YYYY-MM-DD format + + Returns: + Updated filename with today's date + """ + from datetime import datetime + import re + + # Try to find a date pattern in the filename (YYYY-MM-DD or YYYY_MM_DD) + date_pattern = r'(\d{4}[-_]\d{2}[-_]\d{2})' + match = re.search(date_pattern, original_poam_file) + + if match: + # Replace the found date with today's date + return re.sub(date_pattern, today, original_poam_file) + else: + # If no date found, insert today's date before the extension + path = Path(original_poam_file) + return str(path.parent / f"{path.stem} - {today}{path.suffix}") + @click.group() def cli(): @@ -136,35 +247,15 @@ def alerts_diff(poam_file: Path, alerts_csv: Path): Note: Findings with Info severity are automatically excluded. """ try: - # Import findings from CSV - all_findings = import_alerts_from_csv(alerts_csv) - if not all_findings: - click.echo("No findings found in CSV file", err=True) - sys.exit(1) - - # Filter out Info severity findings - findings = [f for f in all_findings if f.original_risk_rating.lower() != 'info'] - info_count = len(all_findings) - len(findings) - - if info_count > 0: - click.echo(f"Excluded {info_count} findings with Info severity") - - if not findings: - click.echo("No findings remaining after filtering out Info severity", err=True) - sys.exit(1) - - # Compare findings against POAMs - diff = compare_findings_to_trivy_poams(findings, poam_file) - - # Print results - diff.print_summary() - - # JSON output file path: json_output_file = alerts_csv.with_suffix('.diff.json') - with open(json_output_file, 'w') as f: - json.dump(diff.to_json(), f) - click.echo(f"JSON output saved to: {json_output_file}") - + generate_alerts_diff( + alerts_csv, + poam_file, + json_output_file, + load_trivy_findings, + compare_findings_to_trivy_poams, + "Trivy" + ) except Exception as e: click.echo(f"Error comparing alerts: {str(e)}", err=True) sys.exit(1) @@ -278,9 +369,9 @@ def weekly_update(): click.echo(f"\n--- Input Files ---") # Find files in working directory - cis_files = list(working_path.glob("*CIS*")) - hail_files = list(working_path.glob("hail_report*")) - poam_files = list(working_path.glob("*POAM*")) + cis_files = list(working_path.glob("*CIS*.xlsx")) + hail_files = list(working_path.glob("hail_report*.csv")) + poam_files = list(working_path.glob("*POAM*.xlsx")) # Use the first found file or empty string if none found cis_default = str(cis_files[0]) if cis_files else "" @@ -298,10 +389,10 @@ def weekly_update(): skip_trivy = False if trivy_findings_file.exists(): - skip_trivy = click.confirm(f"Trivy findings file {trivy_findings_file} already exists. Skip Trivy actions?") + skip_trivy = click.confirm(f"Trivy findings file {trivy_findings_file} already exists. Skip Trivy actions?", default=True) if not skip_trivy: - if click.confirm(f"Download Trivy alerts to {trivy_alerts_file}?"): + if click.confirm(f"Download Trivy alerts to {trivy_alerts_file}?", default=True, abort=True): click.echo(f"> trivy download-alerts -d {trivy_alerts_file}") download_trivy_alerts(trivy_alerts_file.parent) # Rename to the exact file we want @@ -312,7 +403,7 @@ def weekly_update(): file.rename(trivy_alerts_file) break - if click.confirm(f"Convert Trivy alerts to findings CSV?"): + if click.confirm(f"Convert Trivy alerts to findings CSV?", default=True, abort=True): click.echo(f"> trivy convert-alerts {trivy_alerts_file} -o {trivy_findings_file}") convert_alerts_to_poam(trivy_alerts_file, trivy_findings_file) @@ -323,10 +414,10 @@ def weekly_update(): skip_cis = False if cis_findings_file.exists(): - skip_cis = click.confirm(f"CIS findings file {cis_findings_file} already exists. Skip CIS actions?") + skip_cis = click.confirm(f"CIS findings file {cis_findings_file} already exists. Skip CIS actions?", default=True) if not skip_cis: - if click.confirm(f"Split CIS connected sheet?"): + if click.confirm(f"Split CIS connected sheet?", default=True, abort=True): click.echo(f"> cis split-connected-sheet {cis_findings} -o {split_output_dir}") split_connected_sheet(Path(cis_findings), split_output_dir) @@ -337,7 +428,7 @@ def weekly_update(): most_recent_file = max(csv_files, key=lambda f: f.stat().st_mtime) click.echo(f"Most recent findings file: {most_recent_file}") - if click.confirm(f"Convert CIS CSV to findings?"): + if click.confirm(f"Convert CIS CSV to findings?", default=True, abort=True): click.echo(f"> cis csv-to-findings {most_recent_file} -o {cis_findings_file}") convert_to_findings_file(most_recent_file, cis_findings_file) @@ -347,19 +438,91 @@ def weekly_update(): skip_zap = False if zap_findings_file.exists(): - skip_zap = click.confirm(f"ZAP findings file {zap_findings_file} already exists. Skip ZAP actions?") + skip_zap = click.confirm(f"ZAP findings file {zap_findings_file} already exists. Skip ZAP actions?", default=True) if not skip_zap: - if click.confirm(f"Convert ZAP scan to findings?"): + if click.confirm(f"Convert ZAP scan to findings?", default=True, abort=True): click.echo(f"> zap alerts-to-findings {zap_scan} -o {zap_findings_file}") convert_alerts_to_findings(zap_scan, str(zap_findings_file)) + # 7. Generate alerts-diffs + click.echo(f"\n--- Generate Alerts Diffs ---") + + # Trivy alerts diff + trivy_diff_file = trivy_findings_file.with_suffix('.diff.json') + skip_trivy_diff = False + if trivy_diff_file.exists(): + skip_trivy_diff = click.confirm(f"Trivy diff file {trivy_diff_file} already exists. Skip Trivy diff generation?", default=True) + + if not skip_trivy_diff: + if click.confirm(f"Generate Trivy alerts diff?", default=True, abort=True): + generate_alerts_diff( + trivy_findings_file, + Path(poams_file), + trivy_diff_file, + load_trivy_findings, + compare_findings_to_trivy_poams, + "Trivy" + ) + + # CIS alerts diff + cis_diff_file = cis_findings_file.with_suffix('.diff.json') + skip_cis_diff = False + if cis_diff_file.exists(): + skip_cis_diff = click.confirm(f"CIS diff file {cis_diff_file} already exists. Skip CIS diff generation?", default=True) + + if not skip_cis_diff: + if click.confirm(f"Generate CIS alerts diff?", default=True, abort=True): + generate_alerts_diff( + cis_findings_file, + Path(poams_file), + cis_diff_file, + load_json_findings, + compare_findings_to_cis_poams, + "CIS" + ) + + # ZAP alerts diff + zap_diff_file = zap_findings_file.with_suffix('.diff.json') + skip_zap_diff = False + if zap_diff_file.exists(): + skip_zap_diff = click.confirm(f"ZAP diff file {zap_diff_file} already exists. Skip ZAP diff generation?", default=True) + + if not skip_zap_diff: + if click.confirm(f"Generate ZAP alerts diff?", default=True, abort=True): + generate_alerts_diff( + zap_findings_file, + Path(poams_file), + zap_diff_file, + load_json_findings, + compare_findings_to_zap_poams, + "ZAP" + ) + + # 8. Apply diffs + click.echo(f"\n--- Apply Diffs ---") + + # Generate updated POAM filename + updated_poam_file = generate_updated_poam_filename(poams_file, today) + updated_poam_path = Path(updated_poam_file) + + skip_apply_diffs = False + if updated_poam_path.exists(): + skip_apply_diffs = click.confirm(f"Updated POAM file {updated_poam_path} already exists. Skip applying diffs?", default=True) + + if not skip_apply_diffs: + if click.confirm(f"Apply diffs to create updated POAMs?", default=True, abort=True): + diff_files = [trivy_diff_file, cis_diff_file, zap_diff_file] + + apply_poam_diffs(Path(poams_file), diff_files, updated_poam_path) + click.echo(f"\n--- Weekly Update Complete ---") click.echo(f"Working directory: {working_path}") - click.echo(f"Generated files:") + click.echo(f"Output files to upload:") click.echo(f" trivy findings: {trivy_findings_file}") click.echo(f" cis findings: {cis_findings_file}") click.echo(f" zap findings: {zap_findings_file}") + click.echo(f" updated POAMs: {updated_poam_path}") except Exception as e: click.echo(f"Error during weekly update: {str(e)}", err=True) @@ -403,37 +566,18 @@ def alerts_diff(poam_file: str, findings_file: str, json_output: Optional[str]) Note: Findings with Info severity are automatically excluded. """ try: - # Load findings from JSON file - with open(findings_file) as f: - findings_data = json.load(f) - all_findings = [Finding.from_dict(f) for f in findings_data] - - # Filter out Info severity findings - findings = [f for f in all_findings if f.original_risk_rating.lower() != 'info'] - info_count = len(all_findings) - len(findings) - - if info_count > 0: - click.echo(f"Excluded {info_count} findings with Info severity") - - if not findings: - click.echo("No findings remaining after filtering out Info severity", err=True) - sys.exit(1) - - # Compare findings to POAMs - diff = compare_findings_to_zap_poams(findings, poam_file) - - # Print human readable summary - diff.print_summary() - - # Save JSON output if requested if not json_output: - # Finding file as a path: findings_path = Path(findings_file) json_output = findings_path.with_suffix('.diff.json') - json_data = diff.to_json() - with open(json_output, 'w') as f: - json.dump(json_data, f, indent=2) - click.echo(f"JSON output saved to: {json_output}") + + generate_alerts_diff( + Path(findings_file), + Path(poam_file), + Path(json_output), + load_json_findings, + compare_findings_to_zap_poams, + "ZAP" + ) except Exception as e: click.echo(f"Error comparing findings: {str(e)}", err=True) sys.exit(1) @@ -509,37 +653,18 @@ def alerts_diff(poam_file: str, findings_file: str, json_output: Optional[str]) Note: Findings with Info severity are automatically excluded. """ try: - # Load findings from JSON file - with open(findings_file) as f: - findings_data = json.load(f) - all_findings = [Finding.from_dict(f) for f in findings_data] - - # Filter out Info severity findings - findings = [f for f in all_findings if f.original_risk_rating.lower() != 'info'] - info_count = len(all_findings) - len(findings) - - if info_count > 0: - click.echo(f"Excluded {info_count} findings with Info severity") - - if not findings: - click.echo("No findings remaining after filtering out Info severity", err=True) - sys.exit(1) - - # Compare findings to configuration findings - diff = compare_findings_to_cis_poams(findings, poam_file) - - # Print human readable summary - diff.print_summary() - - # Save JSON output if requested if not json_output: - # Finding file as a path: findings_path = Path(findings_file) json_output = findings_path.with_suffix('.diff.json') - json_data = diff.to_json() - with open(json_output, 'w') as f: - json.dump(json_data, f, indent=2) - click.echo(f"JSON output saved to: {json_output}") + + generate_alerts_diff( + Path(findings_file), + Path(poam_file), + Path(json_output), + load_json_findings, + compare_findings_to_cis_poams, + "CIS" + ) except Exception as e: click.echo(f"Error comparing findings: {str(e)}", err=True) sys.exit(1) From 460c3156885485411d47927a46e36f1af95acec9 Mon Sep 17 00:00:00 2001 From: Chris Llanwarne Date: Tue, 14 Oct 2025 15:31:31 -0400 Subject: [PATCH 25/33] diff apply in weekly update --- cli/cli.py | 48 +++++++++++++-------------------------------- tools/diff_apply.py | 27 +++++++++++++++---------- 2 files changed, 31 insertions(+), 44 deletions(-) diff --git a/cli/cli.py b/cli/cli.py index 1540dd7..95e7966 100755 --- a/cli/cli.py +++ b/cli/cli.py @@ -82,33 +82,6 @@ def load_json_findings(json_file: Path): findings_data = json.load(f) return [Finding.from_dict(f) for f in findings_data] -def apply_poam_diffs(poam_file: Path, diff_files: list[Path], output_file: Path) -> None: - """Apply diff changes to a POAM Excel file. - - Args: - poam_file: Path to the original POAM file - diff_files: List of diff files to apply - output_file: Path where the updated POAM file should be saved - - Raises: - FileNotFoundError: If POAM file or diff files don't exist - """ - if not poam_file.exists(): - raise FileNotFoundError(f"POAM file not found: {poam_file}") - - for diff_file in diff_files: - if not diff_file.exists(): - raise FileNotFoundError(f"Diff file not found: {diff_file}") - - click.echo(f"> poams apply-diff {poam_file} {' '.join(str(f) for f in diff_files)}") - - # Copy the original file to the output location - import shutil - shutil.copy2(poam_file, output_file) - - # Apply the diffs - apply_diff_from_files(output_file, diff_files) - click.echo(f"Successfully applied diff changes to {output_file}") def generate_updated_poam_filename(original_poam_file: str, today: str) -> str: """Generate updated POAM filename by replacing date in original filename with today's date. @@ -263,7 +236,8 @@ def alerts_diff(poam_file: Path, alerts_csv: Path): @poams.command('apply-diff') @click.argument('poam_file', type=click.Path(exists=True, path_type=Path)) @click.argument('diff_files', nargs=-1, type=click.Path(exists=True, path_type=Path)) -def apply_diff(poam_file: Path, diff_files: tuple) -> None: +@click.option('--output', '-o', type=click.Path(path_type=Path), help='Output file path (default: creates timestamped backup)') +def apply_diff(poam_file: Path, diff_files: tuple, output: Optional[Path]) -> None: """Apply diff changes to a POAM Excel file. POAM_FILE: Excel file containing POAMs @@ -275,14 +249,16 @@ def apply_diff(poam_file: Path, diff_files: tuple) -> None: - Move closed POAMs from Open to Closed sheet If multiple diff files are provided, they will be merged before applying. + If --output is specified, the updated file will be saved to that location. + Otherwise, a default name will be used. """ try: if not diff_files: click.echo("Error: At least one diff file must be provided", err=True) sys.exit(1) - apply_diff_from_files(poam_file, list(diff_files)) - click.echo(f"Successfully applied diff changes to {poam_file}") + result = apply_diff_from_files(poam_file, list(diff_files), output) + click.echo(f"Successfully applied diff changes to {result}") except Exception as e: click.echo(f"Error applying diff: {str(e)}", err=True) click.echo("\nFull traceback:", err=True) @@ -421,11 +397,11 @@ def weekly_update(): click.echo(f"> cis split-connected-sheet {cis_findings} -o {split_output_dir}") split_connected_sheet(Path(cis_findings), split_output_dir) - # Find the most recent findings file + # Find the most recent findings file (sort by filename since filenames include the date in YYYY-MM-DD format and are otherwise the same) if split_output_dir.exists(): csv_files = list(split_output_dir.glob("*.csv")) if csv_files: - most_recent_file = max(csv_files, key=lambda f: f.stat().st_mtime) + most_recent_file = max(csv_files, key=lambda f: f.name) click.echo(f"Most recent findings file: {most_recent_file}") if click.confirm(f"Convert CIS CSV to findings?", default=True, abort=True): @@ -514,7 +490,11 @@ def weekly_update(): if click.confirm(f"Apply diffs to create updated POAMs?", default=True, abort=True): diff_files = [trivy_diff_file, cis_diff_file, zap_diff_file] - apply_poam_diffs(Path(poams_file), diff_files, updated_poam_path) + click.echo(f"> poams apply-diff {poams_file} {' '.join(str(f) for f in diff_files)} -o {updated_poam_path}") + + # Apply the diffs + result = apply_diff_from_files(poams_file, diff_files, updated_poam_path) + click.echo(f"Successfully applied diff changes to {result}") click.echo(f"\n--- Weekly Update Complete ---") click.echo(f"Working directory: {working_path}") @@ -522,7 +502,7 @@ def weekly_update(): click.echo(f" trivy findings: {trivy_findings_file}") click.echo(f" cis findings: {cis_findings_file}") click.echo(f" zap findings: {zap_findings_file}") - click.echo(f" updated POAMs: {updated_poam_path}") + click.echo(f" updated POAMs: {result}") except Exception as e: click.echo(f"Error during weekly update: {str(e)}", err=True) diff --git a/tools/diff_apply.py b/tools/diff_apply.py index 03d5d29..e45ab4d 100644 --- a/tools/diff_apply.py +++ b/tools/diff_apply.py @@ -8,12 +8,15 @@ import shutil import openpyxl -def create_updateable_copy(file_path: Path) -> Path: - """Create a timestamped backup copy of the Excel file.""" +def create_updateable_copy(file_path: Path, output_file: Path = None) -> Path: + """Create an updateable copy of the Excel file so that the original is not modified.""" timestamp = datetime.now().strftime('%Y%m%d-%H%M%S') - backup_path = file_path.parent / f"{file_path.stem}-diff-applied-{timestamp}{file_path.suffix}" - shutil.copy2(file_path, backup_path) - return backup_path + if output_file is None: + new_file_path = file_path.parent / f"{file_path.stem}-diff-applied-{timestamp}{file_path.suffix}" + else: + new_file_path = output_file + shutil.copy2(file_path, new_file_path) + return new_file_path def dict_to_row(data: Dict[str, Any]) -> Dict[str, Any]: """Convert a dictionary to row format.""" @@ -52,7 +55,7 @@ def dict_to_row(data: Dict[str, Any]) -> Dict[str, Any]: } return {excel_mapping[k]: v for k, v in data.items() if k in excel_mapping} -def apply_diff(poam_file: Path, diff_json: Dict[str, Any]) -> None: +def apply_diff(poam_file: Path, diff_json: Dict[str, Any], output_file: Path = None) -> Path: """ Apply diff changes to a POAM Excel file. @@ -61,7 +64,7 @@ def apply_diff(poam_file: Path, diff_json: Dict[str, Any]) -> None: diff_json: Dictionary containing diff changes """ # Create editable copy - editable_copy = create_updateable_copy(poam_file) + editable_copy = create_updateable_copy(poam_file, output_file) try: # Load workbook from editable copy @@ -170,10 +173,12 @@ def apply_diff(poam_file: Path, diff_json: Dict[str, Any]) -> None: # Save changes to the editable copy wb.save(editable_copy) - + return editable_copy except Exception as e: # If anything goes wrong, leave the half-edited copy for inspection raise type(e)(f"Error applying diff changes. Incomplete edit saved as {editable_copy}. Error: {str(e)}") from e + finally: + wb.close() def merge_diffs(diff_files: List[Path]) -> Dict[str, Any]: """ @@ -216,13 +221,15 @@ def merge_diffs(diff_files: List[Path]) -> Dict[str, Any]: return merged_diff -def apply_diff_from_files(poam_file: Path, diff_files: List[Path]) -> None: +def apply_diff_from_files(poam_file: Path, diff_files: List[Path], output_file: Path = None) -> Path: """ Apply diff changes from a JSON file to a POAM Excel file. Args: poam_file: Path to the POAM Excel file diff_files: List of paths to JSON diff files + output_file: Path to the output Excel file """ merged_diff = merge_diffs(diff_files) - apply_diff(poam_file, merged_diff) \ No newline at end of file + result = apply_diff(poam_file, merged_diff, output_file) + return result \ No newline at end of file From a81ae10e31b6315422acbc6d0427f0b676de6fe3 Mon Sep 17 00:00:00 2001 From: Chris Llanwarne Date: Thu, 5 Mar 2026 16:11:25 -0500 Subject: [PATCH 26/33] claude.md --- CLAUDE.md | 68 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 68 insertions(+) create mode 100644 CLAUDE.md diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 0000000..b543103 --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1,68 @@ +# CLAUDE.md + +This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository. + +## Overview + +A security findings tracker that manages POA&M (Plan of Action and Milestones) lifecycle for compliance purposes. The **CLI is the primary entry point** (`cli/cli.py`). The Streamlit web app in `app/` is abandonware — do not work on it. + +## Commands + +### Setup +```bash +python -m venv venv +source venv/bin/activate +pip install -r requirements.txt +``` + +### CLI Usage +```bash +./cli/cli.py --help +./cli/cli.py poams weekly-update # Interactive guided weekly update +./cli/cli.py poams apply-diff [ ...] +./cli/cli.py poams merge-diffs -o merged.json +./cli/cli.py trivy download-alerts [-d ] +./cli/cli.py trivy convert-alerts [-o ] +./cli/cli.py trivy alerts-diff +./cli/cli.py zap alerts-to-findings [-o ] +./cli/cli.py zap alerts-diff +./cli/cli.py cis split-connected-sheet [-o ] +./cli/cli.py cis csv-to-findings [-o ] +./cli/cli.py cis alerts-diff +``` + +### Tests +```bash +python -m pytest # Run all tests +python -m pytest tests/test_diff.py # Run a single test file +python -m pytest tests/test_diff.py::test_name # Run a single test +``` + +## Architecture + +### Data Flow (weekly update) +1. **Download/collect** raw scan data: Trivy alerts from GitHub API, CIS Excel sheet, ZAP CSV report +2. **Convert** to normalized intermediate formats: `.findings.csv` (Trivy) or `.findings.json` (CIS, ZAP) +3. **Diff** each findings file against the existing POAM Excel → produces `.diff.json` files +4. **Apply** one or more diff JSONs to create an updated POAM Excel file + +### Core Data Structures (`tools/`) +- `tools/findings.py` — `Finding` dataclass: source-agnostic normalized security finding +- `tools/poam.py` — `PoamEntry` dataclass + `PoamFile` class: reads POAM Excel files (headers on row 5 of "Open POA&M Items" / "Closed POA&M Items" sheets) +- `tools/diff.py` — `compare_findings_to_poams()` and `PoamFileDiff`: matches findings to POAMs by exact `weakness_name` + asset coverage; produces lists of new/existing/closed/reopened +- `tools/diff_apply.py` — `apply_diff()`: writes changes back to Excel using openpyxl + +### Source-Specific Modules +Each scanner type (`trivy/`, `zap/`, `cis/`) has: +- `alerts.py` or `converter.py` — converts raw scanner output to `Finding` objects +- `diff.py` — calls `compare_findings_to_poams()` with the right POAM filter and generator +- `poam_generator.py` — generates `PoamEntry` objects with appropriate POAM IDs + +POAM ID formats: Trivy → `YYYY-TRIVYXXXX`, CIS → `CIS--XXXX` + +### Working Directory Convention +By default, files are saved to `working/YYYY-MM-DD/`. The `WORKING` environment variable can override the base path. + +### Authentication +- GitHub (for Trivy downloads): `gh auth login` or `GITHUB_TOKEN` env var +- Google services: `gcloud auth application-default login` or `GOOGLE_APPLICATION_CREDENTIALS` env var From 2a29e777a1c39de955cdcab8867f3b4a78093d23 Mon Sep 17 00:00:00 2001 From: Chris Llanwarne Date: Thu, 5 Mar 2026 16:12:03 -0500 Subject: [PATCH 27/33] better gitignore --- .gitignore | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitignore b/.gitignore index 9171af1..f811134 100644 --- a/.gitignore +++ b/.gitignore @@ -175,6 +175,9 @@ cython_debug/ # PyPI configuration file .pypirc +# macOS +.DS_Store + # Ignore any findings.db files findings.db working/ From 76c8838a29af3050578f150e548e334aca2cd593 Mon Sep 17 00:00:00 2001 From: Chris Llanwarne Date: Thu, 5 Mar 2026 16:30:08 -0500 Subject: [PATCH 28/33] fix missing trivy severity --- tools/trivy/alerts.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/tools/trivy/alerts.py b/tools/trivy/alerts.py index 7c613fd..b1555a9 100644 --- a/tools/trivy/alerts.py +++ b/tools/trivy/alerts.py @@ -32,7 +32,7 @@ "Milestone Changes": "", "Vendor Dependency": "Yes", "Vendor Dependent Product Name": "Ubuntu", - "Original Risk Rating": .rule.security_severity_level, + "Original Risk Rating": (.rule.security_severity_level // (.rule.tags | map(select(test("^(critical|high|medium|low)$"; "i"))) | first | ascii_downcase)), "Adjusted Risk Rating": "", "Risk Adjustment": "", "False Positive": "No", @@ -120,8 +120,11 @@ def convert_alerts_to_poam(alerts_file: Path, output_path: Path = None) -> Path: # Handle dates and intervals orig_date = row["Original Detection Date"] status_date = row["Status Date"] + if not row["Original Risk Rating"]: + print(f"Excluded alert {row['Alert ID']} ({row['CVE']}): no severity rating") + continue sev = row["Original Risk Rating"].lower() - + # Calculate fix date based on severity fix_intervals = {"high": 14, "medium": 90, "low": 180} fix_interval = fix_intervals.get(sev, 0) From c14ee12355a1d26a5d3ac2554d036a55b81a92e1 Mon Sep 17 00:00:00 2001 From: Chris Llanwarne Date: Thu, 5 Mar 2026 17:07:37 -0500 Subject: [PATCH 29/33] remove old cruft --- app/__init__.py | 3 - app/components/IssuesList.py | 29 -- app/components/IssuesTable.py | 181 ------------ app/components/__init__.py | 3 - app/components/data_processor.py | 374 ------------------------- app/components/poam_exports.py | 118 -------- app/database/__init__.py | 3 - app/database/schema.py | 342 ---------------------- app/main.py | 204 -------------- app/pages/1_issue_detail.py | 170 ----------- app/pages/2_scan_history.py | 176 ------------ app/pages/3_benchmark_detail.py | 237 ---------------- app/pages/4_settings.py | 52 ---- app/pages/5_poam_export.py | 99 ------- app/pages/settings.py | 35 --- app/static/style.css | 177 ------------ cli/README.md | 175 ------------ requirements.txt | 5 +- tests/test_diff.py | 2 +- tools/cis/poam_generator.py | 2 +- tools/trivy/trivy_alerts_poaminator.py | 146 ---------- 21 files changed, 3 insertions(+), 2530 deletions(-) delete mode 100644 app/__init__.py delete mode 100644 app/components/IssuesList.py delete mode 100644 app/components/IssuesTable.py delete mode 100644 app/components/__init__.py delete mode 100644 app/components/data_processor.py delete mode 100644 app/components/poam_exports.py delete mode 100644 app/database/__init__.py delete mode 100644 app/database/schema.py delete mode 100644 app/main.py delete mode 100644 app/pages/1_issue_detail.py delete mode 100644 app/pages/2_scan_history.py delete mode 100644 app/pages/3_benchmark_detail.py delete mode 100644 app/pages/4_settings.py delete mode 100644 app/pages/5_poam_export.py delete mode 100644 app/pages/settings.py delete mode 100644 app/static/style.css delete mode 100644 cli/README.md delete mode 100644 tools/trivy/trivy_alerts_poaminator.py diff --git a/app/__init__.py b/app/__init__.py deleted file mode 100644 index d66fd94..0000000 --- a/app/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -""" -Security Findings Tracker application package -""" \ No newline at end of file diff --git a/app/components/IssuesList.py b/app/components/IssuesList.py deleted file mode 100644 index a310df4..0000000 --- a/app/components/IssuesList.py +++ /dev/null @@ -1,29 +0,0 @@ -import streamlit as st -import pandas as pd -from app.components.data_processor import export_issues_to_df -from app.components.IssuesTable import render_issues_table -from typing import Optional - -def render_issues_list(df_in: Optional[pd.DataFrame] = None): - """ - Renders a tabbed interface showing active and resolved issues. - """ - tab1, tab2 = st.tabs(["Active Issues", "Resolved Issues"]) - - def render_tab_content(status, label): - if df_in is None: - df = export_issues_to_df(status=status) - else: - # Filter by status - df = df_in[df_in['status'] == status] - - if not df.empty: - render_issues_table(df) - else: - st.info(f"No {label.lower()}.") - - with tab1: - render_tab_content('open', 'Active Findings') - - with tab2: - render_tab_content('resolved', 'Resolved Findings') \ No newline at end of file diff --git a/app/components/IssuesTable.py b/app/components/IssuesTable.py deleted file mode 100644 index 76aae35..0000000 --- a/app/components/IssuesTable.py +++ /dev/null @@ -1,181 +0,0 @@ -import streamlit as st -import pandas as pd -from datetime import datetime, timedelta - -def get_due_date_status(due_date: pd.Timestamp): - """Return the status and style for a due date.""" - if not due_date: - return "", "" - - today: datetime.date = datetime.now().date() - due_date: datetime.date = pd.to_datetime(due_date).date() - - # Type safe comparison: - if due_date < today: - return "❗ ", "overdue" - elif due_date <= today + timedelta(days=7): - return "⚠️ ", "warning" - return "", "" - - -def style_dataframe(df): - """Apply conditional styling to the dataframe based on due dates.""" - today = datetime.now().date() - - def row_style(row): - if 'due_date' not in row: - return [''] * len(row) - - due_date = row['due_date'].date() if isinstance(row['due_date'], pd.Timestamp) else datetime.strptime(row['due_date'], '%Y-%m-%d 00:00:00').date() - - if due_date < today: - return ['background-color: #f8d7da'] * len(row) - elif due_date <= today + timedelta(days=7): - return ['background-color: #fff3cd'] * len(row) - return [''] * len(row) - - return df.style.apply(row_style, axis=1) - - -def get_issue_page_link(id: str): - """Link to the issue detail page with id as query parameter""" - return f'/issue_detail?id={id}' - - -def render_issues_table( - df: pd.DataFrame, - with_pagination: bool = False, - rows_per_page: int = 5, - page_key: str = "page", -): - """ - Render a table of security issues with consistent styling and optional pagination. - - Args: - df: DataFrame containing the issues data - with_pagination: Whether to enable pagination (defaults to False) - rows_per_page: Number of rows per page when pagination is enabled - page_key: Key to use for the pagination state in session state - """ - if df.empty: - st.info("No issues to display.") - return - - # Add status indicators if not already present - if 'status_icon' not in df.columns and 'due_date' in df.columns: - df['status_icon'], df['status'] = zip(*df['due_date'].apply(get_due_date_status)) - - # Sort by due_date and cvss if present - if 'due_date' in df.columns and 'cvss' in df.columns: - df = df.sort_values(['due_date', 'cvss'], ascending=[True, False]) - - # Add issue_link column - df['issue_link'] = df['id'].apply(get_issue_page_link) - - # Handle pagination if enabled - if with_pagination: - total_pages = len(df) // rows_per_page + (1 if len(df) % rows_per_page > 0 else 0) - page = st.session_state.get(page_key, 0) - start_idx = page * rows_per_page - end_idx = start_idx + rows_per_page - display_df = style_dataframe(df.iloc[start_idx:end_idx].copy()) - else: - display_df = style_dataframe(df) - - # Define column configuration - more compact version matching scan details - column_config = { - "status_icon": st.column_config.TextColumn( - "", - width="small" - ), - "issue_link": st.column_config.LinkColumn( - "Details", - help="Click to view issue details", - display_text="View", - width="small", - ), - "benchmark": st.column_config.TextColumn( - "Benchmark", - width="medium" - ), - "finding_id": st.column_config.TextColumn( - "Finding ID", - width="small" - ), - "level": st.column_config.TextColumn( - "Level", - width="small" - ), - "cvss": st.column_config.NumberColumn( - "CVSS", - format="%.1f", - width="small" - ), - "title": st.column_config.TextColumn( - "Title", - width="large" - ), - "remediation_count": st.column_config.NumberColumn( - "Remediations", - width="small" - ), - "created_at": st.column_config.DateColumn( - "Created", - width="small" - ), - "resolved_at": st.column_config.DateColumn( - "Resolved", - width="small" - ), - "due_date": st.column_config.DateColumn( - "Due Date", - format="YYYY-MM-DD", - width="small" - ), - "failure": st.column_config.TextColumn( - "Failure", - width="large" - ), - "first_seen": st.column_config.DateColumn( - "First Seen", - width="small" - ) - } - - # Only include columns that exist in the dataframe - filtered_config = {k: v for k, v in column_config.items() if k in df.columns} - - # Define default column order - matching scan details layout - default_columns = [ - "status_icon", - "issue_link", - "benchmark", "finding_id", - "level", "cvss", "title", "remediation_count", - "created_at", "resolved_at", "due_date", "failure", "first_seen" - ] - - # Filter column order to only include columns that exist in the dataframe - column_order = [col for col in default_columns if col in df.columns] - - # Display the dataframe with compact styling - st.dataframe( - display_df, - column_config=filtered_config, - hide_index=True, - column_order=column_order, - use_container_width=True - ) - - # Render pagination controls if enabled - if with_pagination: - col1, col2, col3 = st.columns([1, 2, 1]) - with col1: - if st.button("← Previous", disabled=(page == 0)): - st.session_state[page_key] = max(0, page - 1) - st.rerun() - with col2: - st.markdown(f"Page {page + 1} of {total_pages}") - with col3: - if st.button("Next →", disabled=(page >= total_pages - 1)): - st.session_state[page_key] = min(total_pages - 1, page + 1) - st.rerun() \ No newline at end of file diff --git a/app/components/__init__.py b/app/components/__init__.py deleted file mode 100644 index f9a0848..0000000 --- a/app/components/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -""" -Components package for data processing and utilities -""" \ No newline at end of file diff --git a/app/components/data_processor.py b/app/components/data_processor.py deleted file mode 100644 index dc24e10..0000000 --- a/app/components/data_processor.py +++ /dev/null @@ -1,374 +0,0 @@ -import pandas as pd -from datetime import datetime, timedelta -from app.database.schema import ( - get_db_connection, - get_or_create_benchmark, - get_or_create_scan, - insert_finding, - create_remediation, - link_finding_to_remediation, - mark_remediations_resolved_if_not_in_list, - mark_issues_as_resolved_if_no_open_remediations, - create_issue_for_remediations, - check_if_scan_date_is_newer_than_db_scans -) - -def get_cvss_range(cvss): - """Convert CVSS score to range category.""" - try: - score = float(cvss) - if score >= 9.0: - return "Critical" - elif score >= 7.0: - return "High" - elif score >= 4.0: - return "Medium" - elif score > 0: - return "Low" - else: - return "Info" - except (ValueError, TypeError): - if not cvss: - return "Info" - else: - return "Unknown" - - -def calculate_due_date(cvss, analysis_date): - """Calculate due date based on severity level.""" - severity_mapping = { - 'Critical': 15, - 'High': 30, - 'Medium': 90, - 'Low': 180, - 'Info': 180 - } - - time_to_resolve = severity_mapping.get(get_cvss_range(cvss), 180) - - return datetime.strptime(analysis_date, '%Y-%m-%d 00:00:00') + timedelta(days=time_to_resolve) - - - -def validate_columns(df, additional_columns=[]): - """Validate columns in the CSV file.""" - required_columns = [ - 'benchmark', 'id', 'level', 'cvss', 'title', - 'failures', 'description', 'rationale', 'refs' - ] - - # Validate columns - required_columns.extend(additional_columns) - if not all(col in df.columns for col in required_columns): - missing_cols = [col for col in required_columns if col not in df.columns] - raise ValueError(f"Missing required columns: {', '.join(missing_cols)}") - - - -def process_single_scan_upload(file, analysis_date): - """Process a single scan upload.""" - df = pd.read_csv(file) - return process_upload_dataframe(df, analysis_date) - - -def process_multiple_scan_upload(file): - """Process a multiple scan upload with multiple analysis dates.""" - df = pd.read_csv(file) - - column_name_map = { - 'benchmark': 'Benchmark', - 'id': 'CIS_ID', - 'level': 'Level', - 'cvss': 'CVSS', - 'title': 'Title', - 'failures': 'Failures', - 'description': 'Description', - 'rationale': 'Rationale', - 'refs': 'References' - } - - # Map from date to dataframe - results = { - 'new': 0, - 'existing': 0, - 'resolved': 0 - } - - sorted_dates = sorted(df['Date'].unique(), key=lambda x: datetime.strptime(x, '%m/%d/%Y')) - - for date in sorted_dates: - print(f"Processing date: {date}") - reformatted_date = datetime.strptime(date, '%m/%d/%Y').strftime('%Y-%m-%d 00:00:00') - - this_data = df[df['Date'] == date] - this_update = process_upload_dataframe(this_data, reformatted_date, column_name_map) - results['new'] += this_update['new'] - results['existing'] += this_update['existing'] - results['resolved'] += this_update['resolved'] - - - return results - - - -def process_upload_dataframe(df: pd.DataFrame, analysis_date, column_name_map=None) -> dict: - """Process a dataframe of uploaded CSV file and update database.""" - - conn = get_db_connection() - - if not check_if_scan_date_is_newer_than_db_scans(conn, datetime.strptime(analysis_date, '%Y-%m-%d 00:00:00')): - print(f"Scan on {analysis_date} predates the most recent scan; skipping") - return { - 'new': 0, - 'existing': 0, - 'resolved': 0 - } - - required_columns = [ - 'benchmark', 'id', 'level', 'cvss', 'title', - 'failures', 'description', 'rationale', 'refs' - ] - - if not column_name_map: - column_name_map = { x: x for x in required_columns } - - # Validate columns - if not all(column_name_map.get(col) in df.columns for col in required_columns): - missing_cols = [column_name_map.get(col) for col in required_columns if column_name_map.get(col) not in df.columns] - raise ValueError(f"Missing required columns: {', '.join(missing_cols)}") - - active_finding_ids = set() - new_count = 0 - existing_count = 0 - - try: - # Get or create scan record - scan_id = get_or_create_scan(conn, analysis_date) - - active_remediation_ids = set() - remediations_needing_issues = {} - - # Process each row - for _, row in df.iterrows(): - # Create benchmark record - benchmark_data = ( - row[column_name_map['benchmark']], - row[column_name_map['id']], # This is the finding_id from CSV - row[column_name_map['level']], - float(row[column_name_map['cvss']]) if row[column_name_map['cvss']] else 0.0, - row[column_name_map['title']], - row[column_name_map['description']], - row[column_name_map['rationale']], - row[column_name_map['refs']] - ) - benchmark_id = get_or_create_benchmark(conn, benchmark_data) - - # Handle multiple failures - failures = str(row[column_name_map['failures']]).split('\n') - for failure in failures: - failure = failure.strip() - if not failure: - continue - - # Insert finding - finding_id = insert_finding(conn, benchmark_id, failure) - active_finding_ids.add(finding_id) - - # Check if finding is already part of an open remediation - cursor = conn.cursor() - cursor.execute(''' - SELECT r.id - FROM remediations r - JOIN remediation_findings rf ON r.id = rf.remediation_id - WHERE r.benchmark_id = ? - AND r.state = 'open' - AND rf.finding_id = ? - ''', (benchmark_id, finding_id)) - - existing_remediation = cursor.fetchone() - - if existing_remediation: - active_remediation_ids.add(existing_remediation[0]) - existing_count += 1 - else: - # Create new remediation - due_date = calculate_due_date(row[column_name_map['cvss']], analysis_date) - remediation_id = create_remediation(conn, benchmark_id, scan_id, due_date) - link_finding_to_remediation(conn, remediation_id, finding_id) - - # Add this remediation to the list of active remediations: - active_remediation_ids.add(remediation_id) - - # Add this remediation to the list of remediations needing issues: - remediations_for_this_benchmark: list[int] = remediations_needing_issues.get((benchmark_id, due_date), []) - remediations_for_this_benchmark.append(remediation_id) - remediations_needing_issues[(benchmark_id, due_date)] = remediations_for_this_benchmark - new_count += 1 - - for (benchmark_id, due_date), remediation_ids in remediations_needing_issues.items(): - create_issue_for_remediations(conn, remediation_ids, benchmark_id, created_at=analysis_date, due_date=due_date) - - # Mark findings as resolved if they're not in current upload - mark_remediations_resolved_if_not_in_list(conn, scan_id, active_remediation_ids) - - # Mark issues as resolved which no longer have any open remediations - mark_issues_as_resolved_if_no_open_remediations(conn, analysis_date) - - # Get count of resolved findings in this scan - cursor = conn.cursor() - cursor.execute(''' - SELECT COUNT(*) as count - FROM remediations - WHERE resolved_in_scan = ? - ''', (scan_id,)) - resolved_count = cursor.fetchone()['count'] - - conn.commit() - return { - 'new': new_count, - 'existing': existing_count, - 'resolved': resolved_count - } - - finally: - conn.close() - -def get_findings_summary(): - """Get summary of findings for display.""" - conn = get_db_connection() - try: - cursor = conn.cursor() - today = datetime.now().date() - - # Get active findings count - cursor.execute(''' - SELECT COUNT(DISTINCT r.id) as count - FROM remediations r - WHERE r.state = 'open' - ''') - active_count = cursor.fetchone()['count'] - - # Get resolved findings count - cursor.execute(''' - SELECT COUNT(DISTINCT r.id) as count - FROM remediations r - WHERE r.state = 'resolved' - ''') - resolved_count = cursor.fetchone()['count'] - - # Get findings by severity - cursor.execute(''' - SELECT b.level, COUNT(DISTINCT r.id) as count - FROM remediations r - JOIN benchmark b ON r.benchmark_id = b.id - WHERE r.state = 'open' - GROUP BY b.level - ''') - severity_counts = {row['level']: row['count'] for row in cursor.fetchall()} - - # Get findings by CVSS - cursor.execute(''' - SELECT b.cvss - FROM remediations r - JOIN benchmark b ON r.benchmark_id = b.id - WHERE r.state = 'open' - ''') - cvss_scores = [row['cvss'] for row in cursor.fetchall()] - cvss_ranges = [get_cvss_range(score) for score in cvss_scores] - cvss_counts = pd.Series(cvss_ranges).value_counts().to_dict() - - # Get findings due within 28 days - today_str = today.strftime('%Y-%m-%d') - twenty_eight_days = (today + timedelta(days=28)).strftime('%Y-%m-%d') - cursor.execute(''' - SELECT COUNT(DISTINCT r.id) as count - FROM remediations r - WHERE r.state = 'open' - AND r.due_date <= ? - AND r.due_date > ? - ''', (twenty_eight_days, today_str)) - due_within_28_days = cursor.fetchone()['count'] - - # Get findings due this week - week_end = (today + timedelta(days=7)).strftime('%Y-%m-%d') - cursor.execute(''' - SELECT COUNT(DISTINCT r.id) as count - FROM remediations r - WHERE r.state = 'open' - AND r.due_date <= ? - AND r.due_date > ? - ''', (week_end, today_str)) - due_this_week = cursor.fetchone()['count'] - - # Get overdue findings - cursor.execute(''' - SELECT COUNT(DISTINCT r.id) as count - FROM remediations r - WHERE r.state = 'open' - AND r.due_date < ? - ''', (today_str,)) - overdue_count = cursor.fetchone()['count'] - - # Get findings due after 28 days - cursor.execute(''' - SELECT COUNT(DISTINCT r.id) as count - FROM remediations r - WHERE r.state = 'open' - AND r.due_date > ? - ''', (twenty_eight_days,)) - due_after_28_days = cursor.fetchone()['count'] - - return { - 'active_count': active_count, - 'resolved_count': resolved_count, - 'severity_counts': severity_counts, - 'cvss_counts': cvss_counts, - 'due_within_28_days': due_within_28_days, - 'due_this_week': due_this_week, - 'overdue_count': overdue_count, - 'due_after_28_days': due_after_28_days - } - - finally: - conn.close() - -def export_issues_to_df(status='open'): - """Export issues to a DataFrame.""" - conn = get_db_connection() - cursor = conn.cursor() - - - sql = ''' - SELECT - i.id, - i.due_date, - i.created_at, - i.resolved_at, - i.status, - b.benchmark, - b.finding_id, - b.level, - b.cvss, - b.title, - b.description, - b.rationale, - b.refs - FROM issues i - JOIN benchmark b ON i.benchmark_id = b.id - WHERE i.status = ? - ''' - - cursor.execute(sql, (status,)) - - # Convert list of dictionaries to DataFrame - issues = [dict(row) for row in cursor.fetchall()] - - df = pd.DataFrame.from_records(issues) - - # Convert date strings to datetime objects - date_columns = ['due_date', 'created_at', 'resolved_at'] - for col in date_columns: - if col in df.columns: - df[col] = pd.to_datetime(df[col]) - - return df diff --git a/app/components/poam_exports.py b/app/components/poam_exports.py deleted file mode 100644 index bcb3b9d..0000000 --- a/app/components/poam_exports.py +++ /dev/null @@ -1,118 +0,0 @@ -import pandas as pd -from datetime import datetime -from app.database.schema import get_db_connection, get_poam_config -from app.components.data_processor import get_cvss_range - -def get_poam_id(created_at, issue_id): - """Generate a POAM ID in the format YYYY-CISxxxx.""" - year = datetime.strptime(created_at, '%Y-%m-%d %H:%M:%S').year - return f"{year}-CIS{issue_id:04d}" - -def get_weakness_detector_source(scan_date): - """Generate the weakness detector source string.""" - return f"Hail CIS GCP/GKE Compliance (rolling). Scan on {scan_date}" - -def get_asset_identifier(google_project, findings): - """Generate the asset identifier string.""" - return f"[{google_project}]: {findings}" - -def get_planned_milestones(due_date): - """Generate the planned milestones string.""" - return f"(1) {due_date}: Perform required updates to affected assets to remediate this finding." - -def get_issue_findings(conn, issue_id): - """Get all findings associated with an issue.""" - cursor = conn.cursor() - cursor.execute(''' - SELECT DISTINCT f.failure - FROM findings f - JOIN remediation_findings rf ON f.id = rf.finding_id - JOIN issue_remediations ir ON rf.remediation_id = ir.remediation_id - WHERE ir.issue_id = ? - ''', (issue_id,)) - return [row['failure'] for row in cursor.fetchall()] - -def get_first_scan_date(conn, issue_id): - """Get the first scan date for an issue.""" - cursor = conn.cursor() - cursor.execute(''' - SELECT MIN(s.scan_date) as first_scan_date - FROM scans s - JOIN remediations r ON s.id = r.first_seen_scan - JOIN issue_remediations ir ON r.id = ir.remediation_id - WHERE ir.issue_id = ? - ''', (issue_id,)) - result = cursor.fetchone() - return result['first_scan_date'] if result else None - -def generate_poam_export(status='open'): - """Generate a POAM export for issues with the specified status.""" - conn = get_db_connection() - config = get_poam_config(conn) - - if not config: - raise ValueError("POAM configuration not found. Please configure POAM export settings first.") - - # Get issues with the specified status - cursor = conn.cursor() - cursor.execute(''' - SELECT - i.id, - i.created_at, - i.due_date, - i.status, - b.title, - b.description, - b.cvss - FROM issues i - JOIN benchmark b ON i.benchmark_id = b.id - WHERE i.status = ? - ORDER BY i.created_at ASC - ''', (status,)) - - issues = cursor.fetchall() - - # Prepare data for export - export_data = [] - for issue in issues: - findings = get_issue_findings(conn, issue['id']) - findings_str = "\n".join(findings) - first_scan_date = get_first_scan_date(conn, issue['id']) - - export_data.append({ - 'POAM ID': get_poam_id(issue['created_at'], issue['id']), - 'Controls': 'CM-6', - 'Weakness Name': issue['title'], - 'Weakness Description': issue['description'], - 'Weakness Detector Source': get_weakness_detector_source(first_scan_date), - 'Weakness Source Identifier': 'CIS', - 'Asset Identifier': get_asset_identifier(config['google_project'], findings_str), - 'Point of Contact': config['point_of_contact'], - 'Resources Required': 'None', - 'Overall Remediation Plan': 'Perform necessary updates to resolve the vulnerability', - 'Original Detection Date': issue['created_at'], - 'Scheduled Completion Date': issue['due_date'], - 'AGENCY Scheduled Completion Date': issue['due_date'], - 'Planned Milestones': get_planned_milestones(issue['due_date']), - 'Milestone Changes': '', - 'Status Date': datetime.now().strftime('%Y-%m-%d'), - 'Vendor Dependency': 'No', - 'Last Vendor Check-in date': '', - 'Vendor Dependent Product Name': '', - 'Original Risk Rating': get_cvss_range(issue['cvss']), - 'Adjusted Risk Rating': 'N/A', - 'Risk Adjustment': 'No', - 'False Positive': 'No', - 'Operational Requirement': 'No', - 'Deviation Rationale': '', - 'Supporting Documents': '', - 'Comments': '', - 'Auto-Approve': 'No', - 'Binding Operational Directive 22-01 tracking': 'No', - 'Binding Operational Directive 22-01 Due Date': '', - 'CVE': '', - 'Service Name': config['service_name'] - }) - - conn.close() - return pd.DataFrame(export_data) \ No newline at end of file diff --git a/app/database/__init__.py b/app/database/__init__.py deleted file mode 100644 index fed03e4..0000000 --- a/app/database/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -""" -Database package for SQLite operations and schema -""" \ No newline at end of file diff --git a/app/database/schema.py b/app/database/schema.py deleted file mode 100644 index 859925b..0000000 --- a/app/database/schema.py +++ /dev/null @@ -1,342 +0,0 @@ -import sqlite3 -import os -from datetime import datetime, timedelta - -DB_PATH = "app/database/findings.db" - -def get_db_connection(): - """Create a database connection.""" - os.makedirs(os.path.dirname(DB_PATH), exist_ok=True) - conn = sqlite3.connect(DB_PATH) - conn.row_factory = sqlite3.Row - return conn - -def init_db(): - """Initialize the database with required tables.""" - conn = get_db_connection() - cursor = conn.cursor() - - # Create benchmark table - cursor.execute(''' - CREATE TABLE IF NOT EXISTS benchmark ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - benchmark TEXT NOT NULL, - finding_id TEXT NOT NULL, - level TEXT, - cvss REAL, - title TEXT, - description TEXT, - rationale TEXT, - refs TEXT, - UNIQUE(benchmark, finding_id) - ) - ''') - - # Create findings table - cursor.execute(''' - CREATE TABLE IF NOT EXISTS findings ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - benchmark_id INTEGER NOT NULL, - failure TEXT NOT NULL, - FOREIGN KEY (benchmark_id) REFERENCES benchmark(id), - UNIQUE(benchmark_id, failure) - ) - ''') - - # Create scans table - cursor.execute(''' - CREATE TABLE IF NOT EXISTS scans ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - scan_date DATE NOT NULL UNIQUE - ) - ''') - - # Create remediations table - cursor.execute(''' - CREATE TABLE IF NOT EXISTS remediations ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - benchmark_id INTEGER NOT NULL, - first_seen_scan INTEGER NOT NULL, - resolved_in_scan INTEGER, - state TEXT NOT NULL CHECK (state IN ('open', 'resolved')), - due_date DATE NOT NULL, - FOREIGN KEY (benchmark_id) REFERENCES benchmark(id), - FOREIGN KEY (first_seen_scan) REFERENCES scans(id), - FOREIGN KEY (resolved_in_scan) REFERENCES scans(id) - ) - ''') - - # Create remediation_findings table - cursor.execute(''' - CREATE TABLE IF NOT EXISTS remediation_findings ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - remediation_id INTEGER NOT NULL, - finding_id INTEGER NOT NULL, - FOREIGN KEY (remediation_id) REFERENCES remediations(id), - FOREIGN KEY (finding_id) REFERENCES findings(id), - UNIQUE (remediation_id, finding_id) - ) - ''') - - # Create issues table - cursor.execute(''' - CREATE TABLE IF NOT EXISTS issues ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - benchmark_id INTEGER NOT NULL, - due_date DATE NOT NULL, - created_at TIMESTAMP NOT NULL, - status TEXT NOT NULL CHECK (status IN ('open', 'resolved')), - resolved_at TIMESTAMP, - FOREIGN KEY (benchmark_id) REFERENCES benchmark(id) - ) - ''') - - # Create issue_remediations table - cursor.execute(''' - CREATE TABLE IF NOT EXISTS issue_remediations ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - issue_id INTEGER NOT NULL, - remediation_id INTEGER NOT NULL, - FOREIGN KEY (issue_id) REFERENCES issues(id), - FOREIGN KEY (remediation_id) REFERENCES remediations(id), - UNIQUE (issue_id, remediation_id) - ) - ''') - - # Create poam_config table - cursor.execute(''' - CREATE TABLE IF NOT EXISTS poam_config ( - id INTEGER PRIMARY KEY, - point_of_contact TEXT NOT NULL, - google_project TEXT NOT NULL, - service_name TEXT NOT NULL - ) - ''') - - conn.commit() - conn.close() - -def get_or_create_benchmark(conn, benchmark_data): - """Get existing benchmark or create a new one.""" - cursor = conn.cursor() - cursor.execute(''' - INSERT OR IGNORE INTO benchmark ( - benchmark, finding_id, level, cvss, title, - description, rationale, refs - ) VALUES (?, ?, ?, ?, ?, ?, ?, ?) - ''', benchmark_data) - - cursor.execute(''' - SELECT id FROM benchmark - WHERE benchmark = ? AND finding_id = ? - ''', (benchmark_data[0], benchmark_data[1])) - - return cursor.fetchone()['id'] - -def get_or_create_scan(conn, scan_date): - """Get existing scan or create a new one.""" - cursor = conn.cursor() - cursor.execute(''' - INSERT OR IGNORE INTO scans (scan_date) - VALUES (?) - ''', (scan_date,)) - - cursor.execute(''' - SELECT id FROM scans - WHERE scan_date = ? - ''', (scan_date,)) - - return cursor.fetchone()['id'] - -def insert_finding(conn, benchmark_id, failure): - """Insert a new finding into the database.""" - cursor = conn.cursor() - cursor.execute(''' - INSERT OR IGNORE INTO findings (benchmark_id, failure) - VALUES (?, ?) - ''', (benchmark_id, failure)) - - cursor.execute(''' - SELECT id FROM findings - WHERE benchmark_id = ? AND failure = ? - ''', (benchmark_id, failure)) - - return cursor.fetchone()['id'] - -def create_remediation(conn, benchmark_id, scan_id, due_date): - """Create a new remediation.""" - cursor = conn.cursor() - cursor.execute(''' - INSERT INTO remediations ( - benchmark_id, first_seen_scan, state, due_date - ) VALUES (?, ?, 'open', ?) - ''', (benchmark_id, scan_id, due_date)) - - return cursor.lastrowid - -def link_finding_to_remediation(conn, remediation_id, finding_id): - """Link a finding to a remediation.""" - cursor = conn.cursor() - cursor.execute(''' - INSERT OR IGNORE INTO remediation_findings (remediation_id, finding_id) - VALUES (?, ?) - ''', (remediation_id, finding_id)) - -def check_if_scan_exists(conn, scan_date): - """Check if a scan exists for a given date.""" - cursor = conn.cursor() - cursor.execute(''' - SELECT id FROM scans WHERE scan_date = ? - ''', (scan_date,)) - return cursor.fetchone() is not None - -def check_if_scan_date_is_newer_than_db_scans(conn, scan_date): - """Check if a scan is newer than the most recent scan.""" - cursor = conn.cursor() - cursor.execute(''' - SELECT max(scan_date) FROM scans - ''') - - result = cursor.fetchone() - - if result is None or result[0] is None or result[0] == '': - print(f"** No scans found; assuming {scan_date} is the most recent scan") - return True - else: - db_scan_date_object = datetime.strptime(result[0], '%Y-%m-%d 00:00:00') - if db_scan_date_object.date() < scan_date.date(): - print(f"** Scan on {scan_date} is newer than the most recent scan ({result[0]}); processing") - return True - else: - print(f"** Scan on {scan_date} is older than the most recent scan ({result[0]}); skipping") - return False - - -def get_active_issues(conn): - """Get all active issues.""" - cursor = conn.cursor() - cursor.execute(''' - SELECT - b.benchmark, - b.finding_id, - b.level, - b.cvss, - b.title, - b.description, - b.rationale, - b.refs, - i.due_date, - i.created_at, - i.resolved_at, - i.status, - COUNT(ir.remediation_id) as remediation_count - FROM issues i - JOIN issue_remediations ir ON i.id = ir.issue_id - JOIN benchmark b ON i.benchmark_id = b.id - WHERE i.status = 'open' - GROUP BY i.id - ''') - - # Convert rows to dictionaries with column names - columns = [column[0] for column in cursor.description] - return [dict(zip(columns, row)) for row in cursor.fetchall()] - - -def mark_remediations_resolved_if_not_in_list(conn, scan_id, active_remediation_ids): - """Mark remediations as resolved if they're not in the current scan.""" - - print(f"Marking all but {len(active_remediation_ids)} still-active remediations as resolved") - - if not active_remediation_ids: - # No active remediations - update all open remediations to resolved - cursor = conn.cursor() - cursor.execute(''' - UPDATE remediations - SET state = 'resolved' - WHERE state = 'open' - ''') - else: - cursor = conn.cursor() - placeholders = ','.join(['?' for _ in active_remediation_ids]) - cursor.execute(f''' - UPDATE remediations - SET state = 'resolved', - resolved_in_scan = ? - WHERE state = 'open' - AND id NOT IN ({placeholders}) - ''', [scan_id, *active_remediation_ids]) - - conn.commit() - - -def mark_issues_as_resolved_if_no_open_remediations(conn, resolved_date): - """Mark issues as resolved if they no longer have any open remediations.""" - cursor = conn.cursor() - cursor.execute(''' - SELECT i.id - FROM issues i - WHERE i.status = 'open' - AND NOT EXISTS ( - SELECT 1 - FROM remediations r - JOIN issue_remediations ir ON r.id = ir.remediation_id - WHERE r.state = 'open' - AND ir.issue_id = i.id - ) - ''', ()) - - issue_ids = [row[0] for row in cursor.fetchall()] - print(f"Marking {len(issue_ids)} issues as resolved ({issue_ids})") - placeholders = ','.join(['?' for _ in issue_ids]) - - if issue_ids: - cursor.execute(f''' - UPDATE issues - SET status = 'resolved', resolved_at = ? - WHERE id IN ({placeholders}) - ''', (resolved_date, *issue_ids)) - - conn.commit() - - -def create_issue_for_remediations(conn, remediation_ids, benchmark_id, created_at, due_date): - """Create a new issue for a remediation if one doesn't exist.""" - cursor = conn.cursor() - cursor.execute(''' - INSERT INTO issues (benchmark_id, due_date, created_at, status) - VALUES (?, ?, ?, 'open') - ''', (benchmark_id, due_date, created_at)) - - issue_id = cursor.lastrowid - - for remediation_id in remediation_ids: - cursor.execute(''' - INSERT INTO issue_remediations (issue_id, remediation_id) - VALUES (?, ?) - ''', (issue_id, remediation_id)) - - conn.commit() - -def get_poam_config(conn): - """Get the POAM export configuration.""" - cursor = conn.cursor() - cursor.execute('SELECT * FROM poam_config LIMIT 1') - result = cursor.fetchone() - if result: - return dict(result) - return None - -def update_poam_config(conn, point_of_contact, google_project, service_name): - """Update the POAM export configuration.""" - cursor = conn.cursor() - cursor.execute(''' - INSERT OR REPLACE INTO poam_config ( - id, point_of_contact, google_project, service_name - ) VALUES ( - 1, ?, ?, ? - ) - ''', (point_of_contact, google_project, service_name)) - conn.commit() - -# Initialize the database when the module is imported -init_db() \ No newline at end of file diff --git a/app/main.py b/app/main.py deleted file mode 100644 index a319c22..0000000 --- a/app/main.py +++ /dev/null @@ -1,204 +0,0 @@ -import streamlit as st -import pandas as pd -from datetime import datetime, timedelta -import plotly.express as px -from app.components.data_processor import ( - process_single_scan_upload, - process_multiple_scan_upload, - get_findings_summary -) - -from app.components.IssuesList import render_issues_list -import os -import logging -# Set page config -st.set_page_config( - page_title="Security Findings Tracker", - page_icon="🔒", - layout="wide" -) - -# Initialize session state variables -if 'open_modal' not in st.session_state: - st.session_state.open_modal = False -if 'show_success' not in st.session_state: - st.session_state.show_success = False -if 'upload_results' not in st.session_state: - st.session_state.upload_results = {} - -# Load custom CSS -with open(os.path.join(os.path.dirname(__file__), "static/style.css")) as f: - st.markdown(f"", unsafe_allow_html=True) - -def get_due_date_status(due_date: pd.Timestamp): - """Return the status and style for a due date.""" - if not due_date: - return "", "" - - today: datetime.date = datetime.now().date() - due_date: datetime.date = pd.to_datetime(due_date).date() - - # Type safe comparison: - if due_date < today: - return "❗ ", "overdue" - elif due_date <= today + timedelta(days=7): - return "⚠️ ", "warning" - return "", "" - - -# Header with Upload Button -col1, col2 = st.columns([3, 1]) -with col1: - st.markdown(""" -
-

Security Findings Tracker

-
- """, unsafe_allow_html=True) -with col2: - if st.button("Upload Findings", type="primary", use_container_width=True): - st.session_state.open_modal = True - -# Upload Modal Dialog -@st.dialog("Upload Security Findings") -def show_upload_dialog(): - # Show a dropdown to select the type of upload - upload_type = st.selectbox("Upload Type", ["Single Scan", "Multiple Scans"]) - st.subheader("Upload Single Scan") - uploaded_file = st.file_uploader("Select CSV File", type="csv", key="file_uploader") - - if upload_type == "Single Scan": - analysis_date = st.date_input( - "Analysis Date", - value=datetime.now().date(), - key="analysis_date", - help="Date when the security analysis was performed" - ) - - col1, col2 = st.columns([1, 1]) - with col1: - if st.button("Cancel", type="secondary"): - st.session_state.open_modal = False - st.rerun() - with col2: - if uploaded_file is not None: - if st.button("Confirm Upload", type="primary"): - if upload_type == "Single Scan": - results = process_single_scan_upload(uploaded_file, analysis_date.strftime('%Y-%m-%d 00:00:00')) - elif upload_type == "Multiple Scans": - results = process_multiple_scan_upload(uploaded_file) - st.session_state.open_modal = False - st.session_state.show_success = True - st.session_state.upload_results = results - st.rerun() - -# Call the dialog function -if st.session_state.open_modal: - show_upload_dialog() - -# Show success message after upload -if st.session_state.pop("show_success", False): - results = st.session_state.pop("upload_results", {}) - st.success("File processed successfully!") - st.markdown("### Upload Summary") - st.markdown(f""" - - New Findings: {results['new']} - - Updated Findings: {results['existing']} - - Resolved Findings: {results['resolved']} - """) - -# Main Content -summary = get_findings_summary() - -# Charts Section -col1, col2, col3, col4 = st.columns(4) - -with col1: - with st.container(border=True): - # Active Findings - st.markdown(f""" -
-
{summary['active_count']}
-
Active Findings
-
- """, unsafe_allow_html=True) - - # Due This Week - due_this_week = summary.get('due_this_week', 0) - st.markdown(f""" -
-
{'⚠️ ' if due_this_week > 0 else ''}{due_this_week}
-
Due This Week
-
- """, unsafe_allow_html=True) - - # Overdue Findings - overdue_count = summary.get('overdue_count', 0) - st.markdown(f""" -
-
{'❗ ' if overdue_count > 0 else ''}{overdue_count}
-
Overdue Findings
-
- """, unsafe_allow_html=True) - -with col2: - # Level distribution chart - level_df = pd.DataFrame([ - {'Level': k, 'Count': v} - for k, v in summary['severity_counts'].items() - ]) - - if not level_df.empty: - fig1 = px.pie( - level_df, - values='Count', - names='Level', - title='Active Findings by Level', - color='Level', - color_discrete_map={ - 'critical': '#dc3545', - 'high': '#fd7e14', - 'medium': '#ffc107', - 'low': '#20c997', - 'info': '#0dcaf0' - } - ) - fig1.update_traces(textposition='inside', textinfo='percent+label') - st.plotly_chart(fig1, use_container_width=True) - -with col3: - # CVSS distribution chart - if 'cvss_counts' in summary: - cvss_df = pd.DataFrame([ - {'CVSS Range': k, 'Count': v} - for k, v in summary['cvss_counts'].items() - ]) - - if not cvss_df.empty: - fig2 = px.pie( - cvss_df, - values='Count', - names='CVSS Range', - title='Active Findings by CVSS', - color='CVSS Range', - color_discrete_sequence=px.colors.sequential.RdBu - ) - fig2.update_traces(textposition='inside', textinfo='percent+label') - st.plotly_chart(fig2, use_container_width=True) - -with col4: - # Due by date chart - overdue_count = summary.get('overdue_count', 0) - due_this_week = summary.get('due_this_week', 0) - due_within_28_days = summary.get('due_within_28_days', 0) - due_after_28_days = summary.get('due_after_28_days', 0) - - fig3 = px.pie( - values=[overdue_count, due_this_week, due_within_28_days, due_after_28_days], - names=['Overdue', 'Due This Week', 'Due Within 28 Days', 'Due After 28 Days'], - title='Active Findings by Due Date', - color_discrete_sequence=px.colors.sequential.RdBu - ) - st.plotly_chart(fig3, use_container_width=True) - -# Issues List with Tabs -render_issues_list() diff --git a/app/pages/1_issue_detail.py b/app/pages/1_issue_detail.py deleted file mode 100644 index e9ee799..0000000 --- a/app/pages/1_issue_detail.py +++ /dev/null @@ -1,170 +0,0 @@ -import streamlit as st -import pandas as pd -from datetime import datetime -import sqlite3 -from app.database.schema import get_db_connection -from app.components.IssuesList import render_issues_list - -st.set_page_config( - page_title="Issue Details", - page_icon="🔒", - layout="wide" -) - -def get_issue_details(issue_id): - """Get detailed information about a specific issue.""" - conn = get_db_connection() - cursor = conn.cursor() - - # Get issue and benchmark details - cursor.execute(''' - SELECT - i.id, - i.due_date, - i.created_at, - i.resolved_at, - i.status, - b.benchmark, - b.finding_id, - b.level, - b.cvss, - b.title, - b.description, - b.rationale, - b.refs - FROM issues i - JOIN benchmark b ON i.benchmark_id = b.id - WHERE i.id = ? - ''', (issue_id,)) - - issue = cursor.fetchone() - if not issue: - return None, None - - # Get all remediations for this issue - cursor.execute(''' - SELECT - r.id, - r.state, - r.due_date, - s_first.scan_date as first_seen, - s_resolved.scan_date as resolved_date, - f.failure, - b.benchmark, - b.finding_id - FROM remediations r - JOIN issue_remediations ir ON r.id = ir.remediation_id - JOIN remediation_findings rf ON r.id = rf.remediation_id - JOIN findings f ON rf.finding_id = f.id - JOIN benchmark b ON r.benchmark_id = b.id - LEFT JOIN scans s_first ON r.first_seen_scan = s_first.id - LEFT JOIN scans s_resolved ON r.resolved_in_scan = s_resolved.id - WHERE ir.issue_id = ? - ORDER BY r.due_date ASC - ''', (issue_id,)) - - remediations = cursor.fetchall() - conn.close() - - return issue, remediations - -# Get issue ID from URL parameters -issue_id = st.query_params.get("id", None) - -# Basic breadcrumbs: -if issue_id: - st.markdown(f""" -
- Issues - > - {issue_id} -
-""", unsafe_allow_html=True) - - - -if not issue_id: - st.title("Security Issues") - render_issues_list() -else: - issue, remediations = get_issue_details(issue_id) - - if not issue: - st.error(f"Issue {issue_id} not found.") - st.stop() - - # Display issue details - st.title(f"{issue['title']}") - - # Issue metadata - col1, col2, col3 = st.columns(3) - with col1: - st.metric("CVSS Score", f"{issue['cvss']:.1f}") - with col2: - st.metric("Level", issue['level']) - with col3: - st.metric("Status", issue['status'].upper()) - - # Dates and timeline - st.divider() - date_col1, date_col2, date_col3 = st.columns(3) - with date_col1: - st.markdown(f"**Created:** {issue['created_at']}") - with date_col2: - st.markdown(f"**Due Date:** {issue['due_date']}") - with date_col3: - if issue['resolved_at']: - st.markdown(f"**Resolved:** {issue['resolved_at']}") - - # Remediations table - st.divider() - st.subheader("Remediations") - - if remediations: - remediation_data = [] - for r in remediations: - remediation_data.append({ - 'ID': r['id'], - 'State': r['state'].upper(), - 'Due Date': r['due_date'], - 'First Seen': r['first_seen'], - 'Resolved Date': r['resolved_date'] or '', - 'Finding ID': r['finding_id'], - 'Failure': r['failure'] - }) - - df = pd.DataFrame(remediation_data) - - st.dataframe( - df, - column_config={ - 'ID': st.column_config.NumberColumn('ID', width='small'), - 'State': st.column_config.TextColumn('State', width='small'), - 'Due Date': st.column_config.DateColumn('Due Date', width='small'), - 'First Seen': st.column_config.DateColumn('First Seen', width='small'), - 'Resolved Date': st.column_config.DateColumn('Resolved Date', width='small'), - 'Finding ID': st.column_config.TextColumn('Finding ID', width='medium'), - 'Failure': st.column_config.TextColumn('Failure', width='large') - }, - hide_index=True, - use_container_width=True - ) - else: - st.info("No remediations found for this issue.") - - # Description and rationale - st.divider() - st.subheader("Description") - st.write(issue['description']) - - if issue['rationale']: - st.subheader("Rationale") - st.write(issue['rationale']) - - # References - if issue['refs']: - st.divider() - st.subheader("References") - for ref in issue['refs'].split('\n'): - if ref.strip(): - st.markdown(f"- [{ref}]({ref})") diff --git a/app/pages/2_scan_history.py b/app/pages/2_scan_history.py deleted file mode 100644 index 161bcae..0000000 --- a/app/pages/2_scan_history.py +++ /dev/null @@ -1,176 +0,0 @@ -import streamlit as st -import pandas as pd -from app.database.schema import get_db_connection -from app.components.IssuesTable import render_issues_table -st.set_page_config( - page_title="Scan History", - page_icon="🔒", - layout="wide" -) - -def get_all_scans(): - """Get list of all scans ordered by date.""" - conn = get_db_connection() - cursor = conn.cursor() - cursor.execute(''' - SELECT id, scan_date - FROM scans - ORDER BY scan_date DESC - ''') - return [dict(row) for row in cursor.fetchall()] - -def get_scan_details(scan_id): - """Get detailed information about a specific scan.""" - conn = get_db_connection() - cursor = conn.cursor() - - # Get basic scan info - cursor.execute(''' - SELECT id, scan_date - FROM scans - WHERE id = ? - ''', (scan_id,)) - scan = cursor.fetchone() - - if not scan: - return None, None, None, None - - # Get new issues from this scan - cursor.execute(''' - SELECT - i.id, - i.due_date, - i.created_at, - b.benchmark, - b.finding_id, - b.level, - b.cvss, - b.title, - COUNT(r.id) as remediation_count - FROM issues i - JOIN benchmark b ON i.benchmark_id = b.id - JOIN issue_remediations ir ON i.id = ir.issue_id - JOIN remediations r ON ir.remediation_id = r.id - WHERE DATE(i.created_at) = DATE(?) - GROUP BY i.id - ORDER BY b.cvss DESC - ''', (scan['scan_date'],)) - new_issues = [dict(row) for row in cursor.fetchall()] - - # Get issues closed in this scan - cursor.execute(''' - SELECT - i.id, - i.due_date, - i.created_at, - i.resolved_at, - b.benchmark, - b.finding_id, - b.level, - b.cvss, - b.title, - COUNT(r.id) as remediation_count - FROM issues i - JOIN benchmark b ON i.benchmark_id = b.id - JOIN issue_remediations ir ON i.id = ir.issue_id - JOIN remediations r ON ir.remediation_id = r.id - WHERE DATE(i.resolved_at) = DATE(?) - GROUP BY i.id - ORDER BY b.cvss DESC - ''', (scan['scan_date'],)) - closed_issues = [dict(row) for row in cursor.fetchall()] - - # Get issues with remediations which were resolved in this scan - cursor.execute(''' - SELECT - i.id, - i.due_date, - i.created_at, - i.resolved_at, - b.benchmark, - b.finding_id, - b.level, - b.cvss, - b.title, - COUNT(DISTINCT r.id) as remediation_count - FROM issues i - JOIN benchmark b ON i.benchmark_id = b.id - JOIN issue_remediations ir ON i.id = ir.issue_id - JOIN remediations r ON ir.remediation_id = r.id - WHERE EXISTS ( - SELECT 1 - FROM issues i2 - JOIN issue_remediations ir2 ON i2.id = ir2.issue_id - JOIN remediations r2 ON ir2.remediation_id = r2.id - WHERE i2.id = i.id - AND r2.resolved_in_scan = ? - ) - GROUP BY i.id - ORDER BY b.cvss DESC - ''', (scan_id,)) - issues_with_resolved_findings = [dict(row) for row in cursor.fetchall()] - - conn.close() - return scan, new_issues, closed_issues, issues_with_resolved_findings - -# Get list of all scans -scans = get_all_scans() - -if not scans: - st.error("No scans found in the database.") - st.stop() - -# Scan selector -selected_scan = st.selectbox( - "Select Scan", - options=scans, - format_func=lambda x: x['scan_date'], - key="scan_selector" -) - -if selected_scan: - scan, new_issues, closed_issues, issues_with_resolved_findings = get_scan_details(selected_scan['id']) - - if not scan: - st.error(f"Scan {selected_scan['id']} not found.") - st.stop() - - # Display scan summary - st.title(f"Scan Details: {scan['scan_date']}") - - # Summary metrics - col1, col2, col3 = st.columns(3) - with col1: - st.metric("New Issues", len(new_issues)) - with col2: - st.metric("Closed Issues", len(closed_issues)) - with col3: - st.metric("Issues with Resolved Findings", len(issues_with_resolved_findings)) - - # New Issues - st.divider() - st.subheader("New Issues") - if new_issues: - new_issues_df = pd.DataFrame(new_issues) - new_issues_df['issue_link'] = [f"/issue_detail?id={issue['id']}" for issue in new_issues] - render_issues_table(new_issues_df) - else: - st.info("No new issues in this scan.") - - # Closed Issues - st.divider() - st.subheader("Closed Issues") - if closed_issues: - closed_issues_df = pd.DataFrame(closed_issues) - render_issues_table(closed_issues_df) - else: - st.info("No issues were closed in this scan.") - - # Resolved Findings - st.divider() - st.subheader("Issues with Resolved Findings") - if issues_with_resolved_findings: - issues_with_resolved_findings_df = pd.DataFrame(issues_with_resolved_findings) - render_issues_table(issues_with_resolved_findings_df) - else: - st.info("No findings were resolved in this scan.") \ No newline at end of file diff --git a/app/pages/3_benchmark_detail.py b/app/pages/3_benchmark_detail.py deleted file mode 100644 index e453018..0000000 --- a/app/pages/3_benchmark_detail.py +++ /dev/null @@ -1,237 +0,0 @@ -import streamlit as st -import pandas as pd -from app.database.schema import get_db_connection -from app.components.IssuesList import render_issues_list - -st.set_page_config( - page_title="Benchmark Details", - page_icon="🔒", - layout="wide" -) - -def get_all_benchmarks(): - """Get a list of all benchmarks with their stats.""" - conn = get_db_connection() - cursor = conn.cursor() - - cursor.execute(''' - SELECT - b.id, - b.benchmark, - b.finding_id, - b.level, - b.cvss, - b.title, - COUNT(DISTINCT i.id) as total_issues, - SUM(CASE WHEN i.status = 'open' THEN 1 ELSE 0 END) as open_issues, - COUNT(DISTINCT r.id) as total_remediations, - SUM(CASE WHEN r.state = 'open' THEN 1 ELSE 0 END) as open_remediations - FROM benchmark b - LEFT JOIN issues i ON b.id = i.benchmark_id - LEFT JOIN issue_remediations ir ON i.id = ir.issue_id - LEFT JOIN remediations r ON ir.remediation_id = r.id - GROUP BY b.id - ORDER BY b.cvss DESC - ''') - - columns = [column[0] for column in cursor.description] - rows = [dict(zip(columns, row)) for row in cursor.fetchall()] - conn.close() - - return pd.DataFrame(rows) if rows else pd.DataFrame() - -def get_benchmark_details(benchmark_id): - """Get detailed information about a specific benchmark.""" - conn = get_db_connection() - cursor = conn.cursor() - - # Get benchmark details - cursor.execute(''' - SELECT - b.id, - b.benchmark, - b.finding_id, - b.level, - b.cvss, - b.title, - b.description, - b.rationale, - b.refs - FROM benchmark b - WHERE b.id = ? - ''', (benchmark_id,)) - - benchmark = cursor.fetchone() - if not benchmark: - return None, None, None - - # Get all issues for this benchmark - cursor.execute(''' - SELECT - i.id, - i.due_date, - i.created_at, - i.resolved_at, - i.status, - b.benchmark, - b.finding_id, - b.level, - b.cvss, - b.title, - COUNT(r.id) as remediation_count - FROM issues i - JOIN benchmark b ON i.benchmark_id = b.id - JOIN issue_remediations ir ON i.id = ir.issue_id - JOIN remediations r ON ir.remediation_id = r.id - WHERE b.id = ? - GROUP BY i.id - ORDER BY i.created_at DESC - ''', (benchmark_id,)) - - issues = [dict(zip([column[0] for column in cursor.description], row)) - for row in cursor.fetchall()] - - # Get all open remediations for this benchmark - cursor.execute(''' - SELECT - r.id, - r.state, - r.due_date, - s_first.scan_date as first_seen, - s_resolved.scan_date as resolved_date, - f.failure, - i.id as issue_id - FROM remediations r - JOIN issue_remediations ir ON r.id = ir.remediation_id - JOIN issues i ON ir.issue_id = i.id - JOIN remediation_findings rf ON r.id = rf.remediation_id - JOIN findings f ON rf.finding_id = f.id - LEFT JOIN scans s_first ON r.first_seen_scan = s_first.id - LEFT JOIN scans s_resolved ON r.resolved_in_scan = s_resolved.id - WHERE r.benchmark_id = ? AND r.state = 'open' - ORDER BY r.due_date ASC - ''', (benchmark_id,)) - - remediations = [dict(zip([column[0] for column in cursor.description], row)) - for row in cursor.fetchall()] - - conn.close() - return benchmark, issues, remediations - -def get_issue_page_link(id: str): - """Link to the issue detail page with id as query parameter""" - return f'/issue_detail?id={id}' - -def get_benchmark_page_link(id: str): - """Link to the benchmark detail page with id as query parameter""" - return f'/benchmark_detail?id={id}' - -# Get benchmark ID from URL parameters -benchmark_id = st.query_params.get("id", None) - -if not benchmark_id: - st.title("Security Benchmarks") - - # Show table of all benchmarks - benchmarks_df = get_all_benchmarks() - if not benchmarks_df.empty: - # Add link to benchmark details - benchmarks_df['details_page'] = benchmarks_df['id'].apply(get_benchmark_page_link) - - st.dataframe( - benchmarks_df, - column_config={ - 'id': st.column_config.NumberColumn('ID', width='small'), - 'benchmark': st.column_config.TextColumn('Benchmark', width='medium'), - 'finding_id': st.column_config.TextColumn('Finding ID', width='small'), - 'level': st.column_config.TextColumn('Level', width='small'), - 'cvss': st.column_config.NumberColumn('CVSS', format="%.1f", width='small'), - 'title': st.column_config.TextColumn('Title', width='large'), - 'total_issues': st.column_config.NumberColumn('Total Issues', width='small'), - 'open_issues': st.column_config.NumberColumn('Open Issues', width='small'), - 'total_remediations': st.column_config.NumberColumn('Total Remediations', width='small'), - 'open_remediations': st.column_config.NumberColumn('Open Remediations', width='small'), - 'details_page': st.column_config.LinkColumn('Details', display_text="View", width='small') - }, - hide_index=True, - use_container_width=True - ) - else: - st.info("No benchmarks found.") -else: - # Get benchmark details - benchmark, issues, remediations = get_benchmark_details(benchmark_id) - - if not benchmark: - st.error(f"Benchmark {benchmark_id} not found.") - st.stop() - - # Basic breadcrumbs - st.markdown(f""" - - """, unsafe_allow_html=True) - - # Display benchmark details - st.title(f"{benchmark['title']}") - - # Benchmark metadata - col1, col2 = st.columns(2) - with col1: - st.metric("CVSS Score", f"{benchmark['cvss']:.1f}") - with col2: - st.metric("Level", benchmark['level']) - - # Issues list - st.divider() - st.subheader("Issues") - if issues: - issues_df = pd.DataFrame(issues) - issues_df['details_page'] = issues_df['id'].apply(get_issue_page_link) - render_issues_list(issues_df) - else: - st.info("No issues found for this benchmark.") - - # Open remediations - st.divider() - st.subheader("Open Remediations") - if remediations: - remediations_df = pd.DataFrame(remediations) - # Add link to the issue for each remediation - remediations_df['issue_link'] = remediations_df['issue_id'].apply(get_issue_page_link) - - st.dataframe( - remediations_df, - column_config={ - 'id': st.column_config.NumberColumn('ID', width='small'), - 'state': st.column_config.TextColumn('State', width='small'), - 'due_date': st.column_config.DateColumn('Due Date', width='small'), - 'first_seen': st.column_config.DateColumn('First Seen', width='small'), - 'failure': st.column_config.TextColumn('Failure', width='large'), - 'issue_link': st.column_config.LinkColumn('Issue', display_text="View Issue", width='small') - }, - hide_index=True, - use_container_width=True - ) - else: - st.info("No open remediations for this benchmark.") - - # Description and rationale - st.divider() - st.subheader("Description") - st.write(benchmark['description']) - - if benchmark['rationale']: - st.subheader("Rationale") - st.write(benchmark['rationale']) - - # References - if benchmark['refs']: - st.divider() - st.subheader("References") - for ref in benchmark['refs'].split('\n'): - if ref.strip(): - st.markdown(f"- [{ref}]({ref})") diff --git a/app/pages/4_settings.py b/app/pages/4_settings.py deleted file mode 100644 index 8a08669..0000000 --- a/app/pages/4_settings.py +++ /dev/null @@ -1,52 +0,0 @@ -import streamlit as st -from app.database.schema import get_db_connection, get_poam_config, update_poam_config - -st.set_page_config( - page_title="Settings", - page_icon="🔒", - layout="wide" -) - -st.title("Settings") - -# POAM Export Configuration -st.header("POAM Export Configuration") -st.write("Configure settings for POAM CSV exports.") - -# Get current configuration -conn = get_db_connection() -config = get_poam_config(conn) - -# Default values -point_of_contact = config['point_of_contact'] if config else "" -google_project = config['google_project'] if config else "" -service_name = config['service_name'] if config else "" - -# Form for updating configuration -with st.form("poam_config_form"): - new_point_of_contact = st.text_input( - "Point of Contact", - value=point_of_contact, - help="The point of contact for POAM exports" - ) - new_google_project = st.text_input( - "Google Project", - value=google_project, - help="The Google project identifier" - ) - new_service_name = st.text_input( - "Service Name", - value=service_name, - help="The service name for POAM exports" - ) - - if st.form_submit_button("Save Configuration"): - update_poam_config( - conn, - point_of_contact=new_point_of_contact, - google_project=new_google_project, - service_name=new_service_name - ) - st.success("Configuration updated successfully!") - -conn.close() \ No newline at end of file diff --git a/app/pages/5_poam_export.py b/app/pages/5_poam_export.py deleted file mode 100644 index a625ac2..0000000 --- a/app/pages/5_poam_export.py +++ /dev/null @@ -1,99 +0,0 @@ -import streamlit as st -from app.components.poam_exports import generate_poam_export -from app.database.schema import get_db_connection, get_poam_config -from datetime import datetime - -st.set_page_config( - page_title="POAM Export", - page_icon="🔒", - layout="wide" -) - -st.title("POAM Export") - -# Check if POAM configuration exists -conn = get_db_connection() -config = get_poam_config(conn) -conn.close() - -if not config: - st.error("POAM configuration is not set. Please configure POAM export settings in the Settings page first.") - st.stop() - -# Display current configuration -st.write("Current POAM Configuration:") -st.json({ - "Point of Contact": config['point_of_contact'], - "Google Project": config['google_project'], - "Service Name": config['service_name'] -}) - -st.divider() - -# Export buttons for open and closed issues -col1, col2 = st.columns(2) - -with col1: - st.subheader("Export Open Issues") - if st.button("Download Open Issues POAM", type="primary"): - try: - df = generate_poam_export(status='open') - if df.empty: - st.warning("No open issues found.") - else: - # Generate CSV - csv = df.to_csv(index=False) - current_date = datetime.now().strftime('%Y%m%d') - filename = f"poam_open_issues_{current_date}.csv" - - # Create download button - st.download_button( - label="Click to Download Open Issues POAM", - data=csv, - file_name=filename, - mime="text/csv" - ) - st.success(f"Generated POAM export with {len(df)} open issues.") - except Exception as e: - st.error(f"Error generating POAM export: {str(e)}") - -with col2: - st.subheader("Export Closed Issues") - if st.button("Download Closed Issues POAM", type="primary"): - try: - df = generate_poam_export(status='resolved') - if df.empty: - st.warning("No closed issues found.") - else: - # Generate CSV - csv = df.to_csv(index=False) - current_date = datetime.now().strftime('%Y%m%d') - filename = f"poam_closed_issues_{current_date}.csv" - - # Create download button - st.download_button( - label="Click to Download Closed Issues POAM", - data=csv, - file_name=filename, - mime="text/csv" - ) - st.success(f"Generated POAM export with {len(df)} closed issues.") - except Exception as e: - st.error(f"Error generating POAM export: {str(e)}") - -# Add help text -st.divider() -st.markdown(""" -### About POAM Export -This page allows you to export security issues in Plan of Action and Milestones (POAM) format. The export includes: - -- Separate exports for open and closed issues -- POAM IDs in the format YYYY-CISxxxx (based on issue creation year) -- Standard CIS control mapping (CM-6) -- Configured point of contact and project information -- Detailed finding information including: - - Weakness details and descriptions - - Asset identifiers - - Detection and due dates - - Risk ratings based on CVSS scores -""") \ No newline at end of file diff --git a/app/pages/settings.py b/app/pages/settings.py deleted file mode 100644 index db65295..0000000 --- a/app/pages/settings.py +++ /dev/null @@ -1,35 +0,0 @@ -import streamlit as st -from app.database.schema import get_db_connection, get_poam_config, update_poam_config - -def render(): - st.title("Settings") - - # POAM Export Configuration - st.header("POAM Export Configuration") - st.write("Configure settings for POAM CSV exports.") - - # Get current configuration - conn = get_db_connection() - config = get_poam_config(conn) - - # Default values - point_of_contact = config['point_of_contact'] if config else "" - google_project = config['google_project'] if config else "" - service_name = config['service_name'] if config else "" - - # Form for updating configuration - with st.form("poam_config_form"): - new_point_of_contact = st.text_input("Point of Contact", value=point_of_contact) - new_google_project = st.text_input("Google Project", value=google_project) - new_service_name = st.text_input("Service Name", value=service_name) - - if st.form_submit_button("Save Configuration"): - update_poam_config( - conn, - point_of_contact=new_point_of_contact, - google_project=new_google_project, - service_name=new_service_name - ) - st.success("Configuration updated successfully!") - - conn.close() \ No newline at end of file diff --git a/app/static/style.css b/app/static/style.css deleted file mode 100644 index 59606f5..0000000 --- a/app/static/style.css +++ /dev/null @@ -1,177 +0,0 @@ -/* Custom styles inspired by hail.is */ -.stApp { - background-color: #f8f9fa; -} - -.main-header { - color: #1a1a1a; - font-family: 'Inter', sans-serif; - font-weight: 600; - margin-bottom: 2rem; -} - -.stats-container { - background: white; - border-radius: 8px; - padding: 1.5rem; - box-shadow: 0 2px 4px rgba(0,0,0,0.05); - margin-bottom: 2rem; -} - -.stat-card { - background: #ffffff; - border: 1px solid #e9ecef; - border-radius: 8px; - padding: 1.5rem; - text-align: center; - transition: transform 0.2s; -} - -.stat-card:hover { - transform: translateY(-2px); - box-shadow: 0 4px 6px rgba(0,0,0,0.1); -} - -.stat-number { - font-size: 2.5rem; - font-weight: bold; - margin-bottom: 0.5rem; -} - -.stat-label { - font-size: 1rem; - color: #6c757d; -} - -.findings-table { - background: white; - border-radius: 8px; - padding: 1rem; - box-shadow: 0 2px 4px rgba(0,0,0,0.05); -} - -.severity-critical { - color: #dc3545; -} - -.severity-high { - color: #fd7e14; -} - -.severity-medium { - color: #ffc107; -} - -.severity-low { - color: #20c997; -} - -.severity-info { - color: #0dcaf0; -} - -/* Upload section styling */ -.upload-section { - background: white; - border-radius: 8px; - padding: 2rem; - box-shadow: 0 2px 4px rgba(0,0,0,0.05); - margin-bottom: 2rem; -} - -.stButton>button { - background-color: #2d63ed; - color: white; - border: none; - padding: 0.5rem 1rem; - border-radius: 4px; - font-weight: 500; - transition: background-color 0.2s; -} - -.stButton>button:hover { - background-color: #1c4ed8; -} - -/* Custom styling for the date input */ -.date-input { - max-width: 200px; - margin-bottom: 1rem; -} - -/* Tabs styling */ -.stTabs [data-baseweb="tab-list"] { - gap: 2rem; - border-bottom: 1px solid #dee2e6; -} - -.stTabs [data-baseweb="tab"] { - height: 50px; - white-space: pre-wrap; - background-color: transparent !important; - border-radius: 0; - color: #6c757d; - font-size: 1rem; - font-weight: 500; - border-bottom: 2px solid transparent; - padding-bottom: 1rem; -} - -.stTabs [aria-selected="true"] { - background-color: transparent !important; - color: #dc3545 !important; - border-bottom: 2px solid #dc3545 !important; -} - -.stTabs [data-baseweb="tab"]:hover { - color: #dc3545; - border-bottom: 2px solid #dc3545; -} - -.charts-container { - margin: 2rem 0; -} - -.findings-table { - margin: 2rem 0; -} - -/* Due date warning styles */ -.warning-stat { - background-color: rgba(255, 243, 205, 0.7); - border: 1px solid #ffeeba; -} - -.overdue-stat { - background-color: rgba(248, 215, 218, 0.7); - border: 1px solid #f5c6cb; -} - -[data-testid="stDataFrame"] div[data-testid="stHorizontalBlock"] { - gap: 0 !important; -} - -/* Warning and overdue icons */ -.warning-icon::before { - content: "⚠️"; - margin-right: 8px; -} - -.overdue-icon::before { - content: "❗"; - margin-right: 8px; -} - -/* Row highlighting */ -[data-testid="stDataFrame"] [data-testid="StyledDataFrameDataCell"] { - transition: background-color 0.2s ease; -} - -/* Due date text colors */ -.text-warning { - color: #856404 !important; -} - -.text-overdue { - color: #721c24 !important; -} \ No newline at end of file diff --git a/cli/README.md b/cli/README.md deleted file mode 100644 index d01d0da..0000000 --- a/cli/README.md +++ /dev/null @@ -1,175 +0,0 @@ -# Security Findings CLI - -Command-line interface for managing security findings from Trivy scans. - -## Installation - -Ensure you have Python 3.x installed and the required dependencies: - -```bash -pip install -r requirements.txt -``` - -## Testing - -To run the tests, first install pytest and coverage tools: - -```bash -pip install pytest pytest-cov -``` - -The project uses a standard Python test layout: -``` -security-tracker-app/ -├── tests/ -│ ├── __init__.py -│ ├── conftest.py # Test configuration and fixtures -│ └── test_poam.py # Tests for POAM functionality -├── tools/ -│ ├── __init__.py -│ ├── poam.py -│ └── ... -└── cli/ - └── ... -``` - -Then run the tests: - -```bash -# Run all tests -pytest tests/ - -# Run specific test file -pytest tests/test_poam.py - -# Run with verbose output -pytest tests/test_poam.py -v - -# Run tests and show coverage -pytest tests/ --cov=tools -``` - -The test suite includes: -- Unit tests for data conversion utilities -- Field name handling for POAM entries -- Edge cases for text formatting - -## Commands - -The CLI is organized into command groups for better organization and usability. - -### POAM Commands - -Commands for working with POAMs are grouped under the `poams` command: - -```bash -# Interactive weekly update process -./cli.py poams weekly-update - -# Preview POAMs from an Excel file -./cli.py poams preview-trivy [--limit ] - -# Apply diff changes to a POAM Excel file -./cli.py poams apply-diff -``` - -#### Weekly Update Process - -The `weekly-update` command provides an interactive workflow for processing weekly security findings: - -1. **Working Directory Setup**: Prompts for a working directory (default: `working/YYYY-MM-DD`) -2. **Directory Contents**: Shows current directory contents with relative paths -3. **Input Files**: Prompts for paths to: - - Continuous CIS findings sheet (suggests files with "CIS" in name if found) - - Most recent ZAP scan (suggests files starting with "hail_report" if found) - - Current POAMs file (suggests files with "POAM" in name if found) -4. **Trivy Processing**: Interactive prompts for: - - Downloading Trivy alerts - - Converting alerts to findings CSV -5. **CIS Processing**: Interactive prompts for: - - Splitting connected sheet - - Converting most recent findings to JSON -6. **ZAP Processing**: Interactive prompts for: - - Converting ZAP scan to findings JSON - -All prompts show suggested defaults and accept empty input to use the default value. - -### Trivy Commands - -Commands for working with Trivy alerts are grouped under the `trivy` command: - -```bash -# Download Trivy alerts from GitHub's code scanning API -./cli.py trivy download-alerts - -# Convert downloaded GitHub Trivy alerts from JSON to CSV format -./cli.py trivy convert-alerts - -# Compare current Trivy alerts against existing POAMs -./cli.py trivy alerts-diff -``` - -### ZAP Commands - -Commands for working with ZAP scan reports are grouped under the `zap` command: - -```bash -# Convert ZAP CSV alerts to findings JSON format -./cli.py zap alerts-to-findings -``` - -This command: -- Takes a ZAP CSV report file as input -- Converts each alert to a finding object with: - - Finding ID (based on ZAP alert ID) - - Weakness name and description - - Asset identifier (host) - - Risk rating and confidence level - - List of affected instances (URLs, methods, parameters) -- Saves the findings as a JSON file -- Displays the first finding and total count - -Each command includes error handling and will provide helpful error messages if something goes wrong. - -## Example Workflow - -1. Download alerts from GitHub: - ```bash - # Download to default location (WORKING env var or pwd/working) - ./cli.py trivy download-alerts - - # Download to specific file - ./cli.py trivy download-alerts --destination /path/to/alerts.json - ``` - -2. Convert the downloaded JSON to CSV: - ```bash - # Convert with default .findings.csv extension in same directory - ./cli.py trivy convert-alerts alerts_20240513.json - - # Convert to specific output file - ./cli.py trivy convert-alerts alerts_20240513.json --output /path/to/output.csv - ``` - -3. Compare new alerts against existing POAMs: - ```bash - ./cli.py trivy alerts-diff existing_poams.xlsx working/trivy_alerts_20240513_180947.csv - ``` - -4. Preview POAMs in an Excel file: - ```bash - ./cli.py poams preview-trivy existing_poams.xlsx - ``` - -5. Apply diff changes to update POAMs: - ```bash - # Apply single diff file - ./cli.py poams apply-diff existing_poams.xlsx alerts_20240513.diff.json - - # Apply multiple diff files - ./cli.py poams apply-diff existing_poams.xlsx alerts1.diff.json alerts2.diff.json alerts3.diff.json - ``` - - > [!NOTE] - > We can merge the diff files before applying with `./cli.py poams merge-diffs` before applying, but the apply command also - > accepts multiple diff files natively. diff --git a/requirements.txt b/requirements.txt index fa07206..92a6d5b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,9 +1,6 @@ -streamlit>=1.43.0; python_version >= "3.9" and python_version < "3.13" pandas==2.2.1; python_version >= "3.9" and python_version < "3.13" -numpy==1.26.4; python_version >= "3.9" and python_version < "3.13" -plotly==5.19.0; python_version >= "3.9" and python_version < "3.13" python-dateutil==2.8.2; python_version >= "3.9" and python_version < "3.13" click>=8.1.7; python_version >= "3.9" and python_version < "3.13" openpyxl>=3.1.2; python_version >= "3.9" and python_version < "3.13" PyYAML>=6.0.1; python_version >= "3.9" and python_version < "3.13" -jq>=1.6.0; python_version >= "3.9" and python_version < "3.13" \ No newline at end of file +jq>=1.6.0; python_version >= "3.9" and python_version < "3.13" diff --git a/tests/test_diff.py b/tests/test_diff.py index 01939ec..2168ee1 100644 --- a/tests/test_diff.py +++ b/tests/test_diff.py @@ -197,7 +197,7 @@ def test_compare_findings_to_poams(): ] # Compare findings to POAMs - diff = compare_findings_to_poams(findings, open_poams, closed_poams) + diff = compare_findings_to_poams(findings, open_poams, closed_poams, existing_poam_ids=[], poam_generator=lambda f, ids: []) # Verify new findings assert {f.finding_id for f in diff.new_findings} == {"TRIVY-002"} diff --git a/tools/cis/poam_generator.py b/tools/cis/poam_generator.py index fce2170..0d11b37 100644 --- a/tools/cis/poam_generator.py +++ b/tools/cis/poam_generator.py @@ -96,7 +96,7 @@ def generate_poams_from_findings(findings: List[Finding], existing_poam_ids: Lis # Unpack findings and their completion dates findings_list = [f for f, _ in group] first_finding = findings_list[0] - completion_date = first_finding.scheduled_completion_date + completion_date = group[0][1] # Get earliest detection date from group detection_date = min(f.original_detection_date for f in findings_list) diff --git a/tools/trivy/trivy_alerts_poaminator.py b/tools/trivy/trivy_alerts_poaminator.py deleted file mode 100644 index bf05239..0000000 --- a/tools/trivy/trivy_alerts_poaminator.py +++ /dev/null @@ -1,146 +0,0 @@ -# For reference only: the original script that generated the alerts.csv file. -# For the current script that is wired into the CLI, see trivy/alerts.py - -# import csv -# import json -# import jq -# from datetime import datetime, timedelta - - -# def date_plus(iso_date_string, days_to_add): -# """ -# Parses an ISO date string, adds days, and formats it to a custom date string. - -# Args: -# iso_date_string: The ISO date string to parse (e.g., "2023-10-26T12:00:00Z"). -# days_to_add: The number of days to add (can be positive or negative). -# output_format: The desired output date format string (e.g., "%Y-%m-%d"). - -# Returns: -# The formatted date string, or None if parsing fails. -# """ -# try: -# date_object = datetime.fromisoformat(iso_date_string.replace("Z", "+00:00")) -# except ValueError: -# print("Error: Invalid ISO date string format.") -# return None - -# modified_date = date_object + timedelta(days=days_to_add) -# formatted_date = modified_date.strftime("%m/%d/%y") -# return formatted_date - - -# fieldnames = [ -# "Alert ID", -# "Controls", -# "Weakness Name", -# "Weakness Description", -# "Weakness Detector Source", -# "Weakness Source Identifier", -# "Asset Identifier", -# "Point of Contact", -# "Resources Required", -# "Overall Remediation Plan", -# "Original Detection Date", -# "Scheduled Completion Date", -# "AGENCY Scheduled Completion Date", -# "Planned Milestones", -# "Milestone Changes", -# "Status Date", -# "Vendor Dependency", -# "Last Vendor Check-in Date", -# "Vendor Dependent Product Name", -# "Original Risk Rating", -# "Adjusted Risk Rating", -# "Risk Adjustment", -# "False Positive", -# "Operational Requirement", -# "Deviation Rationale", -# "Supporting Documents", -# "Comments", -# "Auto-Approve", -# "Binding Operational Directive 22-01 tracking", -# "Binding Operational Directive 22-01 Due Date", -# "CVE", -# "Service Name", -# ] - -# with open("alerts.json") as inf: -# alerts_data = json.load(inf) - -# alerts_jq = jq.compile(""" -# .[] | { -# "_state": .state, -# "POAM ID": .number, -# "Controls": "RA-5", -# "Weakness Name": .rule.description, -# "Weakness Description": .rule.full_description, -# "Weakness Detector Source": .html_url, -# "Weakness Source Identifier": (.tool.name + " " + .tool.version), -# "Asset Identifier": .rule.most_recent_instance.location.path, -# "Point of Contact": "Chris Llanwarne", -# "Resources Required": "None", -# "Overall Remediation Plan": "Perform necessary updates to resolve the vulnerability", -# "Original Detection Date": .created_at, -# "Status Date": .updated_at, -# "Last Vendor Check-in Date": .rule.updated_at, -# "Scheduled Completion Date": "DATE", -# "AGENCY Scheduled Completion Date": "DATE", -# "Planned Milestones": "DATE: Perform necessary updates to resolve the vulnerability", -# "Milestone Changes": "", -# "Vendor Dependency": "Yes", -# "Vendor Dependent Product Name": "Ubuntu", -# "Original Risk Rating": .rule.security_severity_level, -# "Adjusted Risk Rating": "", -# "Risk Adjustment": "", -# "False Positive": "No", -# "Operational Requirement": "No", -# "Deviation Rationale": "", -# "Supporting Documents": "", -# "Comments": .most_recent_instance.message.text, -# "Auto-Approve": "No", -# "Binding Operational Directive 22-01 tracking": "", -# "Binding Operational Directive 22-01 Due Date": "", -# "CVE": .rule.id, -# "Service Name": "Hail Batch" -# }""") - -# jq_results = alerts_jq.input_value(alerts_data) -# rows: list[dict] = [] - -# for row in jq_results.all(): -# if row["Weakness Source Identifier"][:5] != "Trivy": -# continue -# state = row["_state"] -# del row["_state"] -# if state != "open": -# continue -# message = { -# kv[0]: (kv[1] if len(kv) > 1 else "") -# for kv in [line.split(": ") for line in row["Comments"].split("\n")] -# } -# if "Image" not in message: -# print(message) -# print(repr(row)) -# row["Asset Identifier"] = f"{message['Image']} ({message['Package']})" -# orig_date = row["Original Detection Date"] -# status_date = row["Status Date"] -# sev = row["Original Risk Rating"] -# fix_intervals = {"high": 14, "medium": 90, "low": 180} -# fix_interval = fix_intervals.get(sev) or 0 -# fix_date = date_plus(orig_date, fix_interval) -# row["Original Detection Date"] = date_plus(orig_date, 0) -# row["Status Date"] = date_plus(status_date, 0) -# row["Last Vendor Check-in Date"] = date_plus(status_date, 0) -# row["Scheduled Completion Date"] = date_plus(orig_date, 0) -# row["Original Detection Date"] = date_plus(orig_date, 0) -# row["Scheduled Completion Date"] = fix_date -# row["AGENCY Scheduled Completion Date"] = fix_date -# row["Planned Milestones"] = row["Planned Milestones"].replace("DATE", fix_date) - -# rows.append(row) - -# with open("gh-alerts.csv", "w", newline="") as csvfile: -# writer = csv.DictWriter(csvfile, fieldnames=fieldnames) -# writer.writeheader() -# writer.writerows(rows) From dd4b47b83ed4ab344eb04232aa613d90736c5b09 Mon Sep 17 00:00:00 2001 From: Chris Llanwarne Date: Thu, 5 Mar 2026 17:08:54 -0500 Subject: [PATCH 30/33] update docs --- CLAUDE.md | 2 +- README.md | 41 ++++------------------------------------- 2 files changed, 5 insertions(+), 38 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index b543103..d7e2b3c 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -4,7 +4,7 @@ This file provides guidance to Claude Code (claude.ai/code) when working with co ## Overview -A security findings tracker that manages POA&M (Plan of Action and Milestones) lifecycle for compliance purposes. The **CLI is the primary entry point** (`cli/cli.py`). The Streamlit web app in `app/` is abandonware — do not work on it. +A security findings tracker that manages POA&M (Plan of Action and Milestones) lifecycle for compliance purposes. The CLI (`cli/cli.py`) is the sole entry point. ## Commands diff --git a/README.md b/README.md index 897abae..17dfe82 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # Security Findings Tracker -A Streamlit-based application for tracking and managing security findings from weekly CSV uploads. The application provides an interface for analyzing security findings, tracking their status, and managing their lifecycle. +A CLI tool for tracking and managing POA&M (Plan of Action and Milestones) lifecycle for compliance purposes. Processes weekly security findings from Trivy, ZAP, and CIS scans and applies them to a POAM Excel file. ## Requirements @@ -9,13 +9,10 @@ A Streamlit-based application for tracking and managing security findings from w ## Features -- Upload and process weekly CSV security findings -- Track new, existing, and resolved findings +- Download and process Trivy, ZAP, and CIS findings +- Diff findings against existing POAMs to identify new, closed, and reopened items - Automatic due date assignment based on severity -- Interactive data visualization -- Export capabilities for active and resolved findings -- Modern, responsive UI inspired by hail.is -- Command line tools for automation and data management +- Apply diffs to update the POAM Excel file ## Installation @@ -111,33 +108,3 @@ export GOOGLE_APPLICATION_CREDENTIALS="/path/to/service-account-key.json" The downloaded files will be saved to the `working` directory in the project root. -## Web Application Usage - -1. Start the Streamlit application (make sure you're in the repository root directory): -```bash -# On Unix/macOS: -PYTHONPATH=$PYTHONPATH:. streamlit run app/main.py - -# On Windows (PowerShell): -$env:PYTHONPATH = "$env:PYTHONPATH;." -streamlit run app/main.py - -# On Windows (Command Prompt): -set PYTHONPATH=%PYTHONPATH%;. -streamlit run app/main.py -``` - -2. Open your web browser and navigate to the URL shown in the terminal (typically http://localhost:8501) - -3. Use the application: - - Upload your weekly CSV file using the file uploader - - Set the analysis date - - View the summary statistics and visualizations - - Track recurrances and resolutions through additional scans and uploads. - - Export findings as needed - -## Database - -The application uses SQLite for data storage. The database file is created at `app/database/findings.db` and includes tables for: -- Findings tracking -- Upload history From c2504c9be41d75af03c25a84fe31d3f9e60d6ff2 Mon Sep 17 00:00:00 2001 From: Chris Llanwarne Date: Thu, 5 Mar 2026 17:32:44 -0500 Subject: [PATCH 31/33] feedback --- CLAUDE.md | 2 +- README.md | 11 ----------- cli/cli.py | 8 +++----- tools/diff.py | 4 ++-- tools/diff_apply.py | 4 +++- tools/utils.py | 2 +- tools/zap/diff.py | 4 ++-- 7 files changed, 12 insertions(+), 23 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index d7e2b3c..34cd824 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -58,7 +58,7 @@ Each scanner type (`trivy/`, `zap/`, `cis/`) has: - `diff.py` — calls `compare_findings_to_poams()` with the right POAM filter and generator - `poam_generator.py` — generates `PoamEntry` objects with appropriate POAM IDs -POAM ID formats: Trivy → `YYYY-TRIVYXXXX`, CIS → `CIS--XXXX` +POAM ID formats: Trivy → `YYYY-TRIVYXXXX`, CIS → `YYYY-CISXXXX` ### Working Directory Convention By default, files are saved to `working/YYYY-MM-DD/`. The `WORKING` environment variable can override the base path. diff --git a/README.md b/README.md index 17dfe82..bc7ef5e 100644 --- a/README.md +++ b/README.md @@ -97,14 +97,3 @@ export GOOGLE_APPLICATION_CREDENTIALS="/path/to/service-account-key.json" ### Available Commands -1. Download Google Sheets: -```bash -# Using a Google Sheets URL -./cli/cli.py download-gsheet "https://docs.google.com/spreadsheets/d/YOUR_SHEET_ID" - -# Or using just the file ID -./cli/cli.py download-gsheet "YOUR_SHEET_ID" -``` - -The downloaded files will be saved to the `working` directory in the project root. - diff --git a/cli/cli.py b/cli/cli.py index 95e7966..d19cfef 100755 --- a/cli/cli.py +++ b/cli/cli.py @@ -159,9 +159,7 @@ def download_alerts(destination): If destination is not specified, uses WORKING environment variable or pwd/working and sets filename to trivy-alerts-.json - Requires one of: - 1. GitHub CLI (gh) to be installed and authenticated via 'gh auth login' - 2. GitHub token provided via --token option or GITHUB_TOKEN environment variable + Requires the GitHub CLI (gh) to be installed and authenticated via 'gh auth login'. """ try: if destination: @@ -540,7 +538,7 @@ def alerts_to_findings(csv_file, output): @click.argument('poam_file', type=click.Path(exists=True)) @click.argument('findings_file', type=click.Path(exists=True)) @click.option('--json-output', type=click.Path(), help='Path to save JSON output') -def alerts_diff(poam_file: str, findings_file: str, json_output: Optional[str]) -> None: +def zap_alerts_diff(poam_file: str, findings_file: str, json_output: Optional[str]) -> None: """Compare ZAP findings against existing POAMs. Note: Findings with Info severity are automatically excluded. @@ -623,7 +621,7 @@ def csv_to_findings_cmd(csv_file: Path, output: Optional[Path]) -> None: @click.argument('poam_file', type=click.Path(exists=True)) @click.argument('findings_file', type=click.Path(exists=True)) @click.option('--json-output', type=click.Path(), help='Path to save JSON output') -def alerts_diff(poam_file: str, findings_file: str, json_output: Optional[str]) -> None: +def cis_alerts_diff(poam_file: str, findings_file: str, json_output: Optional[str]) -> None: """ Compare CIS findings against existing configuration findings. diff --git a/tools/diff.py b/tools/diff.py index 64c48c2..dbe8748 100644 --- a/tools/diff.py +++ b/tools/diff.py @@ -2,7 +2,7 @@ Module for comparing findings against existing POAMs. """ from dataclasses import dataclass -from typing import List, Optional, Tuple, Dict, Any +from typing import List, Optional, Tuple, Dict, Any, Union from pathlib import Path from datetime import datetime @@ -33,7 +33,7 @@ def to_json(self) -> Dict[str, Any]: Returns: Dictionary containing the diff results in a structured format """ - def format_datetime(dt: datetime | str) -> str: + def format_datetime(dt: Union[datetime, str]) -> str: if isinstance(dt, str): return dt else: diff --git a/tools/diff_apply.py b/tools/diff_apply.py index e45ab4d..c745e17 100644 --- a/tools/diff_apply.py +++ b/tools/diff_apply.py @@ -66,6 +66,7 @@ def apply_diff(poam_file: Path, diff_json: Dict[str, Any], output_file: Path = N # Create editable copy editable_copy = create_updateable_copy(poam_file, output_file) + wb = None try: # Load workbook from editable copy wb = openpyxl.load_workbook(editable_copy) @@ -178,7 +179,8 @@ def apply_diff(poam_file: Path, diff_json: Dict[str, Any], output_file: Path = N # If anything goes wrong, leave the half-edited copy for inspection raise type(e)(f"Error applying diff changes. Incomplete edit saved as {editable_copy}. Error: {str(e)}") from e finally: - wb.close() + if wb is not None: + wb.close() def merge_diffs(diff_files: List[Path]) -> Dict[str, Any]: """ diff --git a/tools/utils.py b/tools/utils.py index f34980a..b2783ef 100644 --- a/tools/utils.py +++ b/tools/utils.py @@ -22,5 +22,5 @@ def ensure_working_dir() -> Path: # Fall back to pwd/working working_dir = Path(os.getcwd()) / 'working' - working_dir.mkdir(exist_ok=True) + working_dir.mkdir(parents=True, exist_ok=True) return working_dir \ No newline at end of file diff --git a/tools/zap/diff.py b/tools/zap/diff.py index e3e1cda..946c2e8 100644 --- a/tools/zap/diff.py +++ b/tools/zap/diff.py @@ -2,7 +2,7 @@ Module for comparing ZAP findings against existing POAMs. """ from dataclasses import dataclass -from typing import List, Optional, Tuple, Dict, Any +from typing import List, Optional, Tuple, Dict, Any, Union from pathlib import Path from datetime import datetime @@ -33,7 +33,7 @@ def to_json(self) -> Dict[str, Any]: Returns: Dictionary containing the diff results in a structured format """ - def format_datetime(dt: datetime | str) -> str: + def format_datetime(dt: Union[datetime, str]) -> str: if isinstance(dt, str): return dt else: From 439e94e4cc10fb5bd00070025412f648fe8ae6da Mon Sep 17 00:00:00 2001 From: Chris Llanwarne Date: Thu, 5 Mar 2026 17:32:59 -0500 Subject: [PATCH 32/33] helper function for review parsing --- devbin/pr-comments | 42 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) create mode 100755 devbin/pr-comments diff --git a/devbin/pr-comments b/devbin/pr-comments new file mode 100755 index 0000000..2b001f7 --- /dev/null +++ b/devbin/pr-comments @@ -0,0 +1,42 @@ +#!/usr/bin/env bash +# Usage: pr-comments +# Downloads and prints all comments on a GitHub PR: +# - The review summaries (from /pulls/:pr/reviews) +# - Inline code comments (from /pulls/:pr/comments) +# - General issue comments (from /issues/:pr/comments) + +set -euo pipefail + +PR="${1:?Usage: pr-comments }" +REPO=$(gh repo view --json nameWithOwner -q .nameWithOwner) + +echo "═══════════════════════════════════════════════════════════" +echo " PR #$PR — $REPO" +echo "═══════════════════════════════════════════════════════════" + +# ── Review summaries ──────────────────────────────────────────── +echo "" +echo "── REVIEWS ────────────────────────────────────────────────" + +gh api "repos/$REPO/pulls/$PR/reviews" --paginate | \ + jq -r '.[] | (.body // "") as $body | + if ($body | length) > 0 + then "\n[\(.state)] \(.user.login) — \(.submitted_at)\n\($body)" + else "\n[\(.state)] \(.user.login) — \(.submitted_at)" + end' + +# ── Inline code comments ──────────────────────────────────────── +echo "" +echo "── INLINE COMMENTS ────────────────────────────────────────" + +gh api "repos/$REPO/pulls/$PR/comments" --paginate | \ + jq -r '.[] | + "\n\(.path) (line \(.original_line // .line // "?")) — \(.user.login)\n\(.body)"' + +# ── General PR / issue comments ───────────────────────────────── +echo "" +echo "── GENERAL COMMENTS ───────────────────────────────────────" + +gh api "repos/$REPO/issues/$PR/comments" --paginate | \ + jq -r '.[] | + "\n\(.user.login) — \(.created_at)\n\(.body)"' From 7e095e5bd7b100ed8ec7bbd9070085f141555074 Mon Sep 17 00:00:00 2001 From: Chris Llanwarne Date: Thu, 5 Mar 2026 17:45:07 -0500 Subject: [PATCH 33/33] pytest --- .github/workflows/test.yml | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) create mode 100644 .github/workflows/test.yml diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 0000000..9658e9d --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,20 @@ +name: Tests + +on: + pull_request: + +jobs: + test: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + + - name: Install dependencies + run: pip install -r requirements.txt pytest + + - name: Run tests + run: python -m pytest tests/ -q