diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 0000000..9658e9d --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,20 @@ +name: Tests + +on: + pull_request: + +jobs: + test: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + + - name: Install dependencies + run: pip install -r requirements.txt pytest + + - name: Run tests + run: python -m pytest tests/ -q diff --git a/.gitignore b/.gitignore index e3101ee..f811134 100644 --- a/.gitignore +++ b/.gitignore @@ -175,5 +175,9 @@ cython_debug/ # PyPI configuration file .pypirc +# macOS +.DS_Store + # Ignore any findings.db files -findings.db \ No newline at end of file +findings.db +working/ diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 0000000..34cd824 --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1,68 @@ +# CLAUDE.md + +This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository. + +## Overview + +A security findings tracker that manages POA&M (Plan of Action and Milestones) lifecycle for compliance purposes. The CLI (`cli/cli.py`) is the sole entry point. + +## Commands + +### Setup +```bash +python -m venv venv +source venv/bin/activate +pip install -r requirements.txt +``` + +### CLI Usage +```bash +./cli/cli.py --help +./cli/cli.py poams weekly-update # Interactive guided weekly update +./cli/cli.py poams apply-diff [ ...] +./cli/cli.py poams merge-diffs -o merged.json +./cli/cli.py trivy download-alerts [-d ] +./cli/cli.py trivy convert-alerts [-o ] +./cli/cli.py trivy alerts-diff +./cli/cli.py zap alerts-to-findings [-o ] +./cli/cli.py zap alerts-diff +./cli/cli.py cis split-connected-sheet [-o ] +./cli/cli.py cis csv-to-findings [-o ] +./cli/cli.py cis alerts-diff +``` + +### Tests +```bash +python -m pytest # Run all tests +python -m pytest tests/test_diff.py # Run a single test file +python -m pytest tests/test_diff.py::test_name # Run a single test +``` + +## Architecture + +### Data Flow (weekly update) +1. **Download/collect** raw scan data: Trivy alerts from GitHub API, CIS Excel sheet, ZAP CSV report +2. **Convert** to normalized intermediate formats: `.findings.csv` (Trivy) or `.findings.json` (CIS, ZAP) +3. **Diff** each findings file against the existing POAM Excel → produces `.diff.json` files +4. **Apply** one or more diff JSONs to create an updated POAM Excel file + +### Core Data Structures (`tools/`) +- `tools/findings.py` — `Finding` dataclass: source-agnostic normalized security finding +- `tools/poam.py` — `PoamEntry` dataclass + `PoamFile` class: reads POAM Excel files (headers on row 5 of "Open POA&M Items" / "Closed POA&M Items" sheets) +- `tools/diff.py` — `compare_findings_to_poams()` and `PoamFileDiff`: matches findings to POAMs by exact `weakness_name` + asset coverage; produces lists of new/existing/closed/reopened +- `tools/diff_apply.py` — `apply_diff()`: writes changes back to Excel using openpyxl + +### Source-Specific Modules +Each scanner type (`trivy/`, `zap/`, `cis/`) has: +- `alerts.py` or `converter.py` — converts raw scanner output to `Finding` objects +- `diff.py` — calls `compare_findings_to_poams()` with the right POAM filter and generator +- `poam_generator.py` — generates `PoamEntry` objects with appropriate POAM IDs + +POAM ID formats: Trivy → `YYYY-TRIVYXXXX`, CIS → `YYYY-CISXXXX` + +### Working Directory Convention +By default, files are saved to `working/YYYY-MM-DD/`. The `WORKING` environment variable can override the base path. + +### Authentication +- GitHub (for Trivy downloads): `gh auth login` or `GITHUB_TOKEN` env var +- Google services: `gcloud auth application-default login` or `GOOGLE_APPLICATION_CREDENTIALS` env var diff --git a/README.md b/README.md index 0b9d06f..bc7ef5e 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # Security Findings Tracker -A Streamlit-based application for tracking and managing security findings from weekly CSV uploads. The application provides an interface for analyzing security findings, tracking their status, and managing their lifecycle. +A CLI tool for tracking and managing POA&M (Plan of Action and Milestones) lifecycle for compliance purposes. Processes weekly security findings from Trivy, ZAP, and CIS scans and applies them to a POAM Excel file. ## Requirements @@ -9,12 +9,10 @@ A Streamlit-based application for tracking and managing security findings from w ## Features -- Upload and process weekly CSV security findings -- Track new, existing, and resolved findings +- Download and process Trivy, ZAP, and CIS findings +- Diff findings against existing POAMs to identify new, closed, and reopened items - Automatic due date assignment based on severity -- Interactive data visualization -- Export capabilities for active and resolved findings -- Modern, responsive UI inspired by hail.is +- Apply diffs to update the POAM Excel file ## Installation @@ -40,33 +38,62 @@ source venv/bin/activate # On Windows: venv\Scripts\activate pip install -r requirements.txt ``` -## Usage +## Command Line Tools + +The application includes command line tools for automation and data management. These tools are available through the `cli.py` script in the `cli` directory. + +### Command Groups + +The CLI is organized into the following command groups: + +1. `poams` - Commands for working with POAMs: + ```bash + # Interactive weekly update process + ./cli/cli.py poams weekly-update + + # Preview Trivy POAMs from an Excel file + ./cli/cli.py poams preview-trivy [--limit ] + + # Apply diff changes to a POAM Excel file + ./cli/cli.py poams apply-diff + ``` + +2. `trivy` - Commands for working with Trivy: + ```bash + # Download Trivy alerts from GitHub code scanning API + ./cli/cli.py trivy download-alerts [--destination ] + + # Convert GitHub Trivy alerts JSON to POAM CSV format + ./cli/cli.py trivy convert-alerts [--output ] + + # Compare Trivy alerts against existing POAMs + ./cli/cli.py trivy alerts-diff + ``` + +To see all available commands and their options: +```bash +./cli/cli.py --help +``` -1. Start the Streamlit application (make sure you're in the repository root directory): +For help on a specific command group: ```bash -# On Unix/macOS: -PYTHONPATH=$PYTHONPATH:. streamlit run app/main.py +./cli/cli.py poams --help +./cli/cli.py trivy --help +``` -# On Windows (PowerShell): -$env:PYTHONPATH = "$env:PYTHONPATH;." -streamlit run app/main.py +### Authentication -# On Windows (Command Prompt): -set PYTHONPATH=%PYTHONPATH%;. -streamlit run app/main.py -``` +The tools that interact with Google services use Application Default Credentials (ADC). To set up authentication: -2. Open your web browser and navigate to the URL shown in the terminal (typically http://localhost:8501) +1. Using gcloud (recommended for development): +```bash +gcloud auth application-default login +``` -3. Use the application: - - Upload your weekly CSV file using the file uploader - - Set the analysis date - - View the summary statistics and visualizations - - Track recurrances and resolutions through additional scans and uploads. - - Export findings as needed +2. Or using a service account (recommended for production): +```bash +export GOOGLE_APPLICATION_CREDENTIALS="/path/to/service-account-key.json" +``` -## Database +### Available Commands -The application uses SQLite for data storage. The database file is created at `app/database/findings.db` and includes tables for: -- Findings tracking -- Upload history diff --git a/app/__init__.py b/app/__init__.py deleted file mode 100644 index d66fd94..0000000 --- a/app/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -""" -Security Findings Tracker application package -""" \ No newline at end of file diff --git a/app/components/IssuesList.py b/app/components/IssuesList.py deleted file mode 100644 index a310df4..0000000 --- a/app/components/IssuesList.py +++ /dev/null @@ -1,29 +0,0 @@ -import streamlit as st -import pandas as pd -from app.components.data_processor import export_issues_to_df -from app.components.IssuesTable import render_issues_table -from typing import Optional - -def render_issues_list(df_in: Optional[pd.DataFrame] = None): - """ - Renders a tabbed interface showing active and resolved issues. - """ - tab1, tab2 = st.tabs(["Active Issues", "Resolved Issues"]) - - def render_tab_content(status, label): - if df_in is None: - df = export_issues_to_df(status=status) - else: - # Filter by status - df = df_in[df_in['status'] == status] - - if not df.empty: - render_issues_table(df) - else: - st.info(f"No {label.lower()}.") - - with tab1: - render_tab_content('open', 'Active Findings') - - with tab2: - render_tab_content('resolved', 'Resolved Findings') \ No newline at end of file diff --git a/app/components/IssuesTable.py b/app/components/IssuesTable.py deleted file mode 100644 index 76aae35..0000000 --- a/app/components/IssuesTable.py +++ /dev/null @@ -1,181 +0,0 @@ -import streamlit as st -import pandas as pd -from datetime import datetime, timedelta - -def get_due_date_status(due_date: pd.Timestamp): - """Return the status and style for a due date.""" - if not due_date: - return "", "" - - today: datetime.date = datetime.now().date() - due_date: datetime.date = pd.to_datetime(due_date).date() - - # Type safe comparison: - if due_date < today: - return "❗ ", "overdue" - elif due_date <= today + timedelta(days=7): - return "⚠️ ", "warning" - return "", "" - - -def style_dataframe(df): - """Apply conditional styling to the dataframe based on due dates.""" - today = datetime.now().date() - - def row_style(row): - if 'due_date' not in row: - return [''] * len(row) - - due_date = row['due_date'].date() if isinstance(row['due_date'], pd.Timestamp) else datetime.strptime(row['due_date'], '%Y-%m-%d 00:00:00').date() - - if due_date < today: - return ['background-color: #f8d7da'] * len(row) - elif due_date <= today + timedelta(days=7): - return ['background-color: #fff3cd'] * len(row) - return [''] * len(row) - - return df.style.apply(row_style, axis=1) - - -def get_issue_page_link(id: str): - """Link to the issue detail page with id as query parameter""" - return f'/issue_detail?id={id}' - - -def render_issues_table( - df: pd.DataFrame, - with_pagination: bool = False, - rows_per_page: int = 5, - page_key: str = "page", -): - """ - Render a table of security issues with consistent styling and optional pagination. - - Args: - df: DataFrame containing the issues data - with_pagination: Whether to enable pagination (defaults to False) - rows_per_page: Number of rows per page when pagination is enabled - page_key: Key to use for the pagination state in session state - """ - if df.empty: - st.info("No issues to display.") - return - - # Add status indicators if not already present - if 'status_icon' not in df.columns and 'due_date' in df.columns: - df['status_icon'], df['status'] = zip(*df['due_date'].apply(get_due_date_status)) - - # Sort by due_date and cvss if present - if 'due_date' in df.columns and 'cvss' in df.columns: - df = df.sort_values(['due_date', 'cvss'], ascending=[True, False]) - - # Add issue_link column - df['issue_link'] = df['id'].apply(get_issue_page_link) - - # Handle pagination if enabled - if with_pagination: - total_pages = len(df) // rows_per_page + (1 if len(df) % rows_per_page > 0 else 0) - page = st.session_state.get(page_key, 0) - start_idx = page * rows_per_page - end_idx = start_idx + rows_per_page - display_df = style_dataframe(df.iloc[start_idx:end_idx].copy()) - else: - display_df = style_dataframe(df) - - # Define column configuration - more compact version matching scan details - column_config = { - "status_icon": st.column_config.TextColumn( - "", - width="small" - ), - "issue_link": st.column_config.LinkColumn( - "Details", - help="Click to view issue details", - display_text="View", - width="small", - ), - "benchmark": st.column_config.TextColumn( - "Benchmark", - width="medium" - ), - "finding_id": st.column_config.TextColumn( - "Finding ID", - width="small" - ), - "level": st.column_config.TextColumn( - "Level", - width="small" - ), - "cvss": st.column_config.NumberColumn( - "CVSS", - format="%.1f", - width="small" - ), - "title": st.column_config.TextColumn( - "Title", - width="large" - ), - "remediation_count": st.column_config.NumberColumn( - "Remediations", - width="small" - ), - "created_at": st.column_config.DateColumn( - "Created", - width="small" - ), - "resolved_at": st.column_config.DateColumn( - "Resolved", - width="small" - ), - "due_date": st.column_config.DateColumn( - "Due Date", - format="YYYY-MM-DD", - width="small" - ), - "failure": st.column_config.TextColumn( - "Failure", - width="large" - ), - "first_seen": st.column_config.DateColumn( - "First Seen", - width="small" - ) - } - - # Only include columns that exist in the dataframe - filtered_config = {k: v for k, v in column_config.items() if k in df.columns} - - # Define default column order - matching scan details layout - default_columns = [ - "status_icon", - "issue_link", - "benchmark", "finding_id", - "level", "cvss", "title", "remediation_count", - "created_at", "resolved_at", "due_date", "failure", "first_seen" - ] - - # Filter column order to only include columns that exist in the dataframe - column_order = [col for col in default_columns if col in df.columns] - - # Display the dataframe with compact styling - st.dataframe( - display_df, - column_config=filtered_config, - hide_index=True, - column_order=column_order, - use_container_width=True - ) - - # Render pagination controls if enabled - if with_pagination: - col1, col2, col3 = st.columns([1, 2, 1]) - with col1: - if st.button("← Previous", disabled=(page == 0)): - st.session_state[page_key] = max(0, page - 1) - st.rerun() - with col2: - st.markdown(f"Page {page + 1} of {total_pages}") - with col3: - if st.button("Next →", disabled=(page >= total_pages - 1)): - st.session_state[page_key] = min(total_pages - 1, page + 1) - st.rerun() \ No newline at end of file diff --git a/app/components/__init__.py b/app/components/__init__.py deleted file mode 100644 index f9a0848..0000000 --- a/app/components/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -""" -Components package for data processing and utilities -""" \ No newline at end of file diff --git a/app/components/data_processor.py b/app/components/data_processor.py deleted file mode 100644 index dc24e10..0000000 --- a/app/components/data_processor.py +++ /dev/null @@ -1,374 +0,0 @@ -import pandas as pd -from datetime import datetime, timedelta -from app.database.schema import ( - get_db_connection, - get_or_create_benchmark, - get_or_create_scan, - insert_finding, - create_remediation, - link_finding_to_remediation, - mark_remediations_resolved_if_not_in_list, - mark_issues_as_resolved_if_no_open_remediations, - create_issue_for_remediations, - check_if_scan_date_is_newer_than_db_scans -) - -def get_cvss_range(cvss): - """Convert CVSS score to range category.""" - try: - score = float(cvss) - if score >= 9.0: - return "Critical" - elif score >= 7.0: - return "High" - elif score >= 4.0: - return "Medium" - elif score > 0: - return "Low" - else: - return "Info" - except (ValueError, TypeError): - if not cvss: - return "Info" - else: - return "Unknown" - - -def calculate_due_date(cvss, analysis_date): - """Calculate due date based on severity level.""" - severity_mapping = { - 'Critical': 15, - 'High': 30, - 'Medium': 90, - 'Low': 180, - 'Info': 180 - } - - time_to_resolve = severity_mapping.get(get_cvss_range(cvss), 180) - - return datetime.strptime(analysis_date, '%Y-%m-%d 00:00:00') + timedelta(days=time_to_resolve) - - - -def validate_columns(df, additional_columns=[]): - """Validate columns in the CSV file.""" - required_columns = [ - 'benchmark', 'id', 'level', 'cvss', 'title', - 'failures', 'description', 'rationale', 'refs' - ] - - # Validate columns - required_columns.extend(additional_columns) - if not all(col in df.columns for col in required_columns): - missing_cols = [col for col in required_columns if col not in df.columns] - raise ValueError(f"Missing required columns: {', '.join(missing_cols)}") - - - -def process_single_scan_upload(file, analysis_date): - """Process a single scan upload.""" - df = pd.read_csv(file) - return process_upload_dataframe(df, analysis_date) - - -def process_multiple_scan_upload(file): - """Process a multiple scan upload with multiple analysis dates.""" - df = pd.read_csv(file) - - column_name_map = { - 'benchmark': 'Benchmark', - 'id': 'CIS_ID', - 'level': 'Level', - 'cvss': 'CVSS', - 'title': 'Title', - 'failures': 'Failures', - 'description': 'Description', - 'rationale': 'Rationale', - 'refs': 'References' - } - - # Map from date to dataframe - results = { - 'new': 0, - 'existing': 0, - 'resolved': 0 - } - - sorted_dates = sorted(df['Date'].unique(), key=lambda x: datetime.strptime(x, '%m/%d/%Y')) - - for date in sorted_dates: - print(f"Processing date: {date}") - reformatted_date = datetime.strptime(date, '%m/%d/%Y').strftime('%Y-%m-%d 00:00:00') - - this_data = df[df['Date'] == date] - this_update = process_upload_dataframe(this_data, reformatted_date, column_name_map) - results['new'] += this_update['new'] - results['existing'] += this_update['existing'] - results['resolved'] += this_update['resolved'] - - - return results - - - -def process_upload_dataframe(df: pd.DataFrame, analysis_date, column_name_map=None) -> dict: - """Process a dataframe of uploaded CSV file and update database.""" - - conn = get_db_connection() - - if not check_if_scan_date_is_newer_than_db_scans(conn, datetime.strptime(analysis_date, '%Y-%m-%d 00:00:00')): - print(f"Scan on {analysis_date} predates the most recent scan; skipping") - return { - 'new': 0, - 'existing': 0, - 'resolved': 0 - } - - required_columns = [ - 'benchmark', 'id', 'level', 'cvss', 'title', - 'failures', 'description', 'rationale', 'refs' - ] - - if not column_name_map: - column_name_map = { x: x for x in required_columns } - - # Validate columns - if not all(column_name_map.get(col) in df.columns for col in required_columns): - missing_cols = [column_name_map.get(col) for col in required_columns if column_name_map.get(col) not in df.columns] - raise ValueError(f"Missing required columns: {', '.join(missing_cols)}") - - active_finding_ids = set() - new_count = 0 - existing_count = 0 - - try: - # Get or create scan record - scan_id = get_or_create_scan(conn, analysis_date) - - active_remediation_ids = set() - remediations_needing_issues = {} - - # Process each row - for _, row in df.iterrows(): - # Create benchmark record - benchmark_data = ( - row[column_name_map['benchmark']], - row[column_name_map['id']], # This is the finding_id from CSV - row[column_name_map['level']], - float(row[column_name_map['cvss']]) if row[column_name_map['cvss']] else 0.0, - row[column_name_map['title']], - row[column_name_map['description']], - row[column_name_map['rationale']], - row[column_name_map['refs']] - ) - benchmark_id = get_or_create_benchmark(conn, benchmark_data) - - # Handle multiple failures - failures = str(row[column_name_map['failures']]).split('\n') - for failure in failures: - failure = failure.strip() - if not failure: - continue - - # Insert finding - finding_id = insert_finding(conn, benchmark_id, failure) - active_finding_ids.add(finding_id) - - # Check if finding is already part of an open remediation - cursor = conn.cursor() - cursor.execute(''' - SELECT r.id - FROM remediations r - JOIN remediation_findings rf ON r.id = rf.remediation_id - WHERE r.benchmark_id = ? - AND r.state = 'open' - AND rf.finding_id = ? - ''', (benchmark_id, finding_id)) - - existing_remediation = cursor.fetchone() - - if existing_remediation: - active_remediation_ids.add(existing_remediation[0]) - existing_count += 1 - else: - # Create new remediation - due_date = calculate_due_date(row[column_name_map['cvss']], analysis_date) - remediation_id = create_remediation(conn, benchmark_id, scan_id, due_date) - link_finding_to_remediation(conn, remediation_id, finding_id) - - # Add this remediation to the list of active remediations: - active_remediation_ids.add(remediation_id) - - # Add this remediation to the list of remediations needing issues: - remediations_for_this_benchmark: list[int] = remediations_needing_issues.get((benchmark_id, due_date), []) - remediations_for_this_benchmark.append(remediation_id) - remediations_needing_issues[(benchmark_id, due_date)] = remediations_for_this_benchmark - new_count += 1 - - for (benchmark_id, due_date), remediation_ids in remediations_needing_issues.items(): - create_issue_for_remediations(conn, remediation_ids, benchmark_id, created_at=analysis_date, due_date=due_date) - - # Mark findings as resolved if they're not in current upload - mark_remediations_resolved_if_not_in_list(conn, scan_id, active_remediation_ids) - - # Mark issues as resolved which no longer have any open remediations - mark_issues_as_resolved_if_no_open_remediations(conn, analysis_date) - - # Get count of resolved findings in this scan - cursor = conn.cursor() - cursor.execute(''' - SELECT COUNT(*) as count - FROM remediations - WHERE resolved_in_scan = ? - ''', (scan_id,)) - resolved_count = cursor.fetchone()['count'] - - conn.commit() - return { - 'new': new_count, - 'existing': existing_count, - 'resolved': resolved_count - } - - finally: - conn.close() - -def get_findings_summary(): - """Get summary of findings for display.""" - conn = get_db_connection() - try: - cursor = conn.cursor() - today = datetime.now().date() - - # Get active findings count - cursor.execute(''' - SELECT COUNT(DISTINCT r.id) as count - FROM remediations r - WHERE r.state = 'open' - ''') - active_count = cursor.fetchone()['count'] - - # Get resolved findings count - cursor.execute(''' - SELECT COUNT(DISTINCT r.id) as count - FROM remediations r - WHERE r.state = 'resolved' - ''') - resolved_count = cursor.fetchone()['count'] - - # Get findings by severity - cursor.execute(''' - SELECT b.level, COUNT(DISTINCT r.id) as count - FROM remediations r - JOIN benchmark b ON r.benchmark_id = b.id - WHERE r.state = 'open' - GROUP BY b.level - ''') - severity_counts = {row['level']: row['count'] for row in cursor.fetchall()} - - # Get findings by CVSS - cursor.execute(''' - SELECT b.cvss - FROM remediations r - JOIN benchmark b ON r.benchmark_id = b.id - WHERE r.state = 'open' - ''') - cvss_scores = [row['cvss'] for row in cursor.fetchall()] - cvss_ranges = [get_cvss_range(score) for score in cvss_scores] - cvss_counts = pd.Series(cvss_ranges).value_counts().to_dict() - - # Get findings due within 28 days - today_str = today.strftime('%Y-%m-%d') - twenty_eight_days = (today + timedelta(days=28)).strftime('%Y-%m-%d') - cursor.execute(''' - SELECT COUNT(DISTINCT r.id) as count - FROM remediations r - WHERE r.state = 'open' - AND r.due_date <= ? - AND r.due_date > ? - ''', (twenty_eight_days, today_str)) - due_within_28_days = cursor.fetchone()['count'] - - # Get findings due this week - week_end = (today + timedelta(days=7)).strftime('%Y-%m-%d') - cursor.execute(''' - SELECT COUNT(DISTINCT r.id) as count - FROM remediations r - WHERE r.state = 'open' - AND r.due_date <= ? - AND r.due_date > ? - ''', (week_end, today_str)) - due_this_week = cursor.fetchone()['count'] - - # Get overdue findings - cursor.execute(''' - SELECT COUNT(DISTINCT r.id) as count - FROM remediations r - WHERE r.state = 'open' - AND r.due_date < ? - ''', (today_str,)) - overdue_count = cursor.fetchone()['count'] - - # Get findings due after 28 days - cursor.execute(''' - SELECT COUNT(DISTINCT r.id) as count - FROM remediations r - WHERE r.state = 'open' - AND r.due_date > ? - ''', (twenty_eight_days,)) - due_after_28_days = cursor.fetchone()['count'] - - return { - 'active_count': active_count, - 'resolved_count': resolved_count, - 'severity_counts': severity_counts, - 'cvss_counts': cvss_counts, - 'due_within_28_days': due_within_28_days, - 'due_this_week': due_this_week, - 'overdue_count': overdue_count, - 'due_after_28_days': due_after_28_days - } - - finally: - conn.close() - -def export_issues_to_df(status='open'): - """Export issues to a DataFrame.""" - conn = get_db_connection() - cursor = conn.cursor() - - - sql = ''' - SELECT - i.id, - i.due_date, - i.created_at, - i.resolved_at, - i.status, - b.benchmark, - b.finding_id, - b.level, - b.cvss, - b.title, - b.description, - b.rationale, - b.refs - FROM issues i - JOIN benchmark b ON i.benchmark_id = b.id - WHERE i.status = ? - ''' - - cursor.execute(sql, (status,)) - - # Convert list of dictionaries to DataFrame - issues = [dict(row) for row in cursor.fetchall()] - - df = pd.DataFrame.from_records(issues) - - # Convert date strings to datetime objects - date_columns = ['due_date', 'created_at', 'resolved_at'] - for col in date_columns: - if col in df.columns: - df[col] = pd.to_datetime(df[col]) - - return df diff --git a/app/components/poam_exports.py b/app/components/poam_exports.py deleted file mode 100644 index bcb3b9d..0000000 --- a/app/components/poam_exports.py +++ /dev/null @@ -1,118 +0,0 @@ -import pandas as pd -from datetime import datetime -from app.database.schema import get_db_connection, get_poam_config -from app.components.data_processor import get_cvss_range - -def get_poam_id(created_at, issue_id): - """Generate a POAM ID in the format YYYY-CISxxxx.""" - year = datetime.strptime(created_at, '%Y-%m-%d %H:%M:%S').year - return f"{year}-CIS{issue_id:04d}" - -def get_weakness_detector_source(scan_date): - """Generate the weakness detector source string.""" - return f"Hail CIS GCP/GKE Compliance (rolling). Scan on {scan_date}" - -def get_asset_identifier(google_project, findings): - """Generate the asset identifier string.""" - return f"[{google_project}]: {findings}" - -def get_planned_milestones(due_date): - """Generate the planned milestones string.""" - return f"(1) {due_date}: Perform required updates to affected assets to remediate this finding." - -def get_issue_findings(conn, issue_id): - """Get all findings associated with an issue.""" - cursor = conn.cursor() - cursor.execute(''' - SELECT DISTINCT f.failure - FROM findings f - JOIN remediation_findings rf ON f.id = rf.finding_id - JOIN issue_remediations ir ON rf.remediation_id = ir.remediation_id - WHERE ir.issue_id = ? - ''', (issue_id,)) - return [row['failure'] for row in cursor.fetchall()] - -def get_first_scan_date(conn, issue_id): - """Get the first scan date for an issue.""" - cursor = conn.cursor() - cursor.execute(''' - SELECT MIN(s.scan_date) as first_scan_date - FROM scans s - JOIN remediations r ON s.id = r.first_seen_scan - JOIN issue_remediations ir ON r.id = ir.remediation_id - WHERE ir.issue_id = ? - ''', (issue_id,)) - result = cursor.fetchone() - return result['first_scan_date'] if result else None - -def generate_poam_export(status='open'): - """Generate a POAM export for issues with the specified status.""" - conn = get_db_connection() - config = get_poam_config(conn) - - if not config: - raise ValueError("POAM configuration not found. Please configure POAM export settings first.") - - # Get issues with the specified status - cursor = conn.cursor() - cursor.execute(''' - SELECT - i.id, - i.created_at, - i.due_date, - i.status, - b.title, - b.description, - b.cvss - FROM issues i - JOIN benchmark b ON i.benchmark_id = b.id - WHERE i.status = ? - ORDER BY i.created_at ASC - ''', (status,)) - - issues = cursor.fetchall() - - # Prepare data for export - export_data = [] - for issue in issues: - findings = get_issue_findings(conn, issue['id']) - findings_str = "\n".join(findings) - first_scan_date = get_first_scan_date(conn, issue['id']) - - export_data.append({ - 'POAM ID': get_poam_id(issue['created_at'], issue['id']), - 'Controls': 'CM-6', - 'Weakness Name': issue['title'], - 'Weakness Description': issue['description'], - 'Weakness Detector Source': get_weakness_detector_source(first_scan_date), - 'Weakness Source Identifier': 'CIS', - 'Asset Identifier': get_asset_identifier(config['google_project'], findings_str), - 'Point of Contact': config['point_of_contact'], - 'Resources Required': 'None', - 'Overall Remediation Plan': 'Perform necessary updates to resolve the vulnerability', - 'Original Detection Date': issue['created_at'], - 'Scheduled Completion Date': issue['due_date'], - 'AGENCY Scheduled Completion Date': issue['due_date'], - 'Planned Milestones': get_planned_milestones(issue['due_date']), - 'Milestone Changes': '', - 'Status Date': datetime.now().strftime('%Y-%m-%d'), - 'Vendor Dependency': 'No', - 'Last Vendor Check-in date': '', - 'Vendor Dependent Product Name': '', - 'Original Risk Rating': get_cvss_range(issue['cvss']), - 'Adjusted Risk Rating': 'N/A', - 'Risk Adjustment': 'No', - 'False Positive': 'No', - 'Operational Requirement': 'No', - 'Deviation Rationale': '', - 'Supporting Documents': '', - 'Comments': '', - 'Auto-Approve': 'No', - 'Binding Operational Directive 22-01 tracking': 'No', - 'Binding Operational Directive 22-01 Due Date': '', - 'CVE': '', - 'Service Name': config['service_name'] - }) - - conn.close() - return pd.DataFrame(export_data) \ No newline at end of file diff --git a/app/database/__init__.py b/app/database/__init__.py deleted file mode 100644 index fed03e4..0000000 --- a/app/database/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -""" -Database package for SQLite operations and schema -""" \ No newline at end of file diff --git a/app/database/schema.py b/app/database/schema.py deleted file mode 100644 index 859925b..0000000 --- a/app/database/schema.py +++ /dev/null @@ -1,342 +0,0 @@ -import sqlite3 -import os -from datetime import datetime, timedelta - -DB_PATH = "app/database/findings.db" - -def get_db_connection(): - """Create a database connection.""" - os.makedirs(os.path.dirname(DB_PATH), exist_ok=True) - conn = sqlite3.connect(DB_PATH) - conn.row_factory = sqlite3.Row - return conn - -def init_db(): - """Initialize the database with required tables.""" - conn = get_db_connection() - cursor = conn.cursor() - - # Create benchmark table - cursor.execute(''' - CREATE TABLE IF NOT EXISTS benchmark ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - benchmark TEXT NOT NULL, - finding_id TEXT NOT NULL, - level TEXT, - cvss REAL, - title TEXT, - description TEXT, - rationale TEXT, - refs TEXT, - UNIQUE(benchmark, finding_id) - ) - ''') - - # Create findings table - cursor.execute(''' - CREATE TABLE IF NOT EXISTS findings ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - benchmark_id INTEGER NOT NULL, - failure TEXT NOT NULL, - FOREIGN KEY (benchmark_id) REFERENCES benchmark(id), - UNIQUE(benchmark_id, failure) - ) - ''') - - # Create scans table - cursor.execute(''' - CREATE TABLE IF NOT EXISTS scans ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - scan_date DATE NOT NULL UNIQUE - ) - ''') - - # Create remediations table - cursor.execute(''' - CREATE TABLE IF NOT EXISTS remediations ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - benchmark_id INTEGER NOT NULL, - first_seen_scan INTEGER NOT NULL, - resolved_in_scan INTEGER, - state TEXT NOT NULL CHECK (state IN ('open', 'resolved')), - due_date DATE NOT NULL, - FOREIGN KEY (benchmark_id) REFERENCES benchmark(id), - FOREIGN KEY (first_seen_scan) REFERENCES scans(id), - FOREIGN KEY (resolved_in_scan) REFERENCES scans(id) - ) - ''') - - # Create remediation_findings table - cursor.execute(''' - CREATE TABLE IF NOT EXISTS remediation_findings ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - remediation_id INTEGER NOT NULL, - finding_id INTEGER NOT NULL, - FOREIGN KEY (remediation_id) REFERENCES remediations(id), - FOREIGN KEY (finding_id) REFERENCES findings(id), - UNIQUE (remediation_id, finding_id) - ) - ''') - - # Create issues table - cursor.execute(''' - CREATE TABLE IF NOT EXISTS issues ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - benchmark_id INTEGER NOT NULL, - due_date DATE NOT NULL, - created_at TIMESTAMP NOT NULL, - status TEXT NOT NULL CHECK (status IN ('open', 'resolved')), - resolved_at TIMESTAMP, - FOREIGN KEY (benchmark_id) REFERENCES benchmark(id) - ) - ''') - - # Create issue_remediations table - cursor.execute(''' - CREATE TABLE IF NOT EXISTS issue_remediations ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - issue_id INTEGER NOT NULL, - remediation_id INTEGER NOT NULL, - FOREIGN KEY (issue_id) REFERENCES issues(id), - FOREIGN KEY (remediation_id) REFERENCES remediations(id), - UNIQUE (issue_id, remediation_id) - ) - ''') - - # Create poam_config table - cursor.execute(''' - CREATE TABLE IF NOT EXISTS poam_config ( - id INTEGER PRIMARY KEY, - point_of_contact TEXT NOT NULL, - google_project TEXT NOT NULL, - service_name TEXT NOT NULL - ) - ''') - - conn.commit() - conn.close() - -def get_or_create_benchmark(conn, benchmark_data): - """Get existing benchmark or create a new one.""" - cursor = conn.cursor() - cursor.execute(''' - INSERT OR IGNORE INTO benchmark ( - benchmark, finding_id, level, cvss, title, - description, rationale, refs - ) VALUES (?, ?, ?, ?, ?, ?, ?, ?) - ''', benchmark_data) - - cursor.execute(''' - SELECT id FROM benchmark - WHERE benchmark = ? AND finding_id = ? - ''', (benchmark_data[0], benchmark_data[1])) - - return cursor.fetchone()['id'] - -def get_or_create_scan(conn, scan_date): - """Get existing scan or create a new one.""" - cursor = conn.cursor() - cursor.execute(''' - INSERT OR IGNORE INTO scans (scan_date) - VALUES (?) - ''', (scan_date,)) - - cursor.execute(''' - SELECT id FROM scans - WHERE scan_date = ? - ''', (scan_date,)) - - return cursor.fetchone()['id'] - -def insert_finding(conn, benchmark_id, failure): - """Insert a new finding into the database.""" - cursor = conn.cursor() - cursor.execute(''' - INSERT OR IGNORE INTO findings (benchmark_id, failure) - VALUES (?, ?) - ''', (benchmark_id, failure)) - - cursor.execute(''' - SELECT id FROM findings - WHERE benchmark_id = ? AND failure = ? - ''', (benchmark_id, failure)) - - return cursor.fetchone()['id'] - -def create_remediation(conn, benchmark_id, scan_id, due_date): - """Create a new remediation.""" - cursor = conn.cursor() - cursor.execute(''' - INSERT INTO remediations ( - benchmark_id, first_seen_scan, state, due_date - ) VALUES (?, ?, 'open', ?) - ''', (benchmark_id, scan_id, due_date)) - - return cursor.lastrowid - -def link_finding_to_remediation(conn, remediation_id, finding_id): - """Link a finding to a remediation.""" - cursor = conn.cursor() - cursor.execute(''' - INSERT OR IGNORE INTO remediation_findings (remediation_id, finding_id) - VALUES (?, ?) - ''', (remediation_id, finding_id)) - -def check_if_scan_exists(conn, scan_date): - """Check if a scan exists for a given date.""" - cursor = conn.cursor() - cursor.execute(''' - SELECT id FROM scans WHERE scan_date = ? - ''', (scan_date,)) - return cursor.fetchone() is not None - -def check_if_scan_date_is_newer_than_db_scans(conn, scan_date): - """Check if a scan is newer than the most recent scan.""" - cursor = conn.cursor() - cursor.execute(''' - SELECT max(scan_date) FROM scans - ''') - - result = cursor.fetchone() - - if result is None or result[0] is None or result[0] == '': - print(f"** No scans found; assuming {scan_date} is the most recent scan") - return True - else: - db_scan_date_object = datetime.strptime(result[0], '%Y-%m-%d 00:00:00') - if db_scan_date_object.date() < scan_date.date(): - print(f"** Scan on {scan_date} is newer than the most recent scan ({result[0]}); processing") - return True - else: - print(f"** Scan on {scan_date} is older than the most recent scan ({result[0]}); skipping") - return False - - -def get_active_issues(conn): - """Get all active issues.""" - cursor = conn.cursor() - cursor.execute(''' - SELECT - b.benchmark, - b.finding_id, - b.level, - b.cvss, - b.title, - b.description, - b.rationale, - b.refs, - i.due_date, - i.created_at, - i.resolved_at, - i.status, - COUNT(ir.remediation_id) as remediation_count - FROM issues i - JOIN issue_remediations ir ON i.id = ir.issue_id - JOIN benchmark b ON i.benchmark_id = b.id - WHERE i.status = 'open' - GROUP BY i.id - ''') - - # Convert rows to dictionaries with column names - columns = [column[0] for column in cursor.description] - return [dict(zip(columns, row)) for row in cursor.fetchall()] - - -def mark_remediations_resolved_if_not_in_list(conn, scan_id, active_remediation_ids): - """Mark remediations as resolved if they're not in the current scan.""" - - print(f"Marking all but {len(active_remediation_ids)} still-active remediations as resolved") - - if not active_remediation_ids: - # No active remediations - update all open remediations to resolved - cursor = conn.cursor() - cursor.execute(''' - UPDATE remediations - SET state = 'resolved' - WHERE state = 'open' - ''') - else: - cursor = conn.cursor() - placeholders = ','.join(['?' for _ in active_remediation_ids]) - cursor.execute(f''' - UPDATE remediations - SET state = 'resolved', - resolved_in_scan = ? - WHERE state = 'open' - AND id NOT IN ({placeholders}) - ''', [scan_id, *active_remediation_ids]) - - conn.commit() - - -def mark_issues_as_resolved_if_no_open_remediations(conn, resolved_date): - """Mark issues as resolved if they no longer have any open remediations.""" - cursor = conn.cursor() - cursor.execute(''' - SELECT i.id - FROM issues i - WHERE i.status = 'open' - AND NOT EXISTS ( - SELECT 1 - FROM remediations r - JOIN issue_remediations ir ON r.id = ir.remediation_id - WHERE r.state = 'open' - AND ir.issue_id = i.id - ) - ''', ()) - - issue_ids = [row[0] for row in cursor.fetchall()] - print(f"Marking {len(issue_ids)} issues as resolved ({issue_ids})") - placeholders = ','.join(['?' for _ in issue_ids]) - - if issue_ids: - cursor.execute(f''' - UPDATE issues - SET status = 'resolved', resolved_at = ? - WHERE id IN ({placeholders}) - ''', (resolved_date, *issue_ids)) - - conn.commit() - - -def create_issue_for_remediations(conn, remediation_ids, benchmark_id, created_at, due_date): - """Create a new issue for a remediation if one doesn't exist.""" - cursor = conn.cursor() - cursor.execute(''' - INSERT INTO issues (benchmark_id, due_date, created_at, status) - VALUES (?, ?, ?, 'open') - ''', (benchmark_id, due_date, created_at)) - - issue_id = cursor.lastrowid - - for remediation_id in remediation_ids: - cursor.execute(''' - INSERT INTO issue_remediations (issue_id, remediation_id) - VALUES (?, ?) - ''', (issue_id, remediation_id)) - - conn.commit() - -def get_poam_config(conn): - """Get the POAM export configuration.""" - cursor = conn.cursor() - cursor.execute('SELECT * FROM poam_config LIMIT 1') - result = cursor.fetchone() - if result: - return dict(result) - return None - -def update_poam_config(conn, point_of_contact, google_project, service_name): - """Update the POAM export configuration.""" - cursor = conn.cursor() - cursor.execute(''' - INSERT OR REPLACE INTO poam_config ( - id, point_of_contact, google_project, service_name - ) VALUES ( - 1, ?, ?, ? - ) - ''', (point_of_contact, google_project, service_name)) - conn.commit() - -# Initialize the database when the module is imported -init_db() \ No newline at end of file diff --git a/app/main.py b/app/main.py deleted file mode 100644 index a319c22..0000000 --- a/app/main.py +++ /dev/null @@ -1,204 +0,0 @@ -import streamlit as st -import pandas as pd -from datetime import datetime, timedelta -import plotly.express as px -from app.components.data_processor import ( - process_single_scan_upload, - process_multiple_scan_upload, - get_findings_summary -) - -from app.components.IssuesList import render_issues_list -import os -import logging -# Set page config -st.set_page_config( - page_title="Security Findings Tracker", - page_icon="🔒", - layout="wide" -) - -# Initialize session state variables -if 'open_modal' not in st.session_state: - st.session_state.open_modal = False -if 'show_success' not in st.session_state: - st.session_state.show_success = False -if 'upload_results' not in st.session_state: - st.session_state.upload_results = {} - -# Load custom CSS -with open(os.path.join(os.path.dirname(__file__), "static/style.css")) as f: - st.markdown(f"", unsafe_allow_html=True) - -def get_due_date_status(due_date: pd.Timestamp): - """Return the status and style for a due date.""" - if not due_date: - return "", "" - - today: datetime.date = datetime.now().date() - due_date: datetime.date = pd.to_datetime(due_date).date() - - # Type safe comparison: - if due_date < today: - return "❗ ", "overdue" - elif due_date <= today + timedelta(days=7): - return "⚠️ ", "warning" - return "", "" - - -# Header with Upload Button -col1, col2 = st.columns([3, 1]) -with col1: - st.markdown(""" -
-

Security Findings Tracker

-
- """, unsafe_allow_html=True) -with col2: - if st.button("Upload Findings", type="primary", use_container_width=True): - st.session_state.open_modal = True - -# Upload Modal Dialog -@st.dialog("Upload Security Findings") -def show_upload_dialog(): - # Show a dropdown to select the type of upload - upload_type = st.selectbox("Upload Type", ["Single Scan", "Multiple Scans"]) - st.subheader("Upload Single Scan") - uploaded_file = st.file_uploader("Select CSV File", type="csv", key="file_uploader") - - if upload_type == "Single Scan": - analysis_date = st.date_input( - "Analysis Date", - value=datetime.now().date(), - key="analysis_date", - help="Date when the security analysis was performed" - ) - - col1, col2 = st.columns([1, 1]) - with col1: - if st.button("Cancel", type="secondary"): - st.session_state.open_modal = False - st.rerun() - with col2: - if uploaded_file is not None: - if st.button("Confirm Upload", type="primary"): - if upload_type == "Single Scan": - results = process_single_scan_upload(uploaded_file, analysis_date.strftime('%Y-%m-%d 00:00:00')) - elif upload_type == "Multiple Scans": - results = process_multiple_scan_upload(uploaded_file) - st.session_state.open_modal = False - st.session_state.show_success = True - st.session_state.upload_results = results - st.rerun() - -# Call the dialog function -if st.session_state.open_modal: - show_upload_dialog() - -# Show success message after upload -if st.session_state.pop("show_success", False): - results = st.session_state.pop("upload_results", {}) - st.success("File processed successfully!") - st.markdown("### Upload Summary") - st.markdown(f""" - - New Findings: {results['new']} - - Updated Findings: {results['existing']} - - Resolved Findings: {results['resolved']} - """) - -# Main Content -summary = get_findings_summary() - -# Charts Section -col1, col2, col3, col4 = st.columns(4) - -with col1: - with st.container(border=True): - # Active Findings - st.markdown(f""" -
-
{summary['active_count']}
-
Active Findings
-
- """, unsafe_allow_html=True) - - # Due This Week - due_this_week = summary.get('due_this_week', 0) - st.markdown(f""" -
-
{'⚠️ ' if due_this_week > 0 else ''}{due_this_week}
-
Due This Week
-
- """, unsafe_allow_html=True) - - # Overdue Findings - overdue_count = summary.get('overdue_count', 0) - st.markdown(f""" -
-
{'❗ ' if overdue_count > 0 else ''}{overdue_count}
-
Overdue Findings
-
- """, unsafe_allow_html=True) - -with col2: - # Level distribution chart - level_df = pd.DataFrame([ - {'Level': k, 'Count': v} - for k, v in summary['severity_counts'].items() - ]) - - if not level_df.empty: - fig1 = px.pie( - level_df, - values='Count', - names='Level', - title='Active Findings by Level', - color='Level', - color_discrete_map={ - 'critical': '#dc3545', - 'high': '#fd7e14', - 'medium': '#ffc107', - 'low': '#20c997', - 'info': '#0dcaf0' - } - ) - fig1.update_traces(textposition='inside', textinfo='percent+label') - st.plotly_chart(fig1, use_container_width=True) - -with col3: - # CVSS distribution chart - if 'cvss_counts' in summary: - cvss_df = pd.DataFrame([ - {'CVSS Range': k, 'Count': v} - for k, v in summary['cvss_counts'].items() - ]) - - if not cvss_df.empty: - fig2 = px.pie( - cvss_df, - values='Count', - names='CVSS Range', - title='Active Findings by CVSS', - color='CVSS Range', - color_discrete_sequence=px.colors.sequential.RdBu - ) - fig2.update_traces(textposition='inside', textinfo='percent+label') - st.plotly_chart(fig2, use_container_width=True) - -with col4: - # Due by date chart - overdue_count = summary.get('overdue_count', 0) - due_this_week = summary.get('due_this_week', 0) - due_within_28_days = summary.get('due_within_28_days', 0) - due_after_28_days = summary.get('due_after_28_days', 0) - - fig3 = px.pie( - values=[overdue_count, due_this_week, due_within_28_days, due_after_28_days], - names=['Overdue', 'Due This Week', 'Due Within 28 Days', 'Due After 28 Days'], - title='Active Findings by Due Date', - color_discrete_sequence=px.colors.sequential.RdBu - ) - st.plotly_chart(fig3, use_container_width=True) - -# Issues List with Tabs -render_issues_list() diff --git a/app/pages/1_issue_detail.py b/app/pages/1_issue_detail.py deleted file mode 100644 index e9ee799..0000000 --- a/app/pages/1_issue_detail.py +++ /dev/null @@ -1,170 +0,0 @@ -import streamlit as st -import pandas as pd -from datetime import datetime -import sqlite3 -from app.database.schema import get_db_connection -from app.components.IssuesList import render_issues_list - -st.set_page_config( - page_title="Issue Details", - page_icon="🔒", - layout="wide" -) - -def get_issue_details(issue_id): - """Get detailed information about a specific issue.""" - conn = get_db_connection() - cursor = conn.cursor() - - # Get issue and benchmark details - cursor.execute(''' - SELECT - i.id, - i.due_date, - i.created_at, - i.resolved_at, - i.status, - b.benchmark, - b.finding_id, - b.level, - b.cvss, - b.title, - b.description, - b.rationale, - b.refs - FROM issues i - JOIN benchmark b ON i.benchmark_id = b.id - WHERE i.id = ? - ''', (issue_id,)) - - issue = cursor.fetchone() - if not issue: - return None, None - - # Get all remediations for this issue - cursor.execute(''' - SELECT - r.id, - r.state, - r.due_date, - s_first.scan_date as first_seen, - s_resolved.scan_date as resolved_date, - f.failure, - b.benchmark, - b.finding_id - FROM remediations r - JOIN issue_remediations ir ON r.id = ir.remediation_id - JOIN remediation_findings rf ON r.id = rf.remediation_id - JOIN findings f ON rf.finding_id = f.id - JOIN benchmark b ON r.benchmark_id = b.id - LEFT JOIN scans s_first ON r.first_seen_scan = s_first.id - LEFT JOIN scans s_resolved ON r.resolved_in_scan = s_resolved.id - WHERE ir.issue_id = ? - ORDER BY r.due_date ASC - ''', (issue_id,)) - - remediations = cursor.fetchall() - conn.close() - - return issue, remediations - -# Get issue ID from URL parameters -issue_id = st.query_params.get("id", None) - -# Basic breadcrumbs: -if issue_id: - st.markdown(f""" -
- Issues - > - {issue_id} -
-""", unsafe_allow_html=True) - - - -if not issue_id: - st.title("Security Issues") - render_issues_list() -else: - issue, remediations = get_issue_details(issue_id) - - if not issue: - st.error(f"Issue {issue_id} not found.") - st.stop() - - # Display issue details - st.title(f"{issue['title']}") - - # Issue metadata - col1, col2, col3 = st.columns(3) - with col1: - st.metric("CVSS Score", f"{issue['cvss']:.1f}") - with col2: - st.metric("Level", issue['level']) - with col3: - st.metric("Status", issue['status'].upper()) - - # Dates and timeline - st.divider() - date_col1, date_col2, date_col3 = st.columns(3) - with date_col1: - st.markdown(f"**Created:** {issue['created_at']}") - with date_col2: - st.markdown(f"**Due Date:** {issue['due_date']}") - with date_col3: - if issue['resolved_at']: - st.markdown(f"**Resolved:** {issue['resolved_at']}") - - # Remediations table - st.divider() - st.subheader("Remediations") - - if remediations: - remediation_data = [] - for r in remediations: - remediation_data.append({ - 'ID': r['id'], - 'State': r['state'].upper(), - 'Due Date': r['due_date'], - 'First Seen': r['first_seen'], - 'Resolved Date': r['resolved_date'] or '', - 'Finding ID': r['finding_id'], - 'Failure': r['failure'] - }) - - df = pd.DataFrame(remediation_data) - - st.dataframe( - df, - column_config={ - 'ID': st.column_config.NumberColumn('ID', width='small'), - 'State': st.column_config.TextColumn('State', width='small'), - 'Due Date': st.column_config.DateColumn('Due Date', width='small'), - 'First Seen': st.column_config.DateColumn('First Seen', width='small'), - 'Resolved Date': st.column_config.DateColumn('Resolved Date', width='small'), - 'Finding ID': st.column_config.TextColumn('Finding ID', width='medium'), - 'Failure': st.column_config.TextColumn('Failure', width='large') - }, - hide_index=True, - use_container_width=True - ) - else: - st.info("No remediations found for this issue.") - - # Description and rationale - st.divider() - st.subheader("Description") - st.write(issue['description']) - - if issue['rationale']: - st.subheader("Rationale") - st.write(issue['rationale']) - - # References - if issue['refs']: - st.divider() - st.subheader("References") - for ref in issue['refs'].split('\n'): - if ref.strip(): - st.markdown(f"- [{ref}]({ref})") diff --git a/app/pages/2_scan_history.py b/app/pages/2_scan_history.py deleted file mode 100644 index 161bcae..0000000 --- a/app/pages/2_scan_history.py +++ /dev/null @@ -1,176 +0,0 @@ -import streamlit as st -import pandas as pd -from app.database.schema import get_db_connection -from app.components.IssuesTable import render_issues_table -st.set_page_config( - page_title="Scan History", - page_icon="🔒", - layout="wide" -) - -def get_all_scans(): - """Get list of all scans ordered by date.""" - conn = get_db_connection() - cursor = conn.cursor() - cursor.execute(''' - SELECT id, scan_date - FROM scans - ORDER BY scan_date DESC - ''') - return [dict(row) for row in cursor.fetchall()] - -def get_scan_details(scan_id): - """Get detailed information about a specific scan.""" - conn = get_db_connection() - cursor = conn.cursor() - - # Get basic scan info - cursor.execute(''' - SELECT id, scan_date - FROM scans - WHERE id = ? - ''', (scan_id,)) - scan = cursor.fetchone() - - if not scan: - return None, None, None, None - - # Get new issues from this scan - cursor.execute(''' - SELECT - i.id, - i.due_date, - i.created_at, - b.benchmark, - b.finding_id, - b.level, - b.cvss, - b.title, - COUNT(r.id) as remediation_count - FROM issues i - JOIN benchmark b ON i.benchmark_id = b.id - JOIN issue_remediations ir ON i.id = ir.issue_id - JOIN remediations r ON ir.remediation_id = r.id - WHERE DATE(i.created_at) = DATE(?) - GROUP BY i.id - ORDER BY b.cvss DESC - ''', (scan['scan_date'],)) - new_issues = [dict(row) for row in cursor.fetchall()] - - # Get issues closed in this scan - cursor.execute(''' - SELECT - i.id, - i.due_date, - i.created_at, - i.resolved_at, - b.benchmark, - b.finding_id, - b.level, - b.cvss, - b.title, - COUNT(r.id) as remediation_count - FROM issues i - JOIN benchmark b ON i.benchmark_id = b.id - JOIN issue_remediations ir ON i.id = ir.issue_id - JOIN remediations r ON ir.remediation_id = r.id - WHERE DATE(i.resolved_at) = DATE(?) - GROUP BY i.id - ORDER BY b.cvss DESC - ''', (scan['scan_date'],)) - closed_issues = [dict(row) for row in cursor.fetchall()] - - # Get issues with remediations which were resolved in this scan - cursor.execute(''' - SELECT - i.id, - i.due_date, - i.created_at, - i.resolved_at, - b.benchmark, - b.finding_id, - b.level, - b.cvss, - b.title, - COUNT(DISTINCT r.id) as remediation_count - FROM issues i - JOIN benchmark b ON i.benchmark_id = b.id - JOIN issue_remediations ir ON i.id = ir.issue_id - JOIN remediations r ON ir.remediation_id = r.id - WHERE EXISTS ( - SELECT 1 - FROM issues i2 - JOIN issue_remediations ir2 ON i2.id = ir2.issue_id - JOIN remediations r2 ON ir2.remediation_id = r2.id - WHERE i2.id = i.id - AND r2.resolved_in_scan = ? - ) - GROUP BY i.id - ORDER BY b.cvss DESC - ''', (scan_id,)) - issues_with_resolved_findings = [dict(row) for row in cursor.fetchall()] - - conn.close() - return scan, new_issues, closed_issues, issues_with_resolved_findings - -# Get list of all scans -scans = get_all_scans() - -if not scans: - st.error("No scans found in the database.") - st.stop() - -# Scan selector -selected_scan = st.selectbox( - "Select Scan", - options=scans, - format_func=lambda x: x['scan_date'], - key="scan_selector" -) - -if selected_scan: - scan, new_issues, closed_issues, issues_with_resolved_findings = get_scan_details(selected_scan['id']) - - if not scan: - st.error(f"Scan {selected_scan['id']} not found.") - st.stop() - - # Display scan summary - st.title(f"Scan Details: {scan['scan_date']}") - - # Summary metrics - col1, col2, col3 = st.columns(3) - with col1: - st.metric("New Issues", len(new_issues)) - with col2: - st.metric("Closed Issues", len(closed_issues)) - with col3: - st.metric("Issues with Resolved Findings", len(issues_with_resolved_findings)) - - # New Issues - st.divider() - st.subheader("New Issues") - if new_issues: - new_issues_df = pd.DataFrame(new_issues) - new_issues_df['issue_link'] = [f"/issue_detail?id={issue['id']}" for issue in new_issues] - render_issues_table(new_issues_df) - else: - st.info("No new issues in this scan.") - - # Closed Issues - st.divider() - st.subheader("Closed Issues") - if closed_issues: - closed_issues_df = pd.DataFrame(closed_issues) - render_issues_table(closed_issues_df) - else: - st.info("No issues were closed in this scan.") - - # Resolved Findings - st.divider() - st.subheader("Issues with Resolved Findings") - if issues_with_resolved_findings: - issues_with_resolved_findings_df = pd.DataFrame(issues_with_resolved_findings) - render_issues_table(issues_with_resolved_findings_df) - else: - st.info("No findings were resolved in this scan.") \ No newline at end of file diff --git a/app/pages/3_benchmark_detail.py b/app/pages/3_benchmark_detail.py deleted file mode 100644 index e453018..0000000 --- a/app/pages/3_benchmark_detail.py +++ /dev/null @@ -1,237 +0,0 @@ -import streamlit as st -import pandas as pd -from app.database.schema import get_db_connection -from app.components.IssuesList import render_issues_list - -st.set_page_config( - page_title="Benchmark Details", - page_icon="🔒", - layout="wide" -) - -def get_all_benchmarks(): - """Get a list of all benchmarks with their stats.""" - conn = get_db_connection() - cursor = conn.cursor() - - cursor.execute(''' - SELECT - b.id, - b.benchmark, - b.finding_id, - b.level, - b.cvss, - b.title, - COUNT(DISTINCT i.id) as total_issues, - SUM(CASE WHEN i.status = 'open' THEN 1 ELSE 0 END) as open_issues, - COUNT(DISTINCT r.id) as total_remediations, - SUM(CASE WHEN r.state = 'open' THEN 1 ELSE 0 END) as open_remediations - FROM benchmark b - LEFT JOIN issues i ON b.id = i.benchmark_id - LEFT JOIN issue_remediations ir ON i.id = ir.issue_id - LEFT JOIN remediations r ON ir.remediation_id = r.id - GROUP BY b.id - ORDER BY b.cvss DESC - ''') - - columns = [column[0] for column in cursor.description] - rows = [dict(zip(columns, row)) for row in cursor.fetchall()] - conn.close() - - return pd.DataFrame(rows) if rows else pd.DataFrame() - -def get_benchmark_details(benchmark_id): - """Get detailed information about a specific benchmark.""" - conn = get_db_connection() - cursor = conn.cursor() - - # Get benchmark details - cursor.execute(''' - SELECT - b.id, - b.benchmark, - b.finding_id, - b.level, - b.cvss, - b.title, - b.description, - b.rationale, - b.refs - FROM benchmark b - WHERE b.id = ? - ''', (benchmark_id,)) - - benchmark = cursor.fetchone() - if not benchmark: - return None, None, None - - # Get all issues for this benchmark - cursor.execute(''' - SELECT - i.id, - i.due_date, - i.created_at, - i.resolved_at, - i.status, - b.benchmark, - b.finding_id, - b.level, - b.cvss, - b.title, - COUNT(r.id) as remediation_count - FROM issues i - JOIN benchmark b ON i.benchmark_id = b.id - JOIN issue_remediations ir ON i.id = ir.issue_id - JOIN remediations r ON ir.remediation_id = r.id - WHERE b.id = ? - GROUP BY i.id - ORDER BY i.created_at DESC - ''', (benchmark_id,)) - - issues = [dict(zip([column[0] for column in cursor.description], row)) - for row in cursor.fetchall()] - - # Get all open remediations for this benchmark - cursor.execute(''' - SELECT - r.id, - r.state, - r.due_date, - s_first.scan_date as first_seen, - s_resolved.scan_date as resolved_date, - f.failure, - i.id as issue_id - FROM remediations r - JOIN issue_remediations ir ON r.id = ir.remediation_id - JOIN issues i ON ir.issue_id = i.id - JOIN remediation_findings rf ON r.id = rf.remediation_id - JOIN findings f ON rf.finding_id = f.id - LEFT JOIN scans s_first ON r.first_seen_scan = s_first.id - LEFT JOIN scans s_resolved ON r.resolved_in_scan = s_resolved.id - WHERE r.benchmark_id = ? AND r.state = 'open' - ORDER BY r.due_date ASC - ''', (benchmark_id,)) - - remediations = [dict(zip([column[0] for column in cursor.description], row)) - for row in cursor.fetchall()] - - conn.close() - return benchmark, issues, remediations - -def get_issue_page_link(id: str): - """Link to the issue detail page with id as query parameter""" - return f'/issue_detail?id={id}' - -def get_benchmark_page_link(id: str): - """Link to the benchmark detail page with id as query parameter""" - return f'/benchmark_detail?id={id}' - -# Get benchmark ID from URL parameters -benchmark_id = st.query_params.get("id", None) - -if not benchmark_id: - st.title("Security Benchmarks") - - # Show table of all benchmarks - benchmarks_df = get_all_benchmarks() - if not benchmarks_df.empty: - # Add link to benchmark details - benchmarks_df['details_page'] = benchmarks_df['id'].apply(get_benchmark_page_link) - - st.dataframe( - benchmarks_df, - column_config={ - 'id': st.column_config.NumberColumn('ID', width='small'), - 'benchmark': st.column_config.TextColumn('Benchmark', width='medium'), - 'finding_id': st.column_config.TextColumn('Finding ID', width='small'), - 'level': st.column_config.TextColumn('Level', width='small'), - 'cvss': st.column_config.NumberColumn('CVSS', format="%.1f", width='small'), - 'title': st.column_config.TextColumn('Title', width='large'), - 'total_issues': st.column_config.NumberColumn('Total Issues', width='small'), - 'open_issues': st.column_config.NumberColumn('Open Issues', width='small'), - 'total_remediations': st.column_config.NumberColumn('Total Remediations', width='small'), - 'open_remediations': st.column_config.NumberColumn('Open Remediations', width='small'), - 'details_page': st.column_config.LinkColumn('Details', display_text="View", width='small') - }, - hide_index=True, - use_container_width=True - ) - else: - st.info("No benchmarks found.") -else: - # Get benchmark details - benchmark, issues, remediations = get_benchmark_details(benchmark_id) - - if not benchmark: - st.error(f"Benchmark {benchmark_id} not found.") - st.stop() - - # Basic breadcrumbs - st.markdown(f""" - - """, unsafe_allow_html=True) - - # Display benchmark details - st.title(f"{benchmark['title']}") - - # Benchmark metadata - col1, col2 = st.columns(2) - with col1: - st.metric("CVSS Score", f"{benchmark['cvss']:.1f}") - with col2: - st.metric("Level", benchmark['level']) - - # Issues list - st.divider() - st.subheader("Issues") - if issues: - issues_df = pd.DataFrame(issues) - issues_df['details_page'] = issues_df['id'].apply(get_issue_page_link) - render_issues_list(issues_df) - else: - st.info("No issues found for this benchmark.") - - # Open remediations - st.divider() - st.subheader("Open Remediations") - if remediations: - remediations_df = pd.DataFrame(remediations) - # Add link to the issue for each remediation - remediations_df['issue_link'] = remediations_df['issue_id'].apply(get_issue_page_link) - - st.dataframe( - remediations_df, - column_config={ - 'id': st.column_config.NumberColumn('ID', width='small'), - 'state': st.column_config.TextColumn('State', width='small'), - 'due_date': st.column_config.DateColumn('Due Date', width='small'), - 'first_seen': st.column_config.DateColumn('First Seen', width='small'), - 'failure': st.column_config.TextColumn('Failure', width='large'), - 'issue_link': st.column_config.LinkColumn('Issue', display_text="View Issue", width='small') - }, - hide_index=True, - use_container_width=True - ) - else: - st.info("No open remediations for this benchmark.") - - # Description and rationale - st.divider() - st.subheader("Description") - st.write(benchmark['description']) - - if benchmark['rationale']: - st.subheader("Rationale") - st.write(benchmark['rationale']) - - # References - if benchmark['refs']: - st.divider() - st.subheader("References") - for ref in benchmark['refs'].split('\n'): - if ref.strip(): - st.markdown(f"- [{ref}]({ref})") diff --git a/app/pages/4_settings.py b/app/pages/4_settings.py deleted file mode 100644 index 8a08669..0000000 --- a/app/pages/4_settings.py +++ /dev/null @@ -1,52 +0,0 @@ -import streamlit as st -from app.database.schema import get_db_connection, get_poam_config, update_poam_config - -st.set_page_config( - page_title="Settings", - page_icon="🔒", - layout="wide" -) - -st.title("Settings") - -# POAM Export Configuration -st.header("POAM Export Configuration") -st.write("Configure settings for POAM CSV exports.") - -# Get current configuration -conn = get_db_connection() -config = get_poam_config(conn) - -# Default values -point_of_contact = config['point_of_contact'] if config else "" -google_project = config['google_project'] if config else "" -service_name = config['service_name'] if config else "" - -# Form for updating configuration -with st.form("poam_config_form"): - new_point_of_contact = st.text_input( - "Point of Contact", - value=point_of_contact, - help="The point of contact for POAM exports" - ) - new_google_project = st.text_input( - "Google Project", - value=google_project, - help="The Google project identifier" - ) - new_service_name = st.text_input( - "Service Name", - value=service_name, - help="The service name for POAM exports" - ) - - if st.form_submit_button("Save Configuration"): - update_poam_config( - conn, - point_of_contact=new_point_of_contact, - google_project=new_google_project, - service_name=new_service_name - ) - st.success("Configuration updated successfully!") - -conn.close() \ No newline at end of file diff --git a/app/pages/5_poam_export.py b/app/pages/5_poam_export.py deleted file mode 100644 index a625ac2..0000000 --- a/app/pages/5_poam_export.py +++ /dev/null @@ -1,99 +0,0 @@ -import streamlit as st -from app.components.poam_exports import generate_poam_export -from app.database.schema import get_db_connection, get_poam_config -from datetime import datetime - -st.set_page_config( - page_title="POAM Export", - page_icon="🔒", - layout="wide" -) - -st.title("POAM Export") - -# Check if POAM configuration exists -conn = get_db_connection() -config = get_poam_config(conn) -conn.close() - -if not config: - st.error("POAM configuration is not set. Please configure POAM export settings in the Settings page first.") - st.stop() - -# Display current configuration -st.write("Current POAM Configuration:") -st.json({ - "Point of Contact": config['point_of_contact'], - "Google Project": config['google_project'], - "Service Name": config['service_name'] -}) - -st.divider() - -# Export buttons for open and closed issues -col1, col2 = st.columns(2) - -with col1: - st.subheader("Export Open Issues") - if st.button("Download Open Issues POAM", type="primary"): - try: - df = generate_poam_export(status='open') - if df.empty: - st.warning("No open issues found.") - else: - # Generate CSV - csv = df.to_csv(index=False) - current_date = datetime.now().strftime('%Y%m%d') - filename = f"poam_open_issues_{current_date}.csv" - - # Create download button - st.download_button( - label="Click to Download Open Issues POAM", - data=csv, - file_name=filename, - mime="text/csv" - ) - st.success(f"Generated POAM export with {len(df)} open issues.") - except Exception as e: - st.error(f"Error generating POAM export: {str(e)}") - -with col2: - st.subheader("Export Closed Issues") - if st.button("Download Closed Issues POAM", type="primary"): - try: - df = generate_poam_export(status='resolved') - if df.empty: - st.warning("No closed issues found.") - else: - # Generate CSV - csv = df.to_csv(index=False) - current_date = datetime.now().strftime('%Y%m%d') - filename = f"poam_closed_issues_{current_date}.csv" - - # Create download button - st.download_button( - label="Click to Download Closed Issues POAM", - data=csv, - file_name=filename, - mime="text/csv" - ) - st.success(f"Generated POAM export with {len(df)} closed issues.") - except Exception as e: - st.error(f"Error generating POAM export: {str(e)}") - -# Add help text -st.divider() -st.markdown(""" -### About POAM Export -This page allows you to export security issues in Plan of Action and Milestones (POAM) format. The export includes: - -- Separate exports for open and closed issues -- POAM IDs in the format YYYY-CISxxxx (based on issue creation year) -- Standard CIS control mapping (CM-6) -- Configured point of contact and project information -- Detailed finding information including: - - Weakness details and descriptions - - Asset identifiers - - Detection and due dates - - Risk ratings based on CVSS scores -""") \ No newline at end of file diff --git a/app/pages/settings.py b/app/pages/settings.py deleted file mode 100644 index db65295..0000000 --- a/app/pages/settings.py +++ /dev/null @@ -1,35 +0,0 @@ -import streamlit as st -from app.database.schema import get_db_connection, get_poam_config, update_poam_config - -def render(): - st.title("Settings") - - # POAM Export Configuration - st.header("POAM Export Configuration") - st.write("Configure settings for POAM CSV exports.") - - # Get current configuration - conn = get_db_connection() - config = get_poam_config(conn) - - # Default values - point_of_contact = config['point_of_contact'] if config else "" - google_project = config['google_project'] if config else "" - service_name = config['service_name'] if config else "" - - # Form for updating configuration - with st.form("poam_config_form"): - new_point_of_contact = st.text_input("Point of Contact", value=point_of_contact) - new_google_project = st.text_input("Google Project", value=google_project) - new_service_name = st.text_input("Service Name", value=service_name) - - if st.form_submit_button("Save Configuration"): - update_poam_config( - conn, - point_of_contact=new_point_of_contact, - google_project=new_google_project, - service_name=new_service_name - ) - st.success("Configuration updated successfully!") - - conn.close() \ No newline at end of file diff --git a/app/static/style.css b/app/static/style.css deleted file mode 100644 index 59606f5..0000000 --- a/app/static/style.css +++ /dev/null @@ -1,177 +0,0 @@ -/* Custom styles inspired by hail.is */ -.stApp { - background-color: #f8f9fa; -} - -.main-header { - color: #1a1a1a; - font-family: 'Inter', sans-serif; - font-weight: 600; - margin-bottom: 2rem; -} - -.stats-container { - background: white; - border-radius: 8px; - padding: 1.5rem; - box-shadow: 0 2px 4px rgba(0,0,0,0.05); - margin-bottom: 2rem; -} - -.stat-card { - background: #ffffff; - border: 1px solid #e9ecef; - border-radius: 8px; - padding: 1.5rem; - text-align: center; - transition: transform 0.2s; -} - -.stat-card:hover { - transform: translateY(-2px); - box-shadow: 0 4px 6px rgba(0,0,0,0.1); -} - -.stat-number { - font-size: 2.5rem; - font-weight: bold; - margin-bottom: 0.5rem; -} - -.stat-label { - font-size: 1rem; - color: #6c757d; -} - -.findings-table { - background: white; - border-radius: 8px; - padding: 1rem; - box-shadow: 0 2px 4px rgba(0,0,0,0.05); -} - -.severity-critical { - color: #dc3545; -} - -.severity-high { - color: #fd7e14; -} - -.severity-medium { - color: #ffc107; -} - -.severity-low { - color: #20c997; -} - -.severity-info { - color: #0dcaf0; -} - -/* Upload section styling */ -.upload-section { - background: white; - border-radius: 8px; - padding: 2rem; - box-shadow: 0 2px 4px rgba(0,0,0,0.05); - margin-bottom: 2rem; -} - -.stButton>button { - background-color: #2d63ed; - color: white; - border: none; - padding: 0.5rem 1rem; - border-radius: 4px; - font-weight: 500; - transition: background-color 0.2s; -} - -.stButton>button:hover { - background-color: #1c4ed8; -} - -/* Custom styling for the date input */ -.date-input { - max-width: 200px; - margin-bottom: 1rem; -} - -/* Tabs styling */ -.stTabs [data-baseweb="tab-list"] { - gap: 2rem; - border-bottom: 1px solid #dee2e6; -} - -.stTabs [data-baseweb="tab"] { - height: 50px; - white-space: pre-wrap; - background-color: transparent !important; - border-radius: 0; - color: #6c757d; - font-size: 1rem; - font-weight: 500; - border-bottom: 2px solid transparent; - padding-bottom: 1rem; -} - -.stTabs [aria-selected="true"] { - background-color: transparent !important; - color: #dc3545 !important; - border-bottom: 2px solid #dc3545 !important; -} - -.stTabs [data-baseweb="tab"]:hover { - color: #dc3545; - border-bottom: 2px solid #dc3545; -} - -.charts-container { - margin: 2rem 0; -} - -.findings-table { - margin: 2rem 0; -} - -/* Due date warning styles */ -.warning-stat { - background-color: rgba(255, 243, 205, 0.7); - border: 1px solid #ffeeba; -} - -.overdue-stat { - background-color: rgba(248, 215, 218, 0.7); - border: 1px solid #f5c6cb; -} - -[data-testid="stDataFrame"] div[data-testid="stHorizontalBlock"] { - gap: 0 !important; -} - -/* Warning and overdue icons */ -.warning-icon::before { - content: "⚠️"; - margin-right: 8px; -} - -.overdue-icon::before { - content: "❗"; - margin-right: 8px; -} - -/* Row highlighting */ -[data-testid="stDataFrame"] [data-testid="StyledDataFrameDataCell"] { - transition: background-color 0.2s ease; -} - -/* Due date text colors */ -.text-warning { - color: #856404 !important; -} - -.text-overdue { - color: #721c24 !important; -} \ No newline at end of file diff --git a/cli/__init__.py b/cli/__init__.py new file mode 100644 index 0000000..0bf5620 --- /dev/null +++ b/cli/__init__.py @@ -0,0 +1,3 @@ +""" +CLI package for security tracker command line interface. +""" \ No newline at end of file diff --git a/cli/cli.py b/cli/cli.py new file mode 100755 index 0000000..d19cfef --- /dev/null +++ b/cli/cli.py @@ -0,0 +1,651 @@ +#!/usr/bin/env python3 + +import json +import click +import sys +import os +import traceback +from pathlib import Path +from typing import Optional + +# Add the project root to the Python path +sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + +from tools.findings import Finding +from tools.poam import PoamFile +from tools.github import download_trivy_alerts +from tools.trivy.alerts import convert_alerts_to_poam +from tools.trivy.importer import import_alerts_from_csv +from tools.trivy.diff import compare_findings_to_trivy_poams +from tools.diff_apply import apply_diff_from_files, merge_diffs +from tools.zap import convert_alerts_to_findings +from tools.zap.diff import compare_findings_to_zap_poams +from tools.cis.splitter import split_connected_sheet +from tools.cis.converter import convert_to_findings_file +from tools.cis.diff import compare_findings_to_cis_poams + +def generate_alerts_diff(findings_file: Path, poams_file: Path, diff_file: Path, + findings_loader, diff_generator, file_type: str) -> None: + """Generate alerts diff for a findings file. + + Args: + findings_file: Path to the findings file + poams_file: Path to the POAMs file + diff_file: Path where the diff file should be saved + findings_loader: Function to load findings from the file + diff_generator: Function to generate the diff + file_type: Type of findings (for display purposes) + + Raises: + FileNotFoundError: If findings file doesn't exist + """ + if not findings_file.exists(): + raise FileNotFoundError(f"{file_type} findings file not found: {findings_file}") + + click.echo(f"> {file_type.lower()} alerts-diff {poams_file} {findings_file}") + + # Load findings + all_findings = findings_loader(findings_file) + if not all_findings: + click.echo(f"No findings found in {file_type} file") + return + + # Filter out Info severity findings + findings = [f for f in all_findings if f.original_risk_rating.lower() != 'info'] + info_count = len(all_findings) - len(findings) + + if info_count > 0: + click.echo(f"Excluded {info_count} findings with Info severity") + + if not findings: + click.echo("No findings remaining after filtering out Info severity") + return + + # Generate diff + diff = diff_generator(findings, poams_file) + + # Print results + diff.print_summary() + + # Save JSON output + with open(diff_file, 'w') as f: + json.dump(diff.to_json(), f, indent=2) + click.echo(f"{file_type} diff saved to: {diff_file}") + +def load_trivy_findings(csv_file: Path): + """Load Trivy findings from CSV file.""" + return import_alerts_from_csv(csv_file) + +def load_json_findings(json_file: Path): + """Load findings from JSON file.""" + with open(json_file) as f: + findings_data = json.load(f) + return [Finding.from_dict(f) for f in findings_data] + + +def generate_updated_poam_filename(original_poam_file: str, today: str) -> str: + """Generate updated POAM filename by replacing date in original filename with today's date. + + Args: + original_poam_file: Original POAM file path + today: Today's date in YYYY-MM-DD format + + Returns: + Updated filename with today's date + """ + from datetime import datetime + import re + + # Try to find a date pattern in the filename (YYYY-MM-DD or YYYY_MM_DD) + date_pattern = r'(\d{4}[-_]\d{2}[-_]\d{2})' + match = re.search(date_pattern, original_poam_file) + + if match: + # Replace the found date with today's date + return re.sub(date_pattern, today, original_poam_file) + else: + # If no date found, insert today's date before the extension + path = Path(original_poam_file) + return str(path.parent / f"{path.stem} - {today}{path.suffix}") + + +@click.group() +def cli(): + """Security tools CLI.""" + pass + +@cli.group() +def poams(): + """Commands for working with POAMs.""" + pass + +@cli.group() +def trivy(): + """Commands for working with Trivy.""" + pass + +@cli.group() +def zap(): + """Commands for working with ZAP scan reports.""" + pass + +@cli.group() +def cis(): + """Commands for working with CIS findings.""" + pass + +@poams.command('preview-trivy') +@click.argument('file_path', type=click.Path(exists=True)) +@click.option('--limit', '-n', default=5, help='Number of POAMs to preview') +def preview_trivy(file_path, limit): + """Preview Trivy POAMs from an Excel file. + + FILE_PATH should be the path to your POAM Excel file. + The file must contain an "Open POA&M Items" sheet with headers in row 5. + """ + try: + poam_file = PoamFile(file_path) + preview = poam_file.preview_trivy_poams(limit) + click.echo(preview) + except Exception as e: + click.echo(f"Error: {str(e)}", err=True) + sys.exit(1) + +@trivy.command('download-alerts') +@click.option('--destination', '-d', type=click.Path(), help='Destination file path for the alerts JSON file') +def download_alerts(destination): + """Download Trivy alerts from GitHub code scanning API. + + If destination is not specified, uses WORKING environment variable or pwd/working + and sets filename to trivy-alerts-.json + + Requires the GitHub CLI (gh) to be installed and authenticated via 'gh auth login'. + """ + try: + if destination: + # User provided a specific file path + output_file = Path(destination) + output_dir = output_file.parent + output_file = download_trivy_alerts(output_dir) + # Rename to the exact file the user requested + if output_file.name != Path(destination).name: + final_output = Path(destination) + output_file.rename(final_output) + output_file = final_output + else: + # Use default behavior with WORKING env var or pwd/working + output_file = download_trivy_alerts() + click.echo(f"Successfully downloaded alerts to: {output_file}") + except Exception as e: + click.echo(f"Error: {str(e)}", err=True) + sys.exit(1) + +@trivy.command('convert-alerts') +@click.argument('alerts_file', type=click.Path(exists=True)) +@click.option('--output', '-o', type=click.Path(), help='Output file path (default: same directory as input with .findings.csv extension)') +def convert_alerts(alerts_file, output): + """Convert GitHub Trivy alerts JSON to POAM CSV format. + + ALERTS_FILE should be a JSON file containing GitHub code scanning alerts. + The file can be obtained using the download-alerts command. + + The converted POAM data will be saved as a CSV file with .findings.csv extension. + """ + try: + alerts_path = Path(alerts_file) + output_path = Path(output) if output else None + output_file = convert_alerts_to_poam(alerts_path, output_path) + click.echo(f"Successfully converted alerts to POAM format: {output_file}") + except Exception as e: + click.echo(f"Error: {str(e)}", err=True) + sys.exit(1) + +@trivy.command('alerts-diff') +@click.argument('poam_file', type=click.Path(exists=True, path_type=Path)) +@click.argument('alerts_csv', type=click.Path(exists=True, path_type=Path)) +def alerts_diff(poam_file: Path, alerts_csv: Path): + """ + Compare Trivy alerts from CSV against existing POAMs. + + POAM_FILE: Excel file containing existing POAMs + ALERTS_CSV: CSV file containing current Trivy alerts + + Shows: + - New findings that need POAMs created + - Existing findings that already have POAMs + - Closed POAMs that no longer have corresponding findings + + Note: Findings with Info severity are automatically excluded. + """ + try: + json_output_file = alerts_csv.with_suffix('.diff.json') + generate_alerts_diff( + alerts_csv, + poam_file, + json_output_file, + load_trivy_findings, + compare_findings_to_trivy_poams, + "Trivy" + ) + except Exception as e: + click.echo(f"Error comparing alerts: {str(e)}", err=True) + sys.exit(1) + +@poams.command('apply-diff') +@click.argument('poam_file', type=click.Path(exists=True, path_type=Path)) +@click.argument('diff_files', nargs=-1, type=click.Path(exists=True, path_type=Path)) +@click.option('--output', '-o', type=click.Path(path_type=Path), help='Output file path (default: creates timestamped backup)') +def apply_diff(poam_file: Path, diff_files: tuple, output: Optional[Path]) -> None: + """Apply diff changes to a POAM Excel file. + + POAM_FILE: Excel file containing POAMs + DIFF_FILES: One or more JSON files containing diff changes + + This command will: + - Add new POAMs to the Open POA&M Items sheet + - Move reopened POAMs from Closed to Open sheet + - Move closed POAMs from Open to Closed sheet + + If multiple diff files are provided, they will be merged before applying. + If --output is specified, the updated file will be saved to that location. + Otherwise, a default name will be used. + """ + try: + if not diff_files: + click.echo("Error: At least one diff file must be provided", err=True) + sys.exit(1) + + result = apply_diff_from_files(poam_file, list(diff_files), output) + click.echo(f"Successfully applied diff changes to {result}") + except Exception as e: + click.echo(f"Error applying diff: {str(e)}", err=True) + click.echo("\nFull traceback:", err=True) + click.echo(traceback.format_exc(), err=True) + sys.exit(1) + +@poams.command('merge-diffs') +@click.argument('diff_files', nargs=-1, type=click.Path(exists=True, path_type=Path)) +@click.option('--output', '-o', type=click.Path(path_type=Path), help='Output file path (default: merged_diff.json)') +def merge_diffs_cmd(diff_files: tuple, output: Optional[Path]) -> None: + """Merge multiple diff JSON files into a single diff file. + + DIFF_FILES: One or more JSON diff files to merge + --output: Output file path (default: merged_diff.json) + + This command combines all the changes from multiple diff files into a single + diff file that can be applied to a POAM Excel file. + """ + try: + if not diff_files: + click.echo("Error: At least one diff file must be provided", err=True) + sys.exit(1) + + if not output: + output = Path("merged_diff.json") + + merged_diff = merge_diffs(list(diff_files)) + + with open(output, 'w') as f: + json.dump(merged_diff, f, indent=2) + + click.echo(f"Successfully merged {len(diff_files)} diff files into {output}") + + # Print summary of merged content + total_new = len(merged_diff.get("new_poams", [])) + total_reopen = len(merged_diff.get("reopen_poams", [])) + total_close = len(merged_diff.get("close_poams", [])) + total_config_new = len(merged_diff.get("proposed_configuration_findings", [])) + total_config_close = len(merged_diff.get("closed_configuration_findings", [])) + + click.echo(f"Merged content:") + click.echo(f" - New POAMs: {total_new}") + click.echo(f" - Reopen POAMs: {total_reopen}") + click.echo(f" - Close POAMs: {total_close}") + click.echo(f" - New Configuration Findings: {total_config_new}") + click.echo(f" - Close Configuration Findings: {total_config_close}") + + except Exception as e: + click.echo(f"Error merging diffs: {str(e)}", err=True) + click.echo("\nFull traceback:", err=True) + click.echo(traceback.format_exc(), err=True) + sys.exit(1) + +@poams.command('weekly-update') +def weekly_update(): + """Interactive weekly update process for POAMs. + + This command guides you through the weekly update process including: + - Setting up working directory + - Processing Trivy alerts + - Processing CIS findings + - Processing ZAP scans + """ + from datetime import datetime + + try: + # 1. Prompt for working directory + today = datetime.now().strftime("%Y-%m-%d") + default_working_dir = f"working/{today}" + working_dir = click.prompt("Working directory", default=default_working_dir) + working_path = Path(working_dir) + working_path.mkdir(parents=True, exist_ok=True) + + # 2. Print current directory contents + click.echo(f"\nCurrent directory contents:") + items = list(sorted(working_path.iterdir())) + if len(items) > 0: + for item in items: + click.echo(f" {item}") + else: + click.echo(" (empty)") + + # 3. Ask for input file paths + click.echo(f"\n--- Input Files ---") + + # Find files in working directory + cis_files = list(working_path.glob("*CIS*.xlsx")) + hail_files = list(working_path.glob("hail_report*.csv")) + poam_files = list(working_path.glob("*POAM*.xlsx")) + + # Use the first found file or empty string if none found + cis_default = str(cis_files[0]) if cis_files else "" + zap_default = str(hail_files[0]) if hail_files else "" + poams_default = str(poam_files[0]) if poam_files else "" + + cis_findings = click.prompt("Path to continuous CIS findings sheet", default=cis_default) + zap_scan = click.prompt("Path to most recent ZAP scan", default=zap_default) + poams_file = click.prompt("Path to current POAMs file", default=poams_default) + + # 4. Trivy actions + click.echo(f"\n--- Trivy Actions ---") + trivy_alerts_file = working_path / f"trivy-alerts-{today}.json" + trivy_findings_file = working_path / f"trivy-findings-{today}.findings.csv" + + skip_trivy = False + if trivy_findings_file.exists(): + skip_trivy = click.confirm(f"Trivy findings file {trivy_findings_file} already exists. Skip Trivy actions?", default=True) + + if not skip_trivy: + if click.confirm(f"Download Trivy alerts to {trivy_alerts_file}?", default=True, abort=True): + click.echo(f"> trivy download-alerts -d {trivy_alerts_file}") + download_trivy_alerts(trivy_alerts_file.parent) + # Rename to the exact file we want + if trivy_alerts_file.exists(): + trivy_alerts_file.unlink() # Remove if it exists with different name + # Find the actual downloaded file and rename it + for file in working_path.glob("trivy_alerts_*.json"): + file.rename(trivy_alerts_file) + break + + if click.confirm(f"Convert Trivy alerts to findings CSV?", default=True, abort=True): + click.echo(f"> trivy convert-alerts {trivy_alerts_file} -o {trivy_findings_file}") + convert_alerts_to_poam(trivy_alerts_file, trivy_findings_file) + + # 5. CIS actions + click.echo(f"\n--- CIS Actions ---") + split_output_dir = working_path / "Divided CIS Scans" + cis_findings_file = working_path / f"cis-findings-{today}.findings.json" + + skip_cis = False + if cis_findings_file.exists(): + skip_cis = click.confirm(f"CIS findings file {cis_findings_file} already exists. Skip CIS actions?", default=True) + + if not skip_cis: + if click.confirm(f"Split CIS connected sheet?", default=True, abort=True): + click.echo(f"> cis split-connected-sheet {cis_findings} -o {split_output_dir}") + split_connected_sheet(Path(cis_findings), split_output_dir) + + # Find the most recent findings file (sort by filename since filenames include the date in YYYY-MM-DD format and are otherwise the same) + if split_output_dir.exists(): + csv_files = list(split_output_dir.glob("*.csv")) + if csv_files: + most_recent_file = max(csv_files, key=lambda f: f.name) + click.echo(f"Most recent findings file: {most_recent_file}") + + if click.confirm(f"Convert CIS CSV to findings?", default=True, abort=True): + click.echo(f"> cis csv-to-findings {most_recent_file} -o {cis_findings_file}") + convert_to_findings_file(most_recent_file, cis_findings_file) + + # 6. ZAP actions + click.echo(f"\n--- ZAP Actions ---") + zap_findings_file = working_path / f"zap-findings-{today}.findings.json" + + skip_zap = False + if zap_findings_file.exists(): + skip_zap = click.confirm(f"ZAP findings file {zap_findings_file} already exists. Skip ZAP actions?", default=True) + + if not skip_zap: + if click.confirm(f"Convert ZAP scan to findings?", default=True, abort=True): + click.echo(f"> zap alerts-to-findings {zap_scan} -o {zap_findings_file}") + convert_alerts_to_findings(zap_scan, str(zap_findings_file)) + + # 7. Generate alerts-diffs + click.echo(f"\n--- Generate Alerts Diffs ---") + + # Trivy alerts diff + trivy_diff_file = trivy_findings_file.with_suffix('.diff.json') + skip_trivy_diff = False + if trivy_diff_file.exists(): + skip_trivy_diff = click.confirm(f"Trivy diff file {trivy_diff_file} already exists. Skip Trivy diff generation?", default=True) + + if not skip_trivy_diff: + if click.confirm(f"Generate Trivy alerts diff?", default=True, abort=True): + generate_alerts_diff( + trivy_findings_file, + Path(poams_file), + trivy_diff_file, + load_trivy_findings, + compare_findings_to_trivy_poams, + "Trivy" + ) + + # CIS alerts diff + cis_diff_file = cis_findings_file.with_suffix('.diff.json') + skip_cis_diff = False + if cis_diff_file.exists(): + skip_cis_diff = click.confirm(f"CIS diff file {cis_diff_file} already exists. Skip CIS diff generation?", default=True) + + if not skip_cis_diff: + if click.confirm(f"Generate CIS alerts diff?", default=True, abort=True): + generate_alerts_diff( + cis_findings_file, + Path(poams_file), + cis_diff_file, + load_json_findings, + compare_findings_to_cis_poams, + "CIS" + ) + + # ZAP alerts diff + zap_diff_file = zap_findings_file.with_suffix('.diff.json') + skip_zap_diff = False + if zap_diff_file.exists(): + skip_zap_diff = click.confirm(f"ZAP diff file {zap_diff_file} already exists. Skip ZAP diff generation?", default=True) + + if not skip_zap_diff: + if click.confirm(f"Generate ZAP alerts diff?", default=True, abort=True): + generate_alerts_diff( + zap_findings_file, + Path(poams_file), + zap_diff_file, + load_json_findings, + compare_findings_to_zap_poams, + "ZAP" + ) + + # 8. Apply diffs + click.echo(f"\n--- Apply Diffs ---") + + # Generate updated POAM filename + updated_poam_file = generate_updated_poam_filename(poams_file, today) + updated_poam_path = Path(updated_poam_file) + + skip_apply_diffs = False + if updated_poam_path.exists(): + skip_apply_diffs = click.confirm(f"Updated POAM file {updated_poam_path} already exists. Skip applying diffs?", default=True) + + if not skip_apply_diffs: + if click.confirm(f"Apply diffs to create updated POAMs?", default=True, abort=True): + diff_files = [trivy_diff_file, cis_diff_file, zap_diff_file] + + click.echo(f"> poams apply-diff {poams_file} {' '.join(str(f) for f in diff_files)} -o {updated_poam_path}") + + # Apply the diffs + result = apply_diff_from_files(poams_file, diff_files, updated_poam_path) + click.echo(f"Successfully applied diff changes to {result}") + + click.echo(f"\n--- Weekly Update Complete ---") + click.echo(f"Working directory: {working_path}") + click.echo(f"Output files to upload:") + click.echo(f" trivy findings: {trivy_findings_file}") + click.echo(f" cis findings: {cis_findings_file}") + click.echo(f" zap findings: {zap_findings_file}") + click.echo(f" updated POAMs: {result}") + + except Exception as e: + click.echo(f"Error during weekly update: {str(e)}", err=True) + click.echo(traceback.format_exc(), err=True) + sys.exit(1) + +@zap.command('alerts-to-findings') +@click.argument('csv_file', type=click.Path(exists=True)) +@click.option('--output', '-o', type=click.Path(), help='Output file path (default: input file with .findings.json extension)') +def alerts_to_findings(csv_file, output): + """Convert ZAP CSV alerts to findings JSON format. + + CSV_FILE should be a ZAP CSV report file. + The findings will be saved as a JSON file and the first finding will be displayed. + """ + try: + # Convert alerts to findings + output_file = convert_alerts_to_findings(csv_file, output) + + # Load and display first finding + with open(output_file) as f: + findings = json.load(f) + if findings: + click.echo("\nFirst finding from the report:") + click.echo(json.dumps(findings[0], indent=2)) + click.echo(f"\nTotal findings: {len(findings)}") + click.echo(f"All findings saved to: {output_file}") + else: + click.echo("No findings found in the report.") + except Exception as e: + click.echo(f"Error converting alerts: {str(e)}", err=True) + sys.exit(1) + +@zap.command('alerts-diff') +@click.argument('poam_file', type=click.Path(exists=True)) +@click.argument('findings_file', type=click.Path(exists=True)) +@click.option('--json-output', type=click.Path(), help='Path to save JSON output') +def zap_alerts_diff(poam_file: str, findings_file: str, json_output: Optional[str]) -> None: + """Compare ZAP findings against existing POAMs. + + Note: Findings with Info severity are automatically excluded. + """ + try: + if not json_output: + findings_path = Path(findings_file) + json_output = findings_path.with_suffix('.diff.json') + + generate_alerts_diff( + Path(findings_file), + Path(poam_file), + Path(json_output), + load_json_findings, + compare_findings_to_zap_poams, + "ZAP" + ) + except Exception as e: + click.echo(f"Error comparing findings: {str(e)}", err=True) + sys.exit(1) + +@cis.command('split-connected-sheet') +@click.argument('xlsx_file', type=click.Path(exists=True, path_type=Path)) +@click.option('--output', '-o', type=click.Path(path_type=Path), help='Output directory for split CSV files (default: input directory/Divided CIS Scans)') +def split_connected_sheet_cmd(xlsx_file: Path, output: Optional[Path]) -> None: + """Split a CIS connected sheet into separate CSV files by date. + + XLSX_FILE should be a CIS connected sheet Excel file. + + The command will: + - Create a "Divided CIS Scans" directory if it doesn't exist + - Split the file into multiple CSVs based on the Date field + - Name each file as " - YYYY-MM-DD.csv" + - Skip writing if a file for a particular date already exists + """ + try: + output_files = split_connected_sheet(xlsx_file, output) + if output_files: + click.echo(f"Successfully split {xlsx_file.name} into {len(output_files)} files:") + for f in output_files: + click.echo(f" - {f.name}") + else: + click.echo("No new files created (all dates already exist)") + except Exception as e: + click.echo(f"Error splitting connected sheet: {str(e)}", err=True) + sys.exit(1) + +@cis.command('csv-to-findings') +@click.argument('csv_file', type=click.Path(exists=True, path_type=Path)) +@click.option('--output', '-o', type=click.Path(path_type=Path), help='Output file path (default: input file with .findings.json extension)') +def csv_to_findings_cmd(csv_file: Path, output: Optional[Path]) -> None: + """Convert a CIS CSV file to findings JSON format. + + CSV_FILE should be a CIS CSV file (typically from split-connected-sheet). + + The command will: + - Convert each row into one or more findings based on the Failures field + - Generate finding IDs in the format CIS--XXXX + - Save the findings as .findings.json + """ + try: + output_file = convert_to_findings_file(csv_file, output) + + # Load and display summary + with open(output_file) as f: + findings = json.load(f) + click.echo(f"\nSuccessfully converted {csv_file.name} to findings:") + click.echo(f"- Total findings: {len(findings)}") + if findings: + unique_rules = len({f['weakness_name'] for f in findings}) + click.echo(f"- Unique CIS rules: {unique_rules}") + click.echo("\nSample finding:") + click.echo(json.dumps(findings[0], indent=2)) + click.echo(f"\nOutput saved to: {output_file}") + except Exception as e: + click.echo(f"Error converting CSV to findings: {str(e)}", err=True) + sys.exit(1) + +@cis.command('alerts-diff') +@click.argument('poam_file', type=click.Path(exists=True)) +@click.argument('findings_file', type=click.Path(exists=True)) +@click.option('--json-output', type=click.Path(), help='Path to save JSON output') +def cis_alerts_diff(poam_file: str, findings_file: str, json_output: Optional[str]) -> None: + """ + Compare CIS findings against existing configuration findings. + + FINDINGS_FILE: JSON file containing CIS findings + POAM_FILE: Excel file containing configuration findings + + Note: Findings with Info severity are automatically excluded. + """ + try: + if not json_output: + findings_path = Path(findings_file) + json_output = findings_path.with_suffix('.diff.json') + + generate_alerts_diff( + Path(findings_file), + Path(poam_file), + Path(json_output), + load_json_findings, + compare_findings_to_cis_poams, + "CIS" + ) + except Exception as e: + click.echo(f"Error comparing findings: {str(e)}", err=True) + sys.exit(1) + +if __name__ == '__main__': + cli() \ No newline at end of file diff --git a/devbin/pr-comments b/devbin/pr-comments new file mode 100755 index 0000000..2b001f7 --- /dev/null +++ b/devbin/pr-comments @@ -0,0 +1,42 @@ +#!/usr/bin/env bash +# Usage: pr-comments +# Downloads and prints all comments on a GitHub PR: +# - The review summaries (from /pulls/:pr/reviews) +# - Inline code comments (from /pulls/:pr/comments) +# - General issue comments (from /issues/:pr/comments) + +set -euo pipefail + +PR="${1:?Usage: pr-comments }" +REPO=$(gh repo view --json nameWithOwner -q .nameWithOwner) + +echo "═══════════════════════════════════════════════════════════" +echo " PR #$PR — $REPO" +echo "═══════════════════════════════════════════════════════════" + +# ── Review summaries ──────────────────────────────────────────── +echo "" +echo "── REVIEWS ────────────────────────────────────────────────" + +gh api "repos/$REPO/pulls/$PR/reviews" --paginate | \ + jq -r '.[] | (.body // "") as $body | + if ($body | length) > 0 + then "\n[\(.state)] \(.user.login) — \(.submitted_at)\n\($body)" + else "\n[\(.state)] \(.user.login) — \(.submitted_at)" + end' + +# ── Inline code comments ──────────────────────────────────────── +echo "" +echo "── INLINE COMMENTS ────────────────────────────────────────" + +gh api "repos/$REPO/pulls/$PR/comments" --paginate | \ + jq -r '.[] | + "\n\(.path) (line \(.original_line // .line // "?")) — \(.user.login)\n\(.body)"' + +# ── General PR / issue comments ───────────────────────────────── +echo "" +echo "── GENERAL COMMENTS ───────────────────────────────────────" + +gh api "repos/$REPO/issues/$PR/comments" --paginate | \ + jq -r '.[] | + "\n\(.user.login) — \(.created_at)\n\(.body)"' diff --git a/requirements.txt b/requirements.txt index 4079643..92a6d5b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,6 @@ -streamlit>=1.43.0; python_version >= "3.9" and python_version < "3.13" pandas==2.2.1; python_version >= "3.9" and python_version < "3.13" -numpy==1.26.4; python_version >= "3.9" and python_version < "3.13" -plotly==5.19.0; python_version >= "3.9" and python_version < "3.13" -python-dateutil==2.8.2; python_version >= "3.9" and python_version < "3.13" \ No newline at end of file +python-dateutil==2.8.2; python_version >= "3.9" and python_version < "3.13" +click>=8.1.7; python_version >= "3.9" and python_version < "3.13" +openpyxl>=3.1.2; python_version >= "3.9" and python_version < "3.13" +PyYAML>=6.0.1; python_version >= "3.9" and python_version < "3.13" +jq>=1.6.0; python_version >= "3.9" and python_version < "3.13" diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..33fc16c --- /dev/null +++ b/tests/__init__.py @@ -0,0 +1,3 @@ +""" +Test package initialization. +""" \ No newline at end of file diff --git a/tests/cis/__init__.py b/tests/cis/__init__.py new file mode 100644 index 0000000..9e47485 --- /dev/null +++ b/tests/cis/__init__.py @@ -0,0 +1,3 @@ +""" +Tests for CIS-related functionality. +""" \ No newline at end of file diff --git a/tests/cis/test_poam_generator.py b/tests/cis/test_poam_generator.py new file mode 100644 index 0000000..a469e3f --- /dev/null +++ b/tests/cis/test_poam_generator.py @@ -0,0 +1,281 @@ +""" +Tests for CIS POAM generator. +""" +from datetime import datetime, timedelta, timezone +from tools.findings import Finding +from tools.cis.poam_generator import generate_poams_from_findings + +def test_findings_with_same_weakness_are_grouped(): + """Test that findings with the same weakness name and risk rating are grouped into one POAM.""" + detection_date = datetime.now(timezone.utc) + completion_date = detection_date + timedelta(days=30) + status_date = detection_date + + # Create two findings with same weakness name and risk rating + findings = [ + Finding( + finding_id="finding1", + controls="", + weakness_name="[VMS] Ensure that instances are not configured to use the default service account", + weakness_description="Instance 1 is using default service account", + weakness_detector_source="CIS", + weakness_source_identifier="", + asset_identifier="instance-1", + point_of_contact="Security Team", + resources_required="Security Team time", + overall_remediation_plan="Remediate default service account usage", + original_detection_date=detection_date, + scheduled_completion_date=completion_date, + planned_milestones="", + milestone_changes="", + status_date=status_date, + vendor_dependency="No", + last_vendor_check_in_date=None, + vendor_dependent_product_name="", + original_risk_rating="High", + adjusted_risk_rating=None, + risk_adjustment="", + false_positive="No", + operational_requirement="No", + deviation_rationale=None, + supporting_documents=None, + comments=None, + auto_approve="No", + binding_operational_directive_22_01_tracking="No", + binding_operational_directive_22_01_due_date=None, + cve=None, + service_name="" + ), + Finding( + finding_id="finding2", + controls="", + weakness_name="[VMS] Ensure that instances are not configured to use the default service account", + weakness_description="Instance 2 is using default service account", + weakness_detector_source="CIS", + weakness_source_identifier="", + asset_identifier="instance-2", + point_of_contact="Security Team", + resources_required="Security Team time", + overall_remediation_plan="Remediate default service account usage", + original_detection_date=detection_date, + scheduled_completion_date=completion_date, + planned_milestones="", + milestone_changes="", + status_date=status_date, + vendor_dependency="No", + last_vendor_check_in_date=None, + vendor_dependent_product_name="", + original_risk_rating="High", + adjusted_risk_rating=None, + risk_adjustment="", + false_positive="No", + operational_requirement="No", + deviation_rationale=None, + supporting_documents=None, + comments=None, + auto_approve="No", + binding_operational_directive_22_01_tracking="No", + binding_operational_directive_22_01_due_date=None, + cve=None, + service_name="" + ) + ] + + # Generate POAMs + result = generate_poams_from_findings(findings, existing_poam_ids=[]) + + # Verify we only got one POAM + assert len(result) == 1, "Expected findings to be grouped into one POAM" + + # Get the findings and POAM from the result + grouped_findings, poam = result[0] + + # Verify both findings are in the group + assert len(grouped_findings) == 2, "Expected both findings in the group" + assert {f.finding_id for f in grouped_findings} == {"finding1", "finding2"} + + # Verify POAM has combined asset identifiers + assert "instance-1" in poam.asset_identifier + assert "instance-2" in poam.asset_identifier + assert "," in poam.asset_identifier # Should be comma-separated + + # Verify other POAM fields + assert poam.weakness_name == "[VMS] Ensure that instances are not configured to use the default service account" + assert poam.original_risk_rating == "High" + # Due date should be 30 days from now (for High risk) + expected_due_date = datetime.now(timezone.utc) + timedelta(days=30) + assert abs((poam.scheduled_completion_date - expected_due_date).days) <= 1 # Allow 1 day difference due to timing + +def test_findings_with_different_weakness_not_grouped(): + """Test that findings with different weakness names are not grouped.""" + detection_date = datetime.now(timezone.utc) + completion_date = detection_date + timedelta(days=30) + status_date = detection_date + + # Create two findings with different weakness names + findings = [ + Finding( + finding_id="finding1", + controls="", + weakness_name="[VMS] Ensure that instances are not configured to use the default service account", + weakness_description="Instance 1 is using default service account", + weakness_detector_source="CIS", + weakness_source_identifier="", + asset_identifier="instance-1", + point_of_contact="Security Team", + resources_required="Security Team time", + overall_remediation_plan="Remediate default service account usage", + original_detection_date=detection_date, + scheduled_completion_date=completion_date, + planned_milestones="", + milestone_changes="", + status_date=status_date, + vendor_dependency="No", + last_vendor_check_in_date=None, + vendor_dependent_product_name="", + original_risk_rating="High", + adjusted_risk_rating=None, + risk_adjustment="", + false_positive="No", + operational_requirement="No", + deviation_rationale=None, + supporting_documents=None, + comments=None, + auto_approve="No", + binding_operational_directive_22_01_tracking="No", + binding_operational_directive_22_01_due_date=None, + cve=None, + service_name="" + ), + Finding( + finding_id="finding2", + controls="", + weakness_name="[VMS] Ensure that instances do not have public IP addresses", + weakness_description="Instance 2 has public IP", + weakness_detector_source="CIS", + weakness_source_identifier="", + asset_identifier="instance-2", + point_of_contact="Security Team", + resources_required="Security Team time", + overall_remediation_plan="Remove public IP addresses", + original_detection_date=detection_date, + scheduled_completion_date=completion_date, + planned_milestones="", + milestone_changes="", + status_date=status_date, + vendor_dependency="No", + last_vendor_check_in_date=None, + vendor_dependent_product_name="", + original_risk_rating="High", + adjusted_risk_rating=None, + risk_adjustment="", + false_positive="No", + operational_requirement="No", + deviation_rationale=None, + supporting_documents=None, + comments=None, + auto_approve="No", + binding_operational_directive_22_01_tracking="No", + binding_operational_directive_22_01_due_date=None, + cve=None, + service_name="" + ) + ] + + # Generate POAMs + result = generate_poams_from_findings(findings, existing_poam_ids=[]) + + # Verify we got two POAMs + assert len(result) == 2, "Expected findings to generate separate POAMs" + + # Verify each POAM has one finding + for findings_group, poam in result: + assert len(findings_group) == 1, "Expected one finding per POAM" + assert poam.asset_identifier in ["instance-1", "instance-2"] + assert "," not in poam.asset_identifier # Should not be comma-separated + +def test_findings_with_different_risk_ratings_not_grouped(): + """Test that findings with same weakness but different risk ratings are not grouped.""" + detection_date = datetime.now(timezone.utc) + completion_date = detection_date + timedelta(days=30) + status_date = detection_date + + # Create two findings with same weakness but different risk ratings + findings = [ + Finding( + finding_id="finding1", + controls="", + weakness_name="[VMS] Ensure that instances are not configured to use the default service account", + weakness_description="Instance 1 is using default service account", + weakness_detector_source="CIS", + weakness_source_identifier="", + asset_identifier="instance-1", + point_of_contact="Security Team", + resources_required="Security Team time", + overall_remediation_plan="Remediate default service account usage", + original_detection_date=detection_date, + scheduled_completion_date=completion_date, + planned_milestones="", + milestone_changes="", + status_date=status_date, + vendor_dependency="No", + last_vendor_check_in_date=None, + vendor_dependent_product_name="", + original_risk_rating="High", + adjusted_risk_rating=None, + risk_adjustment="", + false_positive="No", + operational_requirement="No", + deviation_rationale=None, + supporting_documents=None, + comments=None, + auto_approve="No", + binding_operational_directive_22_01_tracking="No", + binding_operational_directive_22_01_due_date=None, + cve=None, + service_name="" + ), + Finding( + finding_id="finding2", + controls="", + weakness_name="[VMS] Ensure that instances are not configured to use the default service account", + weakness_description="Instance 2 is using default service account", + weakness_detector_source="CIS", + weakness_source_identifier="", + asset_identifier="instance-2", + point_of_contact="Security Team", + resources_required="Security Team time", + overall_remediation_plan="Remediate default service account usage", + original_detection_date=detection_date, + scheduled_completion_date=completion_date, + planned_milestones="", + milestone_changes="", + status_date=status_date, + vendor_dependency="No", + last_vendor_check_in_date=None, + vendor_dependent_product_name="", + original_risk_rating="Low", # Different risk rating + adjusted_risk_rating=None, + risk_adjustment="", + false_positive="No", + operational_requirement="No", + deviation_rationale=None, + supporting_documents=None, + comments=None, + auto_approve="No", + binding_operational_directive_22_01_tracking="No", + binding_operational_directive_22_01_due_date=None, + cve=None, + service_name="" + ) + ] + + # Generate POAMs + result = generate_poams_from_findings(findings, existing_poam_ids=[]) + + # Verify we got two POAMs (due to different completion dates) + assert len(result) == 2, "Expected findings to generate separate POAMs due to different risk ratings" + + # Verify completion dates are different + completion_dates = {poam.scheduled_completion_date for _, poam in result} + assert len(completion_dates) == 2, "Expected different completion dates for different risk ratings" \ No newline at end of file diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..d6a8cb5 --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,10 @@ +""" +Test configuration and fixtures. +""" +import os +import sys +from pathlib import Path + +# Add the project root directory to the Python path +project_root = Path(__file__).parent.parent +sys.path.append(str(project_root)) \ No newline at end of file diff --git a/tests/test_diff.py b/tests/test_diff.py new file mode 100644 index 0000000..2168ee1 --- /dev/null +++ b/tests/test_diff.py @@ -0,0 +1,285 @@ +""" +Tests for the diff logic between Findings and POAMs. +""" +from datetime import datetime +from pathlib import Path + +from tools.findings import Finding +from tools.poam import PoamEntry +from tools.trivy.diff import ( + _is_exact_match, + _is_asset_covered, + _find_matching_poam, +) +from tools.diff import compare_findings_to_poams + +def create_test_finding(finding_id: str, weakness_name: str, asset_identifier: str) -> Finding: + """Helper to create a test Finding with minimal required fields.""" + return Finding( + finding_id=finding_id, + controls="Test controls", + weakness_name=weakness_name, + weakness_description="Test description", + weakness_detector_source="Trivy", + weakness_source_identifier="TEST-001", + asset_identifier=asset_identifier, + point_of_contact="test@example.com", + resources_required=None, + overall_remediation_plan="Test plan", + original_detection_date=datetime.now(), + scheduled_completion_date=datetime.now(), + planned_milestones="Test milestones", + milestone_changes="None", + status_date=datetime.now(), + vendor_dependency="None", + last_vendor_check_in_date=None, + vendor_dependent_product_name="None", + original_risk_rating="Low", + adjusted_risk_rating=None, + risk_adjustment="None", + false_positive="No", + operational_requirement="No", + deviation_rationale=None, + supporting_documents=None, + comments=None, + auto_approve="No", + binding_operational_directive_22_01_tracking="No", + binding_operational_directive_22_01_due_date=None, + cve=None, + service_name="test-service" + ) + +def create_test_poam(poam_id: str, weakness_name: str, asset_identifier: str) -> PoamEntry: + """Helper to create a test PoamEntry with minimal required fields.""" + return PoamEntry( + poam_id=poam_id, + controls="Test controls", + weakness_name=weakness_name, + weakness_description="Test description", + weakness_detector_source="Trivy", + weakness_source_identifier="TEST-001", + asset_identifier=asset_identifier, + point_of_contact="test@example.com", + resources_required=None, + overall_remediation_plan="Test plan", + original_detection_date=datetime.now(), + scheduled_completion_date=datetime.now(), + planned_milestones="Test milestones", + milestone_changes="None", + status_date=datetime.now(), + vendor_dependency="None", + last_vendor_check_in_date=None, + vendor_dependent_product_name="None", + original_risk_rating="Low", + adjusted_risk_rating=None, + risk_adjustment="None", + false_positive="No", + operational_requirement="No", + deviation_rationale=None, + supporting_documents=None, + comments=None, + auto_approve="No", + binding_operational_directive_22_01_tracking="No", + binding_operational_directive_22_01_due_date=None, + cve=None, + service_name="test-service" + ) + +def test_exact_match(): + """Test the exact matching function.""" + assert _is_exact_match("Test String", "test string") + assert _is_exact_match("test string ", "test string") + assert not _is_exact_match("test string", "different string") + assert not _is_exact_match("", None) + assert not _is_exact_match(None, "test") + +def test_asset_covered(): + """Test the asset coverage function.""" + # Single asset matches + assert _is_asset_covered("app-1", "app-1") + assert _is_asset_covered("app-1", "App-1") + + # Asset list includes the finding's asset + assert _is_asset_covered("app-1", "app-1, app-2, app-3") + assert _is_asset_covered("app-2", "app-1,app-2,app-3") + + # Asset not in list + assert not _is_asset_covered("app-4", "app-1, app-2, app-3") + + # Edge cases + assert not _is_asset_covered("", "app-1") + assert not _is_asset_covered("app-1", "") + assert not _is_asset_covered(None, "app-1") + assert not _is_asset_covered("app-1", None) + +def test_find_matching_poam(): + """Test finding matching POAMs.""" + finding = create_test_finding( + finding_id="TRIVY-001", + weakness_name="SQL Injection; CVE-2023-1234", + asset_identifier="app-1" + ) + + # Exact match + matching_poam = create_test_poam( + poam_id="POAM-001", + weakness_name="SQL Injection; CVE-2023-1234", + asset_identifier="app-1" + ) + + # Same weakness, different asset + different_asset_poam = create_test_poam( + poam_id="POAM-002", + weakness_name="SQL Injection; CVE-2023-1234", + asset_identifier="app-2" + ) + + # Different weakness, same asset + different_weakness_poam = create_test_poam( + poam_id="POAM-003", + weakness_name="XSS; CVE-2023-5678", + asset_identifier="app-1" + ) + + # POAM covering multiple assets + multi_asset_poam = create_test_poam( + poam_id="POAM-004", + weakness_name="SQL Injection; CVE-2023-1234", + asset_identifier="app-1, app-2, app-3" + ) + + # Test single POAM matching + match = _find_matching_poam(finding, [matching_poam]) + assert match is not None + assert match.poam.poam_id == "POAM-001" + + # Test no match for different asset + match = _find_matching_poam(finding, [different_asset_poam]) + assert match is None + + # Test no match for different weakness + match = _find_matching_poam(finding, [different_weakness_poam]) + assert match is None + + # Test match with multi-asset POAM + match = _find_matching_poam(finding, [multi_asset_poam]) + assert match is not None + assert match.poam.poam_id == "POAM-004" + + # Test finding best match from multiple POAMs + all_poams = [different_asset_poam, different_weakness_poam, matching_poam, multi_asset_poam] + match = _find_matching_poam(finding, all_poams) + assert match is not None + assert match.poam.poam_id == "POAM-001" # Should match the first valid match + +def test_compare_findings_to_poams(): + """Test the full comparison logic.""" + # Create test findings + findings = [ + create_test_finding("TRIVY-001", "SQL Injection; CVE-2023-1234", "app-1"), # Should match open POAM + create_test_finding("TRIVY-002", "XSS; CVE-2023-5678", "app-2"), # Should be new (no match) + create_test_finding("TRIVY-003", "CSRF; CVE-2023-9012", "app-3"), # Should match open POAM + create_test_finding("TRIVY-004", "RCE; CVE-2023-4567", "app-4"), # Should match closed POAM + ] + + # Create test open POAMs + open_poams = [ + create_test_poam("POAM-001", "SQL Injection; CVE-2023-1234", "app-1, app-4"), # Should match TRIVY-001 + create_test_poam("POAM-002", "XSS; CVE-2023-5678", "app-5"), # Should be closed (no match) + create_test_poam("POAM-003", "Buffer Overflow; CVE-2023-3456", "app-1"), # Should be closed (no match) + create_test_poam("POAM-004", "CSRF; CVE-2023-9012", "app-1, app-2, app-3"), # Should match TRIVY-003 + ] + + # Create test closed POAMs + closed_poams = [ + create_test_poam("POAM-005", "RCE; CVE-2023-4567", "app-4"), # Should match TRIVY-004 (reopened) + create_test_poam("POAM-006", "XSS; CVE-2023-8901", "app-6"), # Should stay closed (no match) + ] + + # Compare findings to POAMs + diff = compare_findings_to_poams(findings, open_poams, closed_poams, existing_poam_ids=[], poam_generator=lambda f, ids: []) + + # Verify new findings + assert {f.finding_id for f in diff.new_findings} == {"TRIVY-002"} + + # Verify existing matches + assert {match.poam.poam_id for match in diff.existing_matches} == {"POAM-001", "POAM-004"} + assert {match.finding.finding_id for match in diff.existing_matches} == {"TRIVY-001", "TRIVY-003"} + + # Verify reopened findings + assert {match.poam.poam_id for match in diff.reopened_findings} == {"POAM-005"} + assert {match.finding.finding_id for match in diff.reopened_findings} == {"TRIVY-004"} + + # Verify closed POAMs + assert {poam.poam_id for poam in diff.closed_poams} == {"POAM-002", "POAM-003"} + +def test_cis_configuration_findings_closed_when_no_matches(): + """Test that CIS configuration findings are marked as closed when no findings match them.""" + # Create empty findings list (no current findings) + findings = [] + + # Create test configuration findings (simulating your real case) + config_findings = [ + create_test_poam("2025-CIS0005-A", "Some Configuration Issue", "asset-1, asset-2, asset-3, asset-4, asset-5"), + create_test_poam("2025-CIS0006-B", "Another Configuration Issue", "asset-10, asset-11"), + ] + + # Mock poam_generator function + def mock_poam_generator(new_findings, existing_poam_ids): + return [] + + # Compare findings to POAMs with store_as_configuration_findings=True + diff = compare_findings_to_poams( + findings=findings, + open_poams=config_findings, + closed_poams=[], + existing_poam_ids=["2025-CIS0005-A", "2025-CIS0006-B"], + poam_generator=mock_poam_generator, + store_as_configuration_findings=True + ) + + # Verify that all configuration findings are marked as closed + assert len(diff.closed_configuration_findings) == 2 + assert {poam.poam_id for poam in diff.closed_configuration_findings} == {"2025-CIS0005-A", "2025-CIS0006-B"} + + # Verify no new findings or matches + assert len(diff.new_findings) == 0 + assert len(diff.existing_matches) == 0 + assert len(diff.reopened_findings) == 0 + assert len(diff.proposed_configuration_findings) == 0 + +def test_cis_configuration_findings_partial_matches(): + """Test that only unmatched CIS configuration findings are marked as closed.""" + # Create findings that match some but not all configuration findings + findings = [ + create_test_finding("CIS-001", "Some Configuration Issue", "asset-1"), # Should match first config finding + ] + + # Create test configuration findings + config_findings = [ + create_test_poam("2025-CIS0005-A", "Some Configuration Issue", "asset-1, asset-2, asset-3"), + create_test_poam("2025-CIS0006-B", "Another Configuration Issue", "asset-10, asset-11"), + ] + + # Mock poam_generator function + def mock_poam_generator(new_findings, existing_poam_ids): + return [] + + # Compare findings to POAMs with store_as_configuration_findings=True + diff = compare_findings_to_poams( + findings=findings, + open_poams=config_findings, + closed_poams=[], + existing_poam_ids=["2025-CIS0005-A", "2025-CIS0006-B"], + poam_generator=mock_poam_generator, + store_as_configuration_findings=True + ) + + # Verify that only the unmatched configuration finding is marked as closed + assert len(diff.closed_configuration_findings) == 1 + assert diff.closed_configuration_findings[0].poam_id == "2025-CIS0006-B" + + # Verify existing match + assert len(diff.existing_matches) == 1 + assert diff.existing_matches[0].poam.poam_id == "2025-CIS0005-A" + assert diff.existing_matches[0].finding.finding_id == "CIS-001" diff --git a/tests/test_diff_apply.py b/tests/test_diff_apply.py new file mode 100644 index 0000000..1df67c4 --- /dev/null +++ b/tests/test_diff_apply.py @@ -0,0 +1,163 @@ +""" +Tests for diff_apply module. +""" +import json +import tempfile +from pathlib import Path +import pytest +from tools.diff_apply import merge_diffs + + +def test_merge_diffs_single_file(): + """Test merging a single diff file.""" + # Create a temporary diff file + diff_data = { + "new_poams": [{"poam_id": "2025-TEST001", "weakness_name": "Test Weakness"}], + "reopen_poams": [], + "close_poams": ["2025-OLD001"], + "proposed_configuration_findings": [], + "closed_configuration_findings": [] + } + + with tempfile.NamedTemporaryFile(mode='w', suffix='.json', delete=False) as f: + json.dump(diff_data, f) + temp_file = Path(f.name) + + try: + result = merge_diffs([temp_file]) + + assert result["new_poams"] == [{"poam_id": "2025-TEST001", "weakness_name": "Test Weakness"}] + assert result["reopen_poams"] == [] + assert result["close_poams"] == ["2025-OLD001"] + assert result["proposed_configuration_findings"] == [] + assert result["closed_configuration_findings"] == [] + finally: + temp_file.unlink() + + +def test_merge_diffs_multiple_files(): + """Test merging multiple diff files.""" + # Create first diff file + diff1_data = { + "new_poams": [{"poam_id": "2025-TEST001", "weakness_name": "Test Weakness 1"}], + "reopen_poams": [{"poam_id": "2025-REOPEN001"}], + "close_poams": ["2025-OLD001"], + "proposed_configuration_findings": [], + "closed_configuration_findings": [] + } + + # Create second diff file + diff2_data = { + "new_poams": [{"poam_id": "2025-TEST002", "weakness_name": "Test Weakness 2"}], + "reopen_poams": [], + "close_poams": ["2025-OLD002"], + "proposed_configuration_findings": [{"poam_id": "2025-CIS001", "weakness_name": "Config Issue"}], + "closed_configuration_findings": [] + } + + with tempfile.NamedTemporaryFile(mode='w', suffix='.json', delete=False) as f1: + json.dump(diff1_data, f1) + temp_file1 = Path(f1.name) + + with tempfile.NamedTemporaryFile(mode='w', suffix='.json', delete=False) as f2: + json.dump(diff2_data, f2) + temp_file2 = Path(f2.name) + + try: + result = merge_diffs([temp_file1, temp_file2]) + + # Check that all items from both files are merged + assert len(result["new_poams"]) == 2 + assert {"poam_id": "2025-TEST001", "weakness_name": "Test Weakness 1"} in result["new_poams"] + assert {"poam_id": "2025-TEST002", "weakness_name": "Test Weakness 2"} in result["new_poams"] + + assert len(result["reopen_poams"]) == 1 + assert {"poam_id": "2025-REOPEN001"} in result["reopen_poams"] + + assert len(result["close_poams"]) == 2 + assert "2025-OLD001" in result["close_poams"] + assert "2025-OLD002" in result["close_poams"] + + assert len(result["proposed_configuration_findings"]) == 1 + assert {"poam_id": "2025-CIS001", "weakness_name": "Config Issue"} in result["proposed_configuration_findings"] + + assert result["closed_configuration_findings"] == [] + finally: + temp_file1.unlink() + temp_file2.unlink() + + +def test_merge_diffs_empty_lists(): + """Test merging files with empty lists.""" + diff_data = { + "new_poams": [], + "reopen_poams": [], + "close_poams": [], + "proposed_configuration_findings": [], + "closed_configuration_findings": [] + } + + with tempfile.NamedTemporaryFile(mode='w', suffix='.json', delete=False) as f: + json.dump(diff_data, f) + temp_file = Path(f.name) + + try: + result = merge_diffs([temp_file]) + + assert result["new_poams"] == [] + assert result["reopen_poams"] == [] + assert result["close_poams"] == [] + assert result["proposed_configuration_findings"] == [] + assert result["closed_configuration_findings"] == [] + finally: + temp_file.unlink() + + +def test_merge_diffs_no_files(): + """Test that merge_diffs raises error with no files.""" + with pytest.raises(ValueError, match="No diff files provided"): + merge_diffs([]) + + +def test_merge_diffs_nonexistent_file(): + """Test that merge_diffs raises error with nonexistent file.""" + with pytest.raises(ValueError, match="Diff file does not exist"): + merge_diffs([Path("nonexistent.json")]) + + +def test_merge_diffs_invalid_json(): + """Test that merge_diffs raises error with invalid JSON.""" + with tempfile.NamedTemporaryFile(mode='w', suffix='.json', delete=False) as f: + f.write("invalid json content") + temp_file = Path(f.name) + + try: + with pytest.raises(ValueError, match="Error reading diff file"): + merge_diffs([temp_file]) + finally: + temp_file.unlink() + + +def test_merge_diffs_missing_keys(): + """Test merging files with missing keys (should be handled gracefully).""" + # File with only some keys + diff_data = { + "new_poams": [{"poam_id": "2025-TEST001"}], + "close_poams": ["2025-OLD001"] + # Missing other keys + } + + with tempfile.NamedTemporaryFile(mode='w', suffix='.json', delete=False) as f: + json.dump(diff_data, f) + temp_file = Path(f.name) + + try: + result = merge_diffs([temp_file]) + + assert result["new_poams"] == [{"poam_id": "2025-TEST001"}] + assert result["close_poams"] == ["2025-OLD001"] + assert result["reopen_poams"] == [] + assert result["proposed_configuration_findings"] == [] + assert result["closed_configuration_findings"] == [] + finally: + temp_file.unlink() diff --git a/tests/test_poam.py b/tests/test_poam.py new file mode 100644 index 0000000..605f4e5 --- /dev/null +++ b/tests/test_poam.py @@ -0,0 +1,36 @@ +""" +Tests for the POAM module. +""" +import pytest +from tools.poam import convert_to_snake_case + +@pytest.mark.parametrize("input_str,expected", [ + ("Weakness Name", "weakness_name"), + ("POAM ID", "poam_id"), + ("Point of Contact", "point_of_contact"), + ("CVE", "cve"), + ("Auto-Approve", "auto_approve"), + ("Binding Operational Directive 22-01 tracking", "binding_operational_directive_22_01_tracking"), + ("Last Vendor Check-in Date", "last_vendor_check_in_date"), + ("Resources Required", "resources_required"), + ("Overall Remediation Plan", "overall_remediation_plan"), + ("Original Detection Date", "original_detection_date"), + ("Status Date", "status_date"), + ("Vendor Dependency", "vendor_dependency"), + ("Original Risk Rating", "original_risk_rating"), + ("False Positive", "false_positive"), + ("Operational Requirement", "operational_requirement"), + ("Supporting Documents", "supporting_documents"), + ("Comments", "comments"), + ("Service Name", "service_name"), + # Edge cases + ("", ""), + ("alreadysnakecase", "alreadysnakecase"), + ("UPPER CASE", "upper_case"), + ("Mixed Case With-Hyphen", "mixed_case_with_hyphen"), + (" Spaces Around ", "spaces_around"), + ("multiple spaces between", "multiple_spaces_between"), +]) +def test_convert_to_snake_case(input_str, expected): + """Test the convert_to_snake_case function with various inputs.""" + assert convert_to_snake_case(input_str) == expected \ No newline at end of file diff --git a/tests/test_poam_generator.py b/tests/test_poam_generator.py new file mode 100644 index 0000000..bfde1e6 --- /dev/null +++ b/tests/test_poam_generator.py @@ -0,0 +1,146 @@ +""" +Tests for POAM generation functionality. +""" +from datetime import datetime +import pytest +from tools.findings import Finding +from tools.trivy.poam_generator import ( + parse_trivy_id, + get_next_trivy_id, + findings_to_poam, + group_findings_by_weakness, + generate_poams_from_findings +) + +def create_test_finding(finding_id: str, weakness_name: str, asset_identifier: str) -> Finding: + """Helper to create a test Finding with minimal required fields.""" + return Finding( + finding_id=finding_id, + controls="Test controls", + weakness_name=weakness_name, + weakness_description="Test description", + weakness_detector_source="Trivy", + weakness_source_identifier="TEST-001", + asset_identifier=asset_identifier, + point_of_contact="test@example.com", + resources_required=None, + overall_remediation_plan="Test plan", + original_detection_date=datetime.now(), + scheduled_completion_date=datetime.now(), + planned_milestones="Test milestones", + milestone_changes="None", + status_date=datetime.now(), + vendor_dependency="None", + last_vendor_check_in_date=None, + vendor_dependent_product_name="None", + original_risk_rating="Low", + adjusted_risk_rating=None, + risk_adjustment="None", + false_positive="No", + operational_requirement="No", + deviation_rationale=None, + supporting_documents=None, + comments=None, + auto_approve="No", + binding_operational_directive_22_01_tracking="No", + binding_operational_directive_22_01_due_date=None, + cve=None, + service_name="test-service" + ) + +def test_parse_trivy_id(): + """Test parsing Trivy POAM IDs.""" + # Test valid IDs + assert parse_trivy_id("2024-TRIVY0001") == (2024, 1) + assert parse_trivy_id("2023-TRIVY9999") == (2023, 9999) + + # Test invalid IDs + with pytest.raises(ValueError): + parse_trivy_id("invalid") + with pytest.raises(ValueError): + parse_trivy_id("2024-TRIVY") + with pytest.raises(ValueError): + parse_trivy_id("2024-trivy0001") + +def test_get_next_trivy_id(): + """Test generating next Trivy POAM ID.""" + # Test with no existing IDs + assert get_next_trivy_id([], 2024) == "2024-TRIVY0001" + + # Test with existing IDs + existing_ids = ["2024-TRIVY0001", "2024-TRIVY0002", "2023-TRIVY0001"] + assert get_next_trivy_id(existing_ids, 2024) == "2024-TRIVY0003" + + # Test with invalid IDs in the list + existing_ids = ["2024-TRIVY0001", "invalid", "2024-TRIVY0003"] + assert get_next_trivy_id(existing_ids, 2024) == "2024-TRIVY0004" + +def test_findings_to_poam(): + """Test converting findings to a POAM.""" + findings = [ + create_test_finding("TRIVY-001", "SQL Injection", "app-1"), + create_test_finding("TRIVY-002", "SQL Injection", "app-2") + ] + + poam = findings_to_poam(findings, "2024-TRIVY0001") + + # Verify basic fields + assert poam.poam_id == "2024-TRIVY0001" + assert poam.weakness_name == "SQL Injection" + assert poam.asset_identifier == "app-1, app-2" + assert poam.comments == "TRIVY-001, TRIVY-002" + + # Test with different weakness names + findings = [ + create_test_finding("TRIVY-001", "SQL Injection", "app-1"), + create_test_finding("TRIVY-002", "XSS", "app-2") + ] + with pytest.raises(ValueError): + findings_to_poam(findings, "2024-TRIVY0001") + + # Test with empty findings list + with pytest.raises(ValueError): + findings_to_poam([], "2024-TRIVY0001") + +def test_group_findings_by_weakness(): + """Test grouping findings by weakness name.""" + findings = [ + create_test_finding("TRIVY-001", "SQL Injection", "app-1"), + create_test_finding("TRIVY-002", "XSS", "app-2"), + create_test_finding("TRIVY-003", "SQL Injection", "app-3") + ] + + groups = group_findings_by_weakness(findings) + + assert len(groups) == 2 + assert len(groups["SQL Injection"]) == 2 + assert len(groups["XSS"]) == 1 + assert {f.finding_id for f in groups["SQL Injection"]} == {"TRIVY-001", "TRIVY-003"} + assert {f.finding_id for f in groups["XSS"]} == {"TRIVY-002"} + +def test_generate_poams_from_findings(): + """Test generating POAMs from findings.""" + findings = [ + create_test_finding("TRIVY-001", "SQL Injection", "app-1"), + create_test_finding("TRIVY-002", "XSS", "app-2"), + create_test_finding("TRIVY-003", "SQL Injection", "app-3") + ] + + existing_poam_ids = ["2024-TRIVY0001", "2024-TRIVY0002"] + result = generate_poams_from_findings(findings, existing_poam_ids, 2024) + + assert len(result) == 2 # Two groups: SQL Injection and XSS + + # Check SQL Injection group + sql_group = next(r for r in result if r[1].weakness_name == "SQL Injection") + assert len(sql_group[0]) == 2 # Two findings + assert sql_group[1].poam_id == "2024-TRIVY0003" + assert sql_group[1].asset_identifier == "app-1, app-3" + assert sql_group[1].comments == "TRIVY-001, TRIVY-003" + + # Check XSS group + xss_group = next(r for r in result if r[1].weakness_name == "XSS") + assert len(xss_group[0]) == 1 # One finding + assert xss_group[1].poam_id == "2024-TRIVY0004" + assert xss_group[1].asset_identifier == "app-2" + assert xss_group[1].comments == "TRIVY-002" \ No newline at end of file diff --git a/tools/__init__.py b/tools/__init__.py new file mode 100644 index 0000000..48ad642 --- /dev/null +++ b/tools/__init__.py @@ -0,0 +1,7 @@ +""" +Tools package for security tracker CLI utilities. +""" + +""" +Tools package initialization. +""" \ No newline at end of file diff --git a/tools/cis/converter.py b/tools/cis/converter.py new file mode 100644 index 0000000..be56912 --- /dev/null +++ b/tools/cis/converter.py @@ -0,0 +1,143 @@ +""" +Module for converting CIS scan reports to findings. +""" +from pathlib import Path +import pandas as pd +from datetime import datetime, timedelta +from typing import List +import json + +from ..findings import Finding + +def get_cvss_range(cvss: str) -> str: + """Convert CVSS score to range category.""" + try: + score = float(cvss) + if score >= 9.0: + return "Critical" + elif score >= 7.0: + return "High" + elif score >= 4.0: + return "Medium" + elif score > 0: + return "Low" + else: + return "Info" + except (ValueError, TypeError): + if not cvss: + return "Info" + else: + return "Unknown" + +def calculate_due_date(cvss: str, detection_date: datetime) -> datetime: + """Calculate due date based on severity level.""" + severity_mapping = { + 'Critical': 15, + 'High': 30, + 'Medium': 90, + 'Low': 180, + 'Info': 180 + } + severity = get_cvss_range(cvss) + days = severity_mapping.get(severity, 180) # Default to 180 days for unknown + return detection_date + timedelta(days=days) + +def convert_csv_to_findings(input_file: Path) -> List[Finding]: + """ + Convert a CIS CSV file to a list of findings. + + Args: + input_file: Path to the input CSV file + + Returns: + List of Finding objects + """ + # Read the CSV file + df = pd.read_csv(input_file) + + # Extract date from filename + date_str = input_file.stem.split(" - ")[-1] + detection_date = datetime.strptime(date_str, "%Y-%m-%d") + + findings = [] + + # Process each row + for _, row in df.iterrows(): + # Split failures into individual asset identifiers + failures = row['Failures'].strip().split('\n') + + # Calculate completion date based on CVSS + completion_date = calculate_due_date(row['CVSS'], detection_date) + + # Create a finding for each failure + for failure in failures: + finding = Finding( + finding_id=f"CIS-{row['CIS_ID']}-{len(findings)+1:04d}", + controls="CM-6", + weakness_name=row['Title'], + weakness_description=row['Description'], + weakness_detector_source=input_file.name, + weakness_source_identifier="CIS", + asset_identifier=failure.strip(), + point_of_contact="Chris Llanwarne", + resources_required=None, + overall_remediation_plan="Perform necessary updates to resolve the vulnerability", + original_detection_date=detection_date, + scheduled_completion_date=completion_date, + planned_milestones=f"(1) {completion_date.strftime('%Y-%m-%d')} Perform necessary updates to resolve the vulnerability", + milestone_changes="", + status_date=detection_date, + vendor_dependency="No", + last_vendor_check_in_date=None, + vendor_dependent_product_name="", + original_risk_rating=get_cvss_range(row['CVSS']), + adjusted_risk_rating="N/A", + risk_adjustment="", + false_positive="No", + operational_requirement="No", + deviation_rationale=None, + supporting_documents=None, + comments=None, + auto_approve="No", + binding_operational_directive_22_01_tracking="", + binding_operational_directive_22_01_due_date=None, + cve=None, + service_name="Hail" + ) + findings.append(finding) + + return findings + +def convert_to_findings_file(input_file: Path, output_file: Path = None) -> Path: + """ + Convert a CIS CSV file to a findings JSON file. + + Args: + input_file: Path to the input CSV file + output_file: Optional output file path. If None, uses input file with .findings.json extension + + Returns: + Path to the output JSON file + """ + # Generate findings + findings = convert_csv_to_findings(input_file) + + # Convert findings to dictionaries + findings_data = [] + for finding in findings: + finding_dict = vars(finding) + # Convert datetime objects to strings + for key, value in finding_dict.items(): + if isinstance(value, datetime): + finding_dict[key] = value.strftime("%Y-%m-%d") + findings_data.append(finding_dict) + + # Determine output file path + if output_file is None: + output_file = input_file.with_suffix('.findings.json') + + # Write to JSON file + with open(output_file, 'w') as f: + json.dump(findings_data, f, indent=2) + + return output_file \ No newline at end of file diff --git a/tools/cis/diff.py b/tools/cis/diff.py new file mode 100644 index 0000000..3f94f3e --- /dev/null +++ b/tools/cis/diff.py @@ -0,0 +1,66 @@ +""" +Module for comparing CIS findings against existing configuration findings. +""" +from pathlib import Path +from typing import List, Tuple +import pandas as pd +import re + +from ..findings import Finding +from ..poam import PoamFile, PoamEntry +from ..diff import PoamFileDiff, compare_findings_to_poams +from .poam_generator import generate_poams_from_findings + +def get_cis_configuration_findings(poam_file: PoamFile) -> List[PoamEntry]: + """ + Get CIS configuration findings from a POAM file. + + Args: + poam_file: PoamFile instance + + Returns: + List of configuration findings from the Configuration Findings sheet + """ + # Pattern matches YYYY-CISXXXX where XXXX is 4 or more digits + cis_pattern = r'^\d{4}-CIS\d{4,}$' + + # Get configuration findings from the Configuration Findings sheet + config_df = poam_file.workbook.parse( + sheet_name="Configuration Findings", + header=4 # 0-based index for row 5 + ) + config_findings = [ + PoamEntry.from_dict(row) + for _, row in config_df.iterrows() + if pd.notna(row.get('POAM ID')) and re.match(cis_pattern, str(row['POAM ID'])) + ] + + return config_findings + +def compare_findings_to_cis_poams(findings: List[Finding], poam_file: Path) -> PoamFileDiff: + """ + Compare a list of findings against CIS configuration findings. + + Args: + findings: List of current findings from CIS + poam_file: Path to Excel file containing CIS configuration findings + + Returns: + PoamFileDiff containing new, existing, and closed configuration findings + """ + # Load CIS configuration findings + poam_file_handler = PoamFile(poam_file) + config_findings = get_cis_configuration_findings(poam_file_handler) + + # Get all POAM IDs for generating new ones + all_poam_ids = [p.poam_id for p in config_findings] + + # Compare findings using shared function, with empty closed_poams list and store as configuration findings + return compare_findings_to_poams( + findings=findings, + open_poams=config_findings, + closed_poams=[], # No closed POAMs for CIS + existing_poam_ids=all_poam_ids, + poam_generator=generate_poams_from_findings, + store_as_configuration_findings=True + ) diff --git a/tools/cis/poam_generator.py b/tools/cis/poam_generator.py new file mode 100644 index 0000000..0d11b37 --- /dev/null +++ b/tools/cis/poam_generator.py @@ -0,0 +1,152 @@ +""" +Module for generating POAMs from CIS findings. +""" +from datetime import datetime, timedelta, timezone +from typing import List, Tuple, Dict +from collections import defaultdict + +from ..findings import Finding +from ..poam import PoamEntry + +def _get_next_poam_id(existing_poam_ids: List[str], current_year: int = None) -> str: + """ + Generate the next available POAM ID for CIS findings. + + Args: + existing_poam_ids: List of existing POAM IDs + current_year: Year to use for POAM ID (defaults to current year) + + Returns: + Next available POAM ID in format YYYY-CISXXXX + """ + if current_year is None: + current_year = datetime.now(timezone.utc).year + + # Filter to just this year's CIS POAMs + year_prefix = f"{current_year}-CIS" + year_poams = [p for p in existing_poam_ids if p.startswith(year_prefix)] + + if not year_poams: + # First POAM for this year + return f"{year_prefix}0001" + + # Get highest number used + highest = max(int(p[-4:]) for p in year_poams) + + # Return next number + return f"{year_prefix}{highest + 1:04d}" + +def _get_completion_date(risk_rating: str) -> datetime: + """ + Calculate completion date based on risk rating. + + Args: + risk_rating: Risk rating of the finding + + Returns: + Datetime object for completion date + """ + today = datetime.now(timezone.utc) + + if risk_rating.lower() == "critical": + return today + timedelta(days=15) + elif risk_rating.lower() == "high": + return today + timedelta(days=30) + elif risk_rating.lower() == "moderate": + return today + timedelta(days=90) + else: # Low + return today + timedelta(days=180) + +def _group_findings_by_weakness_and_date(findings: List[Finding]) -> Dict[tuple, List[Finding]]: + """ + Group findings by weakness name and completion date. + + Args: + findings: List of findings to group + + Returns: + Dictionary mapping (weakness_name, completion_date) to list of findings + """ + groups = defaultdict(list) + for finding in findings: + completion_date = _get_completion_date(finding.original_risk_rating) + # Use only the date part for grouping key, but store the full datetime with the finding + key = (finding.weakness_name, completion_date.date()) + groups[key].append((finding, completion_date)) + return dict(groups) + +def generate_poams_from_findings(findings: List[Finding], existing_poam_ids: List[str], current_year: int = None) -> List[Tuple[List[Finding], PoamEntry]]: + """ + Generate POAMs from CIS findings. + + Args: + findings: List of findings to generate POAMs for + existing_poam_ids: List of existing POAM IDs + current_year: Year to use for POAM IDs (defaults to current year) + + Returns: + List of tuples containing (findings, generated_poam) + """ + result = [] + + # Group findings by weakness name and completion date + grouped_findings = _group_findings_by_weakness_and_date(findings) + + for (weakness_name, _), group in grouped_findings.items(): + # Unpack findings and their completion dates + findings_list = [f for f, _ in group] + first_finding = findings_list[0] + completion_date = group[0][1] + + # Get earliest detection date from group + detection_date = min(f.original_detection_date for f in findings_list) + + # Get highest risk rating from group + risk_rating = max(f.original_risk_rating for f in findings_list) + + # Combine asset identifiers + asset_ids = sorted(set(f.asset_identifier for f in findings_list)) + combined_asset_id = ", ".join(asset_ids) + + # Generate POAM ID + poam_id = _get_next_poam_id(existing_poam_ids, current_year) + existing_poam_ids.append(poam_id) # Add to list so next ID will be different + + # Create POAM entry + poam = PoamEntry( + poam_id=poam_id, + controls=first_finding.controls, + weakness_name=first_finding.weakness_name, + weakness_description=first_finding.weakness_description, + weakness_detector_source=first_finding.weakness_detector_source, + weakness_source_identifier=first_finding.weakness_source_identifier, + asset_identifier=combined_asset_id, + point_of_contact=first_finding.point_of_contact, + resources_required=first_finding.resources_required, + overall_remediation_plan=first_finding.overall_remediation_plan, + original_detection_date=detection_date, + scheduled_completion_date=completion_date, + planned_milestones=first_finding.planned_milestones, + milestone_changes=first_finding.milestone_changes, + status_date=datetime.now(timezone.utc), + vendor_dependency=first_finding.vendor_dependency, + last_vendor_check_in_date=first_finding.last_vendor_check_in_date, + vendor_dependent_product_name=first_finding.vendor_dependent_product_name, + original_risk_rating=risk_rating, + adjusted_risk_rating=first_finding.adjusted_risk_rating, + risk_adjustment=first_finding.risk_adjustment, + false_positive=first_finding.false_positive, + operational_requirement=first_finding.operational_requirement, + deviation_rationale=first_finding.deviation_rationale, + supporting_documents=first_finding.supporting_documents, + comments="", # No finding IDs for CIS findings + auto_approve=first_finding.auto_approve, + binding_operational_directive_22_01_tracking=first_finding.binding_operational_directive_22_01_tracking, + binding_operational_directive_22_01_due_date=first_finding.binding_operational_directive_22_01_due_date, + cve=first_finding.cve, + service_name=first_finding.service_name + ) + + result.append((findings_list, poam)) + + return result \ No newline at end of file diff --git a/tools/cis/splitter.py b/tools/cis/splitter.py new file mode 100644 index 0000000..36c9d20 --- /dev/null +++ b/tools/cis/splitter.py @@ -0,0 +1,68 @@ +""" +Module for handling CIS scan reports. +""" +from pathlib import Path +import pandas as pd +from datetime import datetime +import os + +def split_connected_sheet(input_file: Path, output_dir: Path = None) -> list[Path]: + """ + Split a CIS connected sheet Excel file into multiple CSV files by date. + + Args: + input_file: Path to the input Excel file + output_dir: Optional output directory. If None, uses input directory/Divided CIS Scans + + Returns: + List of paths to the generated CSV files + + Notes: + - Creates files in a "Divided CIS Scans" subdirectory + - Names files as " - YYYY-MM-DD.csv" + - Preserves original row order + - Skips writing if file for a date already exists + """ + # Read the Excel file + df = pd.read_excel(input_file) + + # Output directory is input directory with a "Divided CIS Scans" subdirectory + if output_dir is None: + output_dir = input_file.parent / "Divided CIS Scans" + + # Ensure output directory exists + output_dir.mkdir(exist_ok=True) + + # Get base filename without "(Connected Sheet)" suffix + base_name = input_file.stem + + # Remove "(Connected Sheet)" (and anything after it) + if "(Connected Sheet)" in base_name: + base_name = base_name.split("(Connected Sheet)")[0].strip() + + # Group by date and write separate files + output_files = [] + for date, group in df.groupby("Date"): + # Parse date and format filename + try: + # Try to parse date if it's not already a datetime + if not isinstance(date, datetime): + date = pd.to_datetime(date) + date_str = date.strftime("%Y-%m-%d") + except: + # If date parsing fails, use the raw value + date_str = str(date) + + # Generate output path + output_file = output_dir / f"{base_name} - {date_str}.csv" + + # Skip if file already exists + if not output_file.exists(): + # Sort by original index to preserve row order + group = group.sort_index() + + # Write to CSV + group.to_csv(output_file, index=False) + output_files.append(output_file) + + return output_files \ No newline at end of file diff --git a/tools/diff.py b/tools/diff.py new file mode 100644 index 0000000..dbe8748 --- /dev/null +++ b/tools/diff.py @@ -0,0 +1,319 @@ +""" +Module for comparing findings against existing POAMs. +""" +from dataclasses import dataclass +from typing import List, Optional, Tuple, Dict, Any, Union +from pathlib import Path +from datetime import datetime + +from .findings import Finding +from .poam import PoamFile, PoamEntry + +@dataclass +class FindingPoamMatch: + """Represents a match between a finding and an existing POAM.""" + finding: Finding + poam: PoamEntry + +@dataclass +class PoamFileDiff: + """Represents the difference between current findings and existing POAMs.""" + new_findings: List[Finding] # Findings without corresponding POAMs + existing_matches: List[FindingPoamMatch] # Findings matched to existing POAMs + closed_poams: List[PoamEntry] # POAMs without corresponding findings + reopened_findings: List[FindingPoamMatch] # Findings that match previously closed POAMs + proposed_poams: List[Tuple[List[Finding], PoamEntry]] # Proposed new POAMs with their findings + proposed_configuration_findings: List[Tuple[List[Finding], PoamEntry]] # Proposed new configuration findings + closed_configuration_findings: List[PoamEntry] # Configuration findings without matches + + def to_json(self) -> Dict[str, Any]: + """ + Convert the diff results to a JSON-serializable dictionary. + + Returns: + Dictionary containing the diff results in a structured format + """ + def format_datetime(dt: Union[datetime, str]) -> str: + if isinstance(dt, str): + return dt + else: + return dt.strftime("%Y-%m-%d") if dt else None + + def poam_to_full_dict(poam: PoamEntry) -> Dict[str, Any]: + """Convert a POAM to a complete dictionary with all fields.""" + return { + "poam_id": poam.poam_id, + "controls": poam.controls, + "weakness_name": poam.weakness_name, + "weakness_description": poam.weakness_description, + "weakness_detector_source": poam.weakness_detector_source, + "weakness_source_identifier": poam.weakness_source_identifier, + "asset_identifier": poam.asset_identifier, + "point_of_contact": poam.point_of_contact, + "resources_required": poam.resources_required, + "overall_remediation_plan": poam.overall_remediation_plan, + "original_detection_date": format_datetime(poam.original_detection_date), + "scheduled_completion_date": format_datetime(poam.scheduled_completion_date), + "planned_milestones": poam.planned_milestones, + "milestone_changes": poam.milestone_changes, + "status_date": format_datetime(poam.status_date), + "vendor_dependency": poam.vendor_dependency, + "last_vendor_check_in_date": format_datetime(poam.last_vendor_check_in_date), + "vendor_dependent_product_name": poam.vendor_dependent_product_name, + "original_risk_rating": poam.original_risk_rating, + "adjusted_risk_rating": poam.adjusted_risk_rating, + "risk_adjustment": poam.risk_adjustment, + "false_positive": poam.false_positive, + "operational_requirement": poam.operational_requirement, + "deviation_rationale": poam.deviation_rationale, + "supporting_documents": poam.supporting_documents, + "comments": poam.comments, + "auto_approve": poam.auto_approve, + "binding_operational_directive_22_01_tracking": poam.binding_operational_directive_22_01_tracking, + "binding_operational_directive_22_01_due_date": format_datetime(poam.binding_operational_directive_22_01_due_date), + "cve": poam.cve, + "service_name": poam.service_name + } + + def finding_to_dict(finding: Finding) -> Dict[str, Any]: + return { + "finding_id": finding.finding_id, + "weakness_name": finding.weakness_name, + "asset_identifier": finding.asset_identifier, + "original_detection_date": format_datetime(finding.original_detection_date), + "original_risk_rating": finding.original_risk_rating, + "cve": finding.cve, + "service_name": finding.service_name + } + + result = { + "metadata": { + "new_findings_count": len(self.new_findings), + "existing_matches_count": len(self.existing_matches), + "closed_poams_count": len(self.closed_poams), + "reopened_findings_count": len(self.reopened_findings), + "proposed_poams_count": len(self.proposed_poams) + }, + "new_poams": [ + { + "poam": poam_to_full_dict(poam), + "findings": [finding_to_dict(f) for f in findings], + "finding_ids": [f.finding_id for f in findings] + } + for findings, poam in self.proposed_poams + ], + "reopen_poams": [ + { + "poam_id": match.poam.poam_id, + "finding_id": match.finding.finding_id + } + for match in self.reopened_findings + ], + "close_poams": [poam.poam_id for poam in self.closed_poams] + } + + # Add configuration findings if present + if self.proposed_configuration_findings is not None: + result["metadata"]["proposed_configuration_findings_count"] = len(self.proposed_configuration_findings) + result["proposed_configuration_findings"] = [ + { + "poam": poam_to_full_dict(poam), + "findings": [finding_to_dict(f) for f in findings], + "finding_ids": [f.finding_id for f in findings] + } + for findings, poam in self.proposed_configuration_findings + ] + + if self.closed_configuration_findings is not None: + result["metadata"]["closed_configuration_findings_count"] = len(self.closed_configuration_findings) + result["closed_configuration_findings"] = [poam.poam_id for poam in self.closed_configuration_findings] + + return result + + def print_summary(self, max_preview: int = 10) -> None: + """Print a human-readable summary of the diff.""" + # Print new findings + print("\n=== New Findings ===") + print(f"Count: {len(self.new_findings)}") + if self.new_findings: + finding_ids = [finding.finding_id for finding in self.new_findings] + print(f"Finding IDs: {', '.join(finding_ids)}") + + # Print existing matches + print("\n=== Existing Matches ===") + print(f"Count: {len(self.existing_matches)}") + if self.existing_matches: + matches = [f"{match.finding.finding_id} -> {match.poam.poam_id}" + for match in self.existing_matches[:max_preview]] + print(f"Preview of matches: {', '.join(matches)}") + if len(self.existing_matches) > max_preview: + print(f"... and {len(self.existing_matches) - max_preview} more") + + # Print reopened findings + print("\n=== Reopened Findings ===") + print(f"Count: {len(self.reopened_findings)}") + if self.reopened_findings: + matches = [f"{match.finding.finding_id} -> {match.poam.poam_id}" + for match in self.reopened_findings[:max_preview]] + print(f"Preview of matches: {', '.join(matches)}") + if len(self.reopened_findings) > max_preview: + print(f"... and {len(self.reopened_findings) - max_preview} more") + + # Print closed POAMs + print("\n=== Closed POAMs ===") + print(f"Count: {len(self.closed_poams)}") + if self.closed_poams: + poam_ids = [poam.poam_id for poam in self.closed_poams] + print(f"POAM IDs no longer active: {', '.join(poam_ids)}") + + # Print proposed POAMs + print("\n=== Proposed POAMs ===") + print(f"Count: {len(self.proposed_poams)}") + if self.proposed_poams: + for findings, poam in self.proposed_poams[:max_preview]: + finding_ids = [f.finding_id for f in findings] + print(f"{', '.join(finding_ids)} => {poam.poam_id}") + if len(self.proposed_poams) > max_preview: + print(f"... and {len(self.proposed_poams) - max_preview} more") + + # Show sample of first proposed POAM + print("\nSample new POAM:") + sample_findings, sample_poam = self.proposed_poams[0] + print(f"POAM ID: {sample_poam.poam_id}") + print(f"Weakness Name: {sample_poam.weakness_name}") + print(f"Asset Identifiers: {sample_poam.asset_identifier}") + print(f"Finding IDs: {sample_poam.comments}") + # Handle case where date might already be a string + detection_date = sample_poam.original_detection_date + if isinstance(detection_date, datetime): + detection_date = detection_date.strftime('%Y-%m-%d') + print(f"Detection Date: {detection_date}") + print(f"Risk Rating: {sample_poam.original_risk_rating}") + if sample_poam.cve: + print(f"CVE: {sample_poam.cve}") + + # Print configuration findings if present + print("\n=== Proposed Configuration Findings ===") + print(f"Count: {len(self.proposed_configuration_findings)}") + if self.proposed_configuration_findings: + for findings, poam in self.proposed_configuration_findings[:max_preview]: + finding_ids = [f.finding_id for f in findings] + print(f"{', '.join(finding_ids)} => {poam.poam_id}") + if len(self.proposed_configuration_findings) > max_preview: + print(f"... and {len(self.proposed_configuration_findings) - max_preview} more") + + # Show sample of first proposed configuration finding + print("\nSample new Configuration Finding:") + sample_findings, sample_poam = self.proposed_configuration_findings[0] + print(f"POAM ID: {sample_poam.poam_id}") + print(f"Weakness Name: {sample_poam.weakness_name}") + print(f"Asset Identifiers: {sample_poam.asset_identifier}") + print(f"Finding IDs: {sample_poam.comments}") + detection_date = sample_poam.original_detection_date + if isinstance(detection_date, datetime): + detection_date = detection_date.strftime('%Y-%m-%d') + print(f"Detection Date: {detection_date}") + print(f"Risk Rating: {sample_poam.original_risk_rating}") + if sample_poam.cve: + print(f"CVE: {sample_poam.cve}") + + print("\n=== Closed Configuration Findings ===") + print(f"Count: {len(self.closed_configuration_findings)}") + if self.closed_configuration_findings: + poam_ids = [poam.poam_id for poam in self.closed_configuration_findings] + print(f"Configuration Finding IDs no longer active: {', '.join(poam_ids)}") + +def _is_exact_match(str1: str, str2: str) -> bool: + """Check if two strings match exactly, ignoring case.""" + if not str1 or not str2: + return False + return str1.lower().strip() == str2.lower().strip() + +def _is_asset_covered(finding_asset: str, poam_assets: str) -> bool: + """ + Check if the finding's asset is included in the POAM's asset list. + + Args: + finding_asset: Asset identifier from the finding + poam_assets: Asset identifier field from the POAM (may contain multiple assets) + + Returns: + True if the finding's asset is contained within the POAM's asset list + """ + if not finding_asset or not poam_assets: + return False + return finding_asset.lower().strip() in poam_assets.lower().strip() + +def _find_matching_poam(finding: Finding, poams: List[PoamEntry]) -> Optional[FindingPoamMatch]: + """ + Find a matching POAM for a given finding. + + Args: + finding: Finding to match + poams: List of POAMs to search + + Returns: + FindingPoamMatch if a match is found, None otherwise + """ + for poam in poams: + # Match based on exact weakness name match and asset coverage + if _is_exact_match(finding.weakness_name, poam.weakness_name) and \ + _is_asset_covered(finding.asset_identifier, poam.asset_identifier): + return FindingPoamMatch(finding=finding, poam=poam) + return None + +def compare_findings_to_poams(findings: List[Finding], + open_poams: List[PoamEntry], + closed_poams: List[PoamEntry], + existing_poam_ids: List[str], + poam_generator, + store_as_configuration_findings: bool = False) -> PoamFileDiff: + """ + Compare a list of findings against existing POAMs. + + Args: + findings: List of current findings + open_poams: List of open POAMs + closed_poams: List of closed POAMs + existing_poam_ids: List of all existing POAM IDs + poam_generator: Function to generate POAMs from findings + store_as_configuration_findings: Whether to store results in configuration findings fields + + Returns: + PoamFileDiff containing new, existing, closed, and reopened findings + """ + # Track which POAMs are matched + matched_poams = set() + new_findings = [] + existing_matches = [] + reopened_findings = [] + + # First check for matches against open POAMs + for finding in findings: + match = _find_matching_poam(finding, open_poams) + if match: + existing_matches.append(match) + matched_poams.add(match.poam) + else: + # If no match in open POAMs, check closed POAMs + closed_match = _find_matching_poam(finding, closed_poams) + if closed_match: + reopened_findings.append(closed_match) + else: + new_findings.append(finding) + + # Find closed POAMs (those without matches) + closed_poams = [poam for poam in open_poams if poam not in matched_poams] + + # Generate proposed POAMs for new findings + proposed_poams = poam_generator(new_findings, existing_poam_ids) + + return PoamFileDiff( + new_findings=new_findings, + existing_matches=existing_matches, + closed_poams=closed_poams if not store_as_configuration_findings else [], + reopened_findings=reopened_findings, + proposed_poams=proposed_poams if not store_as_configuration_findings else [], + proposed_configuration_findings=proposed_poams if store_as_configuration_findings else [], + closed_configuration_findings=closed_poams if store_as_configuration_findings else [] + ) diff --git a/tools/diff_apply.py b/tools/diff_apply.py new file mode 100644 index 0000000..c745e17 --- /dev/null +++ b/tools/diff_apply.py @@ -0,0 +1,237 @@ +""" +Module for applying diff changes to POAM Excel files. +""" +from pathlib import Path +import json +from typing import Dict, Any, List +from datetime import datetime +import shutil +import openpyxl + +def create_updateable_copy(file_path: Path, output_file: Path = None) -> Path: + """Create an updateable copy of the Excel file so that the original is not modified.""" + timestamp = datetime.now().strftime('%Y%m%d-%H%M%S') + if output_file is None: + new_file_path = file_path.parent / f"{file_path.stem}-diff-applied-{timestamp}{file_path.suffix}" + else: + new_file_path = output_file + shutil.copy2(file_path, new_file_path) + return new_file_path + +def dict_to_row(data: Dict[str, Any]) -> Dict[str, Any]: + """Convert a dictionary to row format.""" + excel_mapping = { + "poam_id": "POAM ID", + "controls": "Controls", + "weakness_name": "Weakness Name", + "weakness_description": "Weakness Description", + "weakness_detector_source": "Weakness Detector Source", + "weakness_source_identifier": "Weakness Source Identifier", + "asset_identifier": "Asset Identifier", + "point_of_contact": "Point of Contact", + "resources_required": "Resources Required", + "overall_remediation_plan": "Overall Remediation Plan", + "original_detection_date": "Original Detection Date", + "scheduled_completion_date": "Scheduled Completion Date", + "planned_milestones": "Planned Milestones", + "milestone_changes": "Milestone Changes", + "status_date": "Status Date", + "vendor_dependency": "Vendor Dependency", + "last_vendor_check_in_date": "Last Vendor Check-in Date", + "vendor_dependent_product_name": "Vendor Dependent Product Name", + "original_risk_rating": "Original Risk Rating", + "adjusted_risk_rating": "Adjusted Risk Rating", + "risk_adjustment": "Risk Adjustment", + "false_positive": "False Positive", + "operational_requirement": "Operational Requirement", + "deviation_rationale": "Deviation Rationale", + "supporting_documents": "Supporting Documents", + "comments": "Comments", + "auto_approve": "Auto Approve", + "binding_operational_directive_22_01_tracking": "Binding Operational Directive 22-01 Tracking", + "binding_operational_directive_22_01_due_date": "Binding Operational Directive 22-01 Due Date", + "cve": "CVE", + "service_name": "Service Name" + } + return {excel_mapping[k]: v for k, v in data.items() if k in excel_mapping} + +def apply_diff(poam_file: Path, diff_json: Dict[str, Any], output_file: Path = None) -> Path: + """ + Apply diff changes to a POAM Excel file. + + Args: + poam_file: Path to the POAM Excel file + diff_json: Dictionary containing diff changes + """ + # Create editable copy + editable_copy = create_updateable_copy(poam_file, output_file) + + wb = None + try: + # Load workbook from editable copy + wb = openpyxl.load_workbook(editable_copy) + + # Get sheets + if "Open POA&M Items" not in wb.sheetnames: + raise ValueError('Excel file must contain "Open POA&M Items" sheet') + open_sheet = wb["Open POA&M Items"] + + # Get or create closed sheet + if "Closed POA&M Items" not in wb.sheetnames: + raise ValueError('Excel file must contain "Closed POA&M Items" sheet') + closed_sheet = wb["Closed POA&M Items"] + + # Get or validate Configuration Findings sheet + if "Configuration Findings" not in wb.sheetnames: + raise ValueError('Excel file must contain "Configuration Findings" sheet') + config_sheet = wb["Configuration Findings"] + + # Get column indices from header row (row 5) + header_row = 5 + open_headers = {cell.value: cell.column for cell in open_sheet[header_row]} + closed_headers = {cell.value: cell.column for cell in closed_sheet[header_row]} + config_headers = {cell.value: cell.column for cell in config_sheet[header_row]} + + # Handle new POAMs - add to open sheet + if diff_json.get("new_poams"): + for new_poam in diff_json["new_poams"]: + row_data = dict_to_row(new_poam["poam"]) + # Add row at the end + next_row = open_sheet.max_row + 1 + for header, value in row_data.items(): + if header in open_headers: + open_sheet.cell(row=next_row, column=open_headers[header], value=value) + + # Handle new configuration findings - add to Configuration Findings sheet + if diff_json.get("proposed_configuration_findings"): + for new_finding in diff_json["proposed_configuration_findings"]: + row_data = dict_to_row(new_finding["poam"]) + # Add row at the top (after header) + config_sheet.insert_rows(header_row + 1) + for header, value in row_data.items(): + if header in config_headers: + config_sheet.cell(row=header_row + 1, column=config_headers[header], value=value) + + # Handle reopened POAMs - move from closed to open + if diff_json.get("reopen_poams"): + reopen_ids = {p["poam_id"] for p in diff_json["reopen_poams"]} + poam_id_col = next(col for header, col in open_headers.items() if header == "POAM ID") + + # Find and move rows + rows_to_delete = [] + for row in range(header_row + 1, closed_sheet.max_row + 1): + poam_id = closed_sheet.cell(row=row, column=poam_id_col).value + if poam_id in reopen_ids: + # Copy row to open sheet + next_row = open_sheet.max_row + 1 + for col in range(1, closed_sheet.max_column + 1): + open_sheet.cell(row=next_row, column=col, value=closed_sheet.cell(row=row, column=col).value) + rows_to_delete.append(row) + + # Delete moved rows from closed sheet (in reverse order to maintain indices) + for row in sorted(rows_to_delete, reverse=True): + closed_sheet.delete_rows(row) + + # Handle closed POAMs - move from open to closed + if diff_json.get("close_poams"): + close_ids = set(diff_json["close_poams"]) + poam_id_col = next(col for header, col in open_headers.items() if header == "POAM ID") + + # Find and move rows + rows_to_delete = [] + for row in range(header_row + 1, open_sheet.max_row + 1): + poam_id = open_sheet.cell(row=row, column=poam_id_col).value + if poam_id in close_ids: + # Copy row to closed sheet + next_row = closed_sheet.max_row + 1 + for col in range(1, open_sheet.max_column + 1): + closed_sheet.cell(row=next_row, column=col, value=open_sheet.cell(row=row, column=col).value) + rows_to_delete.append(row) + + # Delete moved rows from open sheet (in reverse order to maintain indices) + for row in sorted(rows_to_delete, reverse=True): + open_sheet.delete_rows(row) + + # Handle closed configuration findings - move to Closed POA&M Items and delete from Configuration Findings + if diff_json.get("closed_configuration_findings"): + close_ids = set(diff_json["closed_configuration_findings"]) + poam_id_col = next(col for header, col in config_headers.items() if header == "POAM ID") + + # Find and move rows + rows_to_delete = [] + for row in range(header_row + 1, config_sheet.max_row + 1): + poam_id = config_sheet.cell(row=row, column=poam_id_col).value + if poam_id in close_ids: + # Copy row to closed sheet + next_row = closed_sheet.max_row + 1 + for col in range(1, config_sheet.max_column + 1): + closed_sheet.cell(row=next_row, column=col, value=config_sheet.cell(row=row, column=col).value) + rows_to_delete.append(row) + + # Delete moved rows from config sheet (in reverse order to maintain indices) + for row in sorted(rows_to_delete, reverse=True): + config_sheet.delete_rows(row) + + # Save changes to the editable copy + wb.save(editable_copy) + return editable_copy + except Exception as e: + # If anything goes wrong, leave the half-edited copy for inspection + raise type(e)(f"Error applying diff changes. Incomplete edit saved as {editable_copy}. Error: {str(e)}") from e + finally: + if wb is not None: + wb.close() + +def merge_diffs(diff_files: List[Path]) -> Dict[str, Any]: + """ + Merge multiple diff JSON files into a single diff. + + Args: + diff_files: List of paths to diff JSON files + + Returns: + Merged diff dictionary + + Raises: + ValueError: If no diff files provided or if files are invalid + """ + if not diff_files: + raise ValueError("No diff files provided") + + merged_diff = { + "new_poams": [], + "reopen_poams": [], + "close_poams": [], + "proposed_configuration_findings": [], + "closed_configuration_findings": [] + } + + for diff_file in diff_files: + if not diff_file.exists(): + raise ValueError(f"Diff file does not exist: {diff_file}") + + try: + with open(diff_file, 'r') as f: + diff_data = json.load(f) + except (json.JSONDecodeError, IOError) as e: + raise ValueError(f"Error reading diff file {diff_file}: {e}") from e + + # Merge each section + for key in merged_diff.keys(): + if key in diff_data and isinstance(diff_data[key], list): + merged_diff[key].extend(diff_data[key]) + + return merged_diff + +def apply_diff_from_files(poam_file: Path, diff_files: List[Path], output_file: Path = None) -> Path: + """ + Apply diff changes from a JSON file to a POAM Excel file. + + Args: + poam_file: Path to the POAM Excel file + diff_files: List of paths to JSON diff files + output_file: Path to the output Excel file + """ + merged_diff = merge_diffs(diff_files) + result = apply_diff(poam_file, merged_diff, output_file) + return result \ No newline at end of file diff --git a/tools/findings.py b/tools/findings.py new file mode 100644 index 0000000..9e47f5a --- /dev/null +++ b/tools/findings.py @@ -0,0 +1,84 @@ +""" +Module for handling security findings data structures. +""" +from dataclasses import dataclass +from datetime import datetime +from typing import Optional +import pandas as pd + +@dataclass +class Finding: + """Represents a single security finding.""" + finding_id: str + controls: str + weakness_name: str + weakness_description: str + weakness_detector_source: str + weakness_source_identifier: str + asset_identifier: str + point_of_contact: str + resources_required: Optional[str] + overall_remediation_plan: str + original_detection_date: datetime + scheduled_completion_date: datetime + planned_milestones: str + milestone_changes: str + status_date: datetime + vendor_dependency: str + last_vendor_check_in_date: Optional[datetime] + vendor_dependent_product_name: str + original_risk_rating: str + adjusted_risk_rating: Optional[str] + risk_adjustment: str + false_positive: str + operational_requirement: str + deviation_rationale: Optional[str] + supporting_documents: Optional[str] + comments: Optional[str] + auto_approve: str + binding_operational_directive_22_01_tracking: str + binding_operational_directive_22_01_due_date: Optional[datetime] + cve: Optional[str] + service_name: str + + @classmethod + def from_dict(cls, data: dict) -> 'Finding': + """Create a Finding from a dictionary, handling timestamp conversion.""" + # Convert pandas timestamps to datetime + date_fields = [ + 'original_detection_date', + 'scheduled_completion_date', + 'status_date', + 'last_vendor_check_in_date', + 'binding_operational_directive_22_01_due_date' + ] + + for field in date_fields: + if field in data and pd.notna(data[field]): + if isinstance(data[field], pd._libs.tslibs.timestamps.Timestamp): + data[field] = data[field].to_pydatetime() + else: + data[field] = None + + # Convert NaN to None for optional fields + for key, value in data.items(): + if pd.isna(value): + data[key] = None + elif isinstance(value, float) and key != 'comments': # Keep numeric comments + data[key] = str(int(value)) if value.is_integer() else str(value) + elif isinstance(value, str): + data[key] = value.strip() + + # Rename ID field if necessary + if 'POAM ID' in data: + data['finding_id'] = data.pop('POAM ID') + elif 'Alert ID' in data: + data['finding_id'] = data.pop('Alert ID') + + # Convert keys to snake_case + converted_data = {} + for key, value in data.items(): + snake_key = ''.join(['_' + c.lower() if c.isupper() else c.lower() for c in key]).lstrip('_') + converted_data[snake_key] = value + + return cls(**converted_data) \ No newline at end of file diff --git a/tools/github.py b/tools/github.py new file mode 100644 index 0000000..88da878 --- /dev/null +++ b/tools/github.py @@ -0,0 +1,62 @@ +""" +Tool for interacting with GitHub APIs. +""" +import json +import subprocess +from datetime import datetime +from pathlib import Path + +from .utils import ensure_working_dir + +def download_trivy_alerts(output_dir: Path = None) -> Path: + """ + Download Trivy alerts from GitHub code scanning API. + Uses gh CLI tool to handle authentication and pagination. + + Args: + output_dir: Directory to save the alerts file. If None, uses working directory. + + Returns: + Path to the downloaded JSON file + """ + if output_dir is None: + output_dir = ensure_working_dir() + + # Generate timestamp for the filename + timestamp = datetime.now().strftime("%Y%m%d") + output_file = output_dir / f"trivy_alerts_{timestamp}.json" + + try: + # Run gh command and capture output + cmd = [ + "gh", "api", "--paginate", + "-X", "GET", + "/repos/hail-is/hail/code-scanning/alerts", + "-f", "q=branch:main tool:Trivy" + ] + + result = subprocess.run( + cmd, + capture_output=True, + text=True, + check=True + ) + + # Parse and save the JSON response + alerts = json.loads(result.stdout) + output_file.write_text(json.dumps(alerts, indent=2)) + + return output_file + + except subprocess.CalledProcessError as e: + raise Exception( + f"Failed to download alerts. Make sure:\n" + f"1. The gh CLI tool is installed\n" + f"2. You are authenticated with gh auth login\n" + f"3. You have access to the hail-is/hail repository\n" + f"\nError: {e.stderr}" + ) + except json.JSONDecodeError as e: + raise Exception(f"Failed to parse GitHub API response: {e}") + except Exception as e: + raise Exception(f"Unexpected error downloading alerts: {e}") diff --git a/tools/poam.py b/tools/poam.py new file mode 100644 index 0000000..d1386e4 --- /dev/null +++ b/tools/poam.py @@ -0,0 +1,245 @@ +""" +Tool for handling POAM Excel files. +""" +import pandas as pd +import re +import yaml +from pathlib import Path +from dataclasses import dataclass +from datetime import datetime +from typing import Optional + +def convert_to_snake_case(text: str) -> str: + """ + Convert a string to snake_case format. + + Args: + text: Input string in any format (e.g., "Weakness Name", "POAM ID", etc.) + + Returns: + String converted to snake_case format + + Examples: + >>> convert_to_snake_case("Weakness Name") + 'weakness_name' + >>> convert_to_snake_case("POAM ID") + 'poam_id' + >>> convert_to_snake_case("Auto-Approve") + 'auto_approve' + """ + if not text: + return text + + # Replace hyphens with spaces + text = text.replace('-', ' ') + + # Normalize spaces (remove extra spaces) + text = ' '.join(text.split()) + + # Convert to lowercase and replace spaces with underscores + return text.strip().lower().replace(' ', '_') + +@dataclass(frozen=True) +class PoamEntry: + """Represents a single POAM entry.""" + poam_id: str + controls: str + weakness_name: str + weakness_description: str + weakness_detector_source: str + weakness_source_identifier: str + asset_identifier: str + point_of_contact: str + resources_required: Optional[str] + overall_remediation_plan: str + original_detection_date: datetime + scheduled_completion_date: datetime + planned_milestones: str + milestone_changes: str + status_date: datetime + vendor_dependency: str + last_vendor_check_in_date: Optional[datetime] + vendor_dependent_product_name: str + original_risk_rating: str + adjusted_risk_rating: Optional[str] + risk_adjustment: str + false_positive: str + operational_requirement: str + deviation_rationale: Optional[str] + supporting_documents: Optional[str] + comments: Optional[str] + auto_approve: str + binding_operational_directive_22_01_tracking: str + binding_operational_directive_22_01_due_date: Optional[datetime] + cve: Optional[str] + service_name: str + + def __hash__(self) -> int: + """Make PoamEntry hashable based on its poam_id.""" + return hash(self.poam_id) + + def __eq__(self, other) -> bool: + """Define equality based on poam_id.""" + if not isinstance(other, PoamEntry): + return NotImplemented + return self.poam_id == other.poam_id + + @classmethod + def from_dict(cls, data: dict) -> 'PoamEntry': + """Create a PoamEntry from a dictionary, handling timestamp conversion.""" + # Convert pandas timestamps to datetime + date_fields = [ + 'original_detection_date', + 'scheduled_completion_date', + 'status_date', + 'last_vendor_check_in_date', + 'binding_operational_directive_22_01_due_date' + ] + + for field in date_fields: + if field in data and pd.notna(data[field]): + if isinstance(data[field], pd._libs.tslibs.timestamps.Timestamp): + data[field] = data[field].to_pydatetime() + else: + data[field] = None + + # Convert NaN to None for optional fields + for key, value in data.items(): + if pd.isna(value): + data[key] = None + elif isinstance(value, float) and key != 'comments': # Keep numeric comments + data[key] = str(int(value)) if value.is_integer() else str(value) + elif isinstance(value, str): + data[key] = value.strip() + + # Rename POAM ID field if necessary + if 'POAM ID' in data: + data['poam_id'] = data.pop('POAM ID') + + # Convert keys to snake_case + converted_data = { + convert_to_snake_case(key): value + for key, value in data.items() + } + + return cls(**converted_data) + +class PoamFile: + """Handler for POAM Excel files with specific support for Trivy findings.""" + + def __init__(self, file_path: str): + """ + Initialize a POAM file handler. + + Args: + file_path: Path to the XLSX file + """ + self.file_path = Path(file_path) + if not self.file_path.exists(): + raise FileNotFoundError(f"POAM file not found: {file_path}") + + # Load the Excel file + self.workbook = pd.ExcelFile(self.file_path) + + # Validate required sheet exists + if "Open POA&M Items" not in self.workbook.sheet_names: + raise ValueError('Excel file must contain "Open POA&M Items" sheet') + + # Load the open POAMs data with headers in row 5 (0-based index is 4) + self.df = pd.read_excel( + self.workbook, + sheet_name="Open POA&M Items", + header=4, # 0-based index for row 5 + engine='openpyxl' + ) + + # Load closed POAMs if available + self.closed_df = None + if "Closed POA&M Items" in self.workbook.sheet_names: + self.closed_df = pd.read_excel( + self.workbook, + sheet_name="Closed POA&M Items", + header=4, # 0-based index for row 5 + engine='openpyxl' + ) + + def get_trivy_poams(self) -> pd.DataFrame: + """ + Filter and return Trivy POAMs. + + Returns: + DataFrame containing only Trivy POAMs + """ + # Pattern matches YYYY-TRIVYXXXX where XXXX is 4 or more digits + trivy_pattern = r'^\d{4}-TRIVY\d{4,}$' + + # Filter for POAM IDs matching the Trivy pattern + return self.df[self.df['POAM ID'].str.match(trivy_pattern, na=False)] + + def get_closed_trivy_poams(self) -> pd.DataFrame: + """ + Filter and return closed Trivy POAMs. + + Returns: + DataFrame containing only closed Trivy POAMs, or empty DataFrame if no closed POAMs exist + """ + if self.closed_df is None: + return pd.DataFrame() + + # Pattern matches YYYY-TRIVYXXXX where XXXX is 4 or more digits + trivy_pattern = r'^\d{4}-TRIVY\d{4,}$' + + # Filter for POAM IDs matching the Trivy pattern + return self.closed_df[self.closed_df['POAM ID'].str.match(trivy_pattern, na=False)] + + def get_trivy_poam_entries(self, limit: Optional[int] = None) -> tuple[list[PoamEntry], list[PoamEntry]]: + """ + Get Trivy POAMs as PoamEntry objects. + + Args: + limit: Optional number of entries to return + + Returns: + Tuple of (open_poams, closed_poams) where each is a list of PoamEntry objects + """ + # Get open POAMs + open_df = self.get_trivy_poams() + if limit: + open_df = open_df.head(limit) + open_poams = [PoamEntry.from_dict(row) for _, row in open_df.iterrows()] + + # Get closed POAMs + closed_df = self.get_closed_trivy_poams() + if limit: + closed_df = closed_df.head(limit) + closed_poams = [PoamEntry.from_dict(row) for _, row in closed_df.iterrows()] + + return open_poams, closed_poams + + def preview_trivy_poams(self, limit: int = 5) -> str: + """ + Get a YAML preview of the first N Trivy POAMs. + + Args: + limit: Number of POAMs to preview (default 5) + + Returns: + YAML formatted string of the POAMs + """ + open_entries, closed_entries = self.get_trivy_poam_entries(limit) + + # Convert to dict format expected in output + preview_data = [] + for entry in open_entries: + # Convert datetime objects to strings + entry_dict = {} + for field, value in entry.__dict__.items(): + if isinstance(value, datetime): + value = value.strftime('%Y-%m-%d') + # Convert snake_case back to Title Case for keys + key = ' '.join(word.capitalize() for word in field.split('_')) + entry_dict[key] = value + preview_data.append(entry_dict) + + # Convert to YAML with proper formatting + return yaml.dump(preview_data, sort_keys=False, allow_unicode=True) \ No newline at end of file diff --git a/tools/trivy/alerts.py b/tools/trivy/alerts.py new file mode 100644 index 0000000..b1555a9 --- /dev/null +++ b/tools/trivy/alerts.py @@ -0,0 +1,156 @@ +""" +Tool for converting Trivy alerts to POAM format. +""" +import csv +import json +from datetime import datetime, timedelta +from pathlib import Path +import jq + +from ..utils import ensure_working_dir + +# JQ query to transform GitHub alerts into POAM format +ALERTS_TO_POAM_QUERY = """ +.[] | { + "_state": .state, + "Alert ID": .number, + "Controls": "RA-5", + "Weakness Name": (.rule.description + "; " + .rule.id), + "Weakness Description": .rule.full_description, + "Weakness Detector Source": .html_url, + "Weakness Source Identifier": (.tool.name + " " + .tool.version), + "Asset Identifier": .rule.most_recent_instance.location.path, + "Point of Contact": "Chris Llanwarne", + "Resources Required": "None", + "Overall Remediation Plan": "Perform necessary updates to resolve the vulnerability", + "Original Detection Date": .created_at, + "Status Date": .updated_at, + "Last Vendor Check-in Date": .rule.updated_at, + "Scheduled Completion Date": "DATE", + "AGENCY Scheduled Completion Date": "DATE", + "Planned Milestones": "DATE: Perform necessary updates to resolve the vulnerability", + "Milestone Changes": "", + "Vendor Dependency": "Yes", + "Vendor Dependent Product Name": "Ubuntu", + "Original Risk Rating": (.rule.security_severity_level // (.rule.tags | map(select(test("^(critical|high|medium|low)$"; "i"))) | first | ascii_downcase)), + "Adjusted Risk Rating": "", + "Risk Adjustment": "", + "False Positive": "No", + "Operational Requirement": "No", + "Deviation Rationale": "", + "Supporting Documents": "", + "Comments": .most_recent_instance.message.text, + "Auto-Approve": "No", + "Binding Operational Directive 22-01 tracking": "", + "Binding Operational Directive 22-01 Due Date": "", + "CVE": .rule.id, + "Service Name": "Hail Batch" +}""" + +# POAM CSV field names in order +FIELDNAMES = [ + "Alert ID", "Controls", "Weakness Name", "Weakness Description", + "Weakness Detector Source", "Weakness Source Identifier", "Asset Identifier", + "Point of Contact", "Resources Required", "Overall Remediation Plan", + "Original Detection Date", "Scheduled Completion Date", + "AGENCY Scheduled Completion Date", "Planned Milestones", "Milestone Changes", + "Status Date", "Vendor Dependency", "Last Vendor Check-in Date", + "Vendor Dependent Product Name", "Original Risk Rating", "Adjusted Risk Rating", + "Risk Adjustment", "False Positive", "Operational Requirement", + "Deviation Rationale", "Supporting Documents", "Comments", "Auto-Approve", + "Binding Operational Directive 22-01 tracking", + "Binding Operational Directive 22-01 Due Date", "CVE", "Service Name" +] + +def date_plus(iso_date_string: str, days_to_add: int) -> str: + """ + Parses an ISO date string, adds days, and formats it to MM/DD/YY. + + Args: + iso_date_string: The ISO date string to parse (e.g., "2023-10-26T12:00:00Z") + days_to_add: The number of days to add (can be positive or negative) + + Returns: + The formatted date string (MM/DD/YY), or None if parsing fails + """ + try: + date_object = datetime.fromisoformat(iso_date_string.replace("Z", "+00:00")) + modified_date = date_object + timedelta(days=days_to_add) + return modified_date.strftime("%m/%d/%y") + except ValueError as e: + raise ValueError(f"Invalid ISO date string format: {iso_date_string}") from e + +def convert_alerts_to_poam(alerts_file: Path, output_path: Path = None) -> Path: + """ + Convert GitHub Trivy alerts JSON to POAM CSV format. + + Args: + alerts_file: Path to the JSON file containing GitHub alerts + output_path: Optional path for the output file. If None, uses same parent directory as input file. + + Returns: + Path to the generated CSV file + """ + # Load alerts data + alerts_data = json.loads(alerts_file.read_text()) + + # Compile and run JQ query + alerts_jq = jq.compile(ALERTS_TO_POAM_QUERY) + jq_results = alerts_jq.input_value(alerts_data) + rows: list[dict] = [] + + # Process each alert + for row in jq_results.all(): + # Skip non-Trivy and closed alerts + if row["Weakness Source Identifier"][:5] != "Trivy" or row.pop("_state") != "open": + continue + + # Parse message for asset information + message = { + kv[0]: (kv[1] if len(kv) > 1 else "") + for kv in [line.split(": ") for line in row["Comments"].split("\n")] + } + + if "Image" not in message or "Package" not in message: + continue + + # Update asset identifier + row["Asset Identifier"] = f"{message['Image']} ({message['Package']})" + + # Handle dates and intervals + orig_date = row["Original Detection Date"] + status_date = row["Status Date"] + if not row["Original Risk Rating"]: + print(f"Excluded alert {row['Alert ID']} ({row['CVE']}): no severity rating") + continue + sev = row["Original Risk Rating"].lower() + + # Calculate fix date based on severity + fix_intervals = {"high": 14, "medium": 90, "low": 180} + fix_interval = fix_intervals.get(sev, 0) + fix_date = date_plus(orig_date, fix_interval) + + # Update all dates + row["Original Detection Date"] = date_plus(orig_date, 0) + row["Status Date"] = date_plus(status_date, 0) + row["Last Vendor Check-in Date"] = date_plus(status_date, 0) + row["Scheduled Completion Date"] = fix_date + row["AGENCY Scheduled Completion Date"] = fix_date + row["Planned Milestones"] = row["Planned Milestones"].replace("DATE", fix_date) + + rows.append(row) + + # Determine output file path + if output_path is None: + # Use same parent directory as input file, change extension to .findings.csv + output_file = alerts_file.parent / f"{alerts_file.stem}.findings.csv" + else: + output_file = output_path + + # Write CSV file + with output_file.open('w', newline='') as csvfile: + writer = csv.DictWriter(csvfile, fieldnames=FIELDNAMES) + writer.writeheader() + writer.writerows(rows) + + return output_file \ No newline at end of file diff --git a/tools/trivy/diff.py b/tools/trivy/diff.py new file mode 100644 index 0000000..c396a7a --- /dev/null +++ b/tools/trivy/diff.py @@ -0,0 +1,280 @@ +""" +Module for comparing Trivy findings against existing POAMs. +""" +from dataclasses import dataclass +from typing import List, Optional, Tuple, Dict, Any +from pathlib import Path +from datetime import datetime + +from ..findings import Finding +from ..poam import PoamFile, PoamEntry +from ..diff import PoamFileDiff, compare_findings_to_poams +from .poam_generator import generate_poams_from_findings + +@dataclass +class FindingPoamMatch: + """Represents a match between a finding and an existing POAM.""" + finding: Finding + poam: PoamEntry + +@dataclass +class TrivyAlertsDiff: + """Represents the difference between current findings and existing POAMs.""" + new_findings: List[Finding] # Findings without corresponding POAMs + existing_matches: List[FindingPoamMatch] # Findings matched to existing POAMs + closed_poams: List[PoamEntry] # POAMs without corresponding findings + reopened_findings: List[FindingPoamMatch] # Findings that match previously closed POAMs + proposed_poams: List[Tuple[List[Finding], PoamEntry]] # Proposed new POAMs with their findings + + def to_json(self) -> Dict[str, Any]: + """ + Convert the diff results to a JSON-serializable dictionary. + + Returns: + Dictionary containing the diff results in a structured format + """ + def format_datetime(dt: datetime) -> str: + return dt.strftime("%Y-%m-%d") if dt else None + + def poam_to_full_dict(poam: PoamEntry) -> Dict[str, Any]: + """Convert a POAM to a complete dictionary with all fields.""" + return { + "poam_id": poam.poam_id, + "controls": poam.controls, + "weakness_name": poam.weakness_name, + "weakness_description": poam.weakness_description, + "weakness_detector_source": poam.weakness_detector_source, + "weakness_source_identifier": poam.weakness_source_identifier, + "asset_identifier": poam.asset_identifier, + "point_of_contact": poam.point_of_contact, + "resources_required": poam.resources_required, + "overall_remediation_plan": poam.overall_remediation_plan, + "original_detection_date": format_datetime(poam.original_detection_date), + "scheduled_completion_date": format_datetime(poam.scheduled_completion_date), + "planned_milestones": poam.planned_milestones, + "milestone_changes": poam.milestone_changes, + "status_date": format_datetime(poam.status_date), + "vendor_dependency": poam.vendor_dependency, + "last_vendor_check_in_date": format_datetime(poam.last_vendor_check_in_date), + "vendor_dependent_product_name": poam.vendor_dependent_product_name, + "original_risk_rating": poam.original_risk_rating, + "adjusted_risk_rating": poam.adjusted_risk_rating, + "risk_adjustment": poam.risk_adjustment, + "false_positive": poam.false_positive, + "operational_requirement": poam.operational_requirement, + "deviation_rationale": poam.deviation_rationale, + "supporting_documents": poam.supporting_documents, + "comments": poam.comments, + "auto_approve": poam.auto_approve, + "binding_operational_directive_22_01_tracking": poam.binding_operational_directive_22_01_tracking, + "binding_operational_directive_22_01_due_date": format_datetime(poam.binding_operational_directive_22_01_due_date), + "cve": poam.cve, + "service_name": poam.service_name + } + + def finding_to_dict(finding: Finding) -> Dict[str, Any]: + return { + "finding_id": finding.finding_id, + "weakness_name": finding.weakness_name, + "asset_identifier": finding.asset_identifier, + "original_detection_date": format_datetime(finding.original_detection_date), + "original_risk_rating": finding.original_risk_rating, + "cve": finding.cve, + "service_name": finding.service_name + } + + return { + "metadata": { + "new_findings_count": len(self.new_findings), + "existing_matches_count": len(self.existing_matches), + "closed_poams_count": len(self.closed_poams), + "reopened_findings_count": len(self.reopened_findings), + "proposed_poams_count": len(self.proposed_poams) + }, + "new_poams": [ + { + "poam": poam_to_full_dict(poam), + "findings": [finding_to_dict(f) for f in findings], + "finding_ids": [f.finding_id for f in findings] + } + for findings, poam in self.proposed_poams + ], + "reopen_poams": [ + { + "poam_id": match.poam.poam_id, + "finding_id": match.finding.finding_id + } + for match in self.reopened_findings + ], + "close_poams": [poam.poam_id for poam in self.closed_poams] + } + + def print_summary(self, max_preview: int = 10) -> None: + """Print a human-readable summary of the diff.""" + # Print new findings + print("\n=== New Findings ===") + print(f"Count: {len(self.new_findings)}") + if self.new_findings: + finding_ids = [finding.finding_id for finding in self.new_findings] + print(f"Finding IDs: {', '.join(finding_ids)}") + + # Print existing matches + print("\n=== Existing Matches ===") + print(f"Count: {len(self.existing_matches)}") + if self.existing_matches: + matches = [f"{match.finding.finding_id} -> {match.poam.poam_id}" + for match in self.existing_matches[:max_preview]] + print(f"Preview of matches: {', '.join(matches)}") + if len(self.existing_matches) > max_preview: + print(f"... and {len(self.existing_matches) - max_preview} more") + + # Print reopened findings + print("\n=== Reopened Findings ===") + print(f"Count: {len(self.reopened_findings)}") + if self.reopened_findings: + matches = [f"{match.finding.finding_id} -> {match.poam.poam_id}" + for match in self.reopened_findings[:max_preview]] + print(f"Preview of matches: {', '.join(matches)}") + if len(self.reopened_findings) > max_preview: + print(f"... and {len(self.reopened_findings) - max_preview} more") + + # Print closed POAMs + print("\n=== Closed POAMs ===") + print(f"Count: {len(self.closed_poams)}") + if self.closed_poams: + poam_ids = [poam.poam_id for poam in self.closed_poams] + print(f"POAM IDs no longer active: {', '.join(poam_ids)}") + + # Print proposed POAMs + print("\n=== Proposed POAMs ===") + print(f"Count: {len(self.proposed_poams)}") + if self.proposed_poams: + for findings, poam in self.proposed_poams[:max_preview]: + finding_ids = [f.finding_id for f in findings] + print(f"{', '.join(finding_ids)} => {poam.poam_id}") + if len(self.proposed_poams) > max_preview: + print(f"... and {len(self.proposed_poams) - max_preview} more") + + # Show sample of first proposed POAM + print("\nSample new POAM:") + sample_findings, sample_poam = self.proposed_poams[0] + print(f"POAM ID: {sample_poam.poam_id}") + print(f"Weakness Name: {sample_poam.weakness_name}") + print(f"Asset Identifiers: {sample_poam.asset_identifier}") + print(f"Finding IDs: {sample_poam.comments}") + print(f"Detection Date: {sample_poam.original_detection_date.strftime('%Y-%m-%d')}") + print(f"Risk Rating: {sample_poam.original_risk_rating}") + if sample_poam.cve: + print(f"CVE: {sample_poam.cve}") + +def _is_exact_match(text1: str, text2: str) -> bool: + """ + Check if two strings match exactly (case-insensitive). + + Args: + text1: First text string + text2: Second text string + + Returns: + True if strings match exactly (ignoring case), False otherwise + """ + if not text1 or not text2: + return False + return text1.lower().strip() == text2.lower().strip() + +def _is_asset_covered(finding_asset: str, poam_assets: str) -> bool: + """ + Check if the finding's asset is included in the POAM's asset list. + + Args: + finding_asset: Asset identifier from the finding + poam_assets: Asset identifier field from the POAM (may contain multiple assets) + + Returns: + True if the finding's asset is contained within the POAM's asset list + """ + if not finding_asset or not poam_assets: + return False + return finding_asset.lower().strip() in poam_assets.lower().strip() + +def _find_matching_poam(finding: Finding, poams: List[PoamEntry]) -> Optional[FindingPoamMatch]: + """Find a matching POAM for a given finding based on exact weakness name match and asset coverage.""" + for poam in poams: + # Weakness name must match exactly, and the finding's asset must be included in the POAM's assets + if (_is_exact_match(finding.weakness_name, poam.weakness_name) and + _is_asset_covered(finding.asset_identifier, poam.asset_identifier)): + return FindingPoamMatch(finding=finding, poam=poam) + + return None + + +def compare_findings_to_trivy_poams(findings: List[Finding], poam_file: Path) -> PoamFileDiff: + """ + Compare a list of findings against Trivy POAMs. + + Args: + findings: List of current findings from Trivy + poam_file: Path to Excel file containing Trivy POAMs + + Returns: + PoamFileDiff containing new, existing, closed, and reopened findings + """ + # Load Trivy POAMs + poam_file_handler = PoamFile(poam_file) + open_poams, closed_poams = poam_file_handler.get_trivy_poam_entries() + + # Get all POAM IDs for generating new ones + all_poam_ids = [p.poam_id for p in [*open_poams, *closed_poams]] + + return compare_findings_to_poams(findings, open_poams, closed_poams, all_poam_ids, generate_poams_from_findings, store_as_configuration_findings=False) + + +# def compare_findings_to_poams(findings: List[Finding], +# open_poams: List[PoamEntry], +# closed_poams: List[PoamEntry], +# existing_poam_ids: List[str]) -> TrivyAlertsDiff: +# """ +# Compare a list of findings against existing POAMs. + +# Args: +# findings: List of current findings from Trivy +# open_poams: List of open POAMs +# closed_poams: List of closed POAMs +# existing_poam_ids: List of all existing POAM IDs + +# Returns: +# TrivyAlertsDiff containing new, existing, closed, and reopened findings +# """ +# # Track which POAMs are matched +# matched_poams = set() +# new_findings = [] +# existing_matches = [] +# reopened_findings = [] + +# # First check for matches against open POAMs +# for finding in findings: +# match = _find_matching_poam(finding, open_poams) +# if match: +# existing_matches.append(match) +# matched_poams.add(match.poam) +# else: +# # If no match in open POAMs, check closed POAMs +# closed_match = _find_matching_poam(finding, closed_poams) +# if closed_match: +# reopened_findings.append(closed_match) +# else: +# new_findings.append(finding) + +# # Find closed POAMs (those without matches) +# closed_poams = [poam for poam in open_poams if poam not in matched_poams] + +# # Generate proposed POAMs for new findings +# proposed_poams = generate_poams_from_findings(new_findings, existing_poam_ids) + +# return TrivyAlertsDiff( +# new_findings=new_findings, +# existing_matches=existing_matches, +# closed_poams=closed_poams, +# reopened_findings=reopened_findings, +# proposed_poams=proposed_poams +# ) \ No newline at end of file diff --git a/tools/trivy/importer.py b/tools/trivy/importer.py new file mode 100644 index 0000000..0027f44 --- /dev/null +++ b/tools/trivy/importer.py @@ -0,0 +1,71 @@ +""" +Tool for importing Trivy alerts from CSV format into Finding objects. +""" +import csv +from datetime import datetime +from pathlib import Path +from typing import List + +from ..findings import Finding + +def parse_date(date_str: str) -> datetime: + """Parse a date string in MM/DD/YY format into a datetime object.""" + try: + return datetime.strptime(date_str, "%m/%d/%y") + except ValueError as e: + raise ValueError(f"Invalid date format (expected MM/DD/YY): {date_str}") from e + +def import_alerts_from_csv(csv_file: Path) -> List[Finding]: + """ + Import Trivy alerts from a CSV file and convert them to Finding objects. + + Args: + csv_file: Path to the CSV file containing Trivy alerts + + Returns: + List of Finding objects + """ + entries = [] + + with csv_file.open('r', newline='') as f: + reader = csv.DictReader(f) + + for row in reader: + # Convert CSV field names to Finding field names + entry_data = { + 'finding_id': row['Alert ID'], + 'controls': row['Controls'], + 'weakness_name': row['Weakness Name'], + 'weakness_description': row['Weakness Description'], + 'weakness_detector_source': row['Weakness Detector Source'], + 'weakness_source_identifier': row['Weakness Source Identifier'], + 'asset_identifier': row['Asset Identifier'], + 'point_of_contact': row['Point of Contact'], + 'resources_required': row['Resources Required'] or None, + 'overall_remediation_plan': row['Overall Remediation Plan'], + 'original_detection_date': parse_date(row['Original Detection Date']), + 'scheduled_completion_date': parse_date(row['Scheduled Completion Date']), + 'planned_milestones': row['Planned Milestones'], + 'milestone_changes': row['Milestone Changes'], + 'status_date': parse_date(row['Status Date']), + 'vendor_dependency': row['Vendor Dependency'], + 'last_vendor_check_in_date': parse_date(row['Last Vendor Check-in Date']) if row['Last Vendor Check-in Date'] else None, + 'vendor_dependent_product_name': row['Vendor Dependent Product Name'], + 'original_risk_rating': row['Original Risk Rating'], + 'adjusted_risk_rating': row['Adjusted Risk Rating'] or None, + 'risk_adjustment': row['Risk Adjustment'], + 'false_positive': row['False Positive'], + 'operational_requirement': row['Operational Requirement'], + 'deviation_rationale': row['Deviation Rationale'] or None, + 'supporting_documents': row['Supporting Documents'] or None, + 'comments': row['Comments'] or None, + 'auto_approve': row['Auto-Approve'], + 'binding_operational_directive_22_01_tracking': row['Binding Operational Directive 22-01 tracking'], + 'binding_operational_directive_22_01_due_date': parse_date(row['Binding Operational Directive 22-01 Due Date']) if row['Binding Operational Directive 22-01 Due Date'] else None, + 'cve': row['CVE'] or None, + 'service_name': row['Service Name'] + } + + entries.append(Finding(**entry_data)) + + return entries \ No newline at end of file diff --git a/tools/trivy/poam_generator.py b/tools/trivy/poam_generator.py new file mode 100644 index 0000000..ecb1bdc --- /dev/null +++ b/tools/trivy/poam_generator.py @@ -0,0 +1,172 @@ +""" +Module for generating POAMs from findings. +""" +from datetime import datetime +from typing import List, Dict, Tuple +import re +from ..findings import Finding +from ..poam import PoamEntry + +def parse_trivy_id(poam_id: str) -> Tuple[int, int]: + """ + Parse a Trivy POAM ID into year and sequence components. + + Args: + poam_id: POAM ID in format YYYY-TRIVYXXXX + + Returns: + Tuple of (year, sequence_number) + + Raises: + ValueError: If the ID format is invalid + """ + match = re.match(r'^(\d{4})-TRIVY(\d{4,})$', poam_id) + if not match: + raise ValueError(f"Invalid Trivy POAM ID format: {poam_id}") + + year = int(match.group(1)) + sequence = int(match.group(2)) + return year, sequence + +def get_next_trivy_id(existing_poam_ids: List[str], current_year: int = None) -> str: + """ + Generate the next available Trivy POAM ID. + + Args: + existing_poam_ids: List of existing POAM IDs + current_year: Optional year to use (defaults to current year) + + Returns: + Next available POAM ID in YYYY-TRIVYXXXX format + """ + if current_year is None: + current_year = datetime.now().year + + # Find the highest sequence number for the current year + max_sequence = 0 + for poam_id in existing_poam_ids: + try: + year, sequence = parse_trivy_id(poam_id) + if year == current_year and sequence > max_sequence: + max_sequence = sequence + except ValueError: + continue # Skip invalid IDs + + return f"{current_year}-TRIVY{max_sequence + 1:04d}" + +def findings_to_poam(findings: List[Finding], poam_id: str) -> PoamEntry: + """ + Convert a list of findings with the same weakness into a single POAM. + + Args: + findings: List of findings with the same weakness + poam_id: POAM ID to use for the new POAM + + Returns: + PoamEntry combining all findings + + Raises: + ValueError: If findings have different weakness names + """ + if not findings: + raise ValueError("Cannot create POAM from empty findings list") + + # Verify all findings have the same weakness name + weakness_name = findings[0].weakness_name + if not all(f.weakness_name == weakness_name for f in findings): + raise ValueError("All findings must have the same weakness name") + + # Combine asset identifiers and finding IDs + asset_identifiers = [f.asset_identifier for f in findings] + finding_ids = [f.finding_id for f in findings] + + # Use the first finding as a template + first = findings[0] + return PoamEntry( + poam_id=poam_id, + controls=first.controls, + weakness_name=first.weakness_name, + weakness_description=first.weakness_description, + weakness_detector_source=first.weakness_detector_source, + weakness_source_identifier=first.weakness_source_identifier, + asset_identifier=", ".join(asset_identifiers), + point_of_contact=first.point_of_contact, + resources_required=first.resources_required, + overall_remediation_plan=first.overall_remediation_plan, + original_detection_date=first.original_detection_date, + scheduled_completion_date=first.scheduled_completion_date, + planned_milestones=first.planned_milestones, + milestone_changes=first.milestone_changes, + status_date=first.status_date, + vendor_dependency=first.vendor_dependency, + last_vendor_check_in_date=first.last_vendor_check_in_date, + vendor_dependent_product_name=first.vendor_dependent_product_name, + original_risk_rating=first.original_risk_rating, + adjusted_risk_rating=first.adjusted_risk_rating, + risk_adjustment=first.risk_adjustment, + false_positive=first.false_positive, + operational_requirement=first.operational_requirement, + deviation_rationale=first.deviation_rationale, + supporting_documents=first.supporting_documents, + comments=", ".join(finding_ids), # Store finding IDs in comments + auto_approve=first.auto_approve, + binding_operational_directive_22_01_tracking=first.binding_operational_directive_22_01_tracking, + binding_operational_directive_22_01_due_date=first.binding_operational_directive_22_01_due_date, + cve=first.cve, + service_name=first.service_name + ) + +def group_findings_by_weakness(findings: List[Finding]) -> Dict[str, List[Finding]]: + """ + Group findings by weakness name. + + Args: + findings: List of findings to group + + Returns: + Dictionary mapping weakness names to lists of findings, with findings sorted by ID + """ + groups: Dict[str, List[Finding]] = {} + for finding in findings: + groups.setdefault(finding.weakness_name, []).append(finding) + + # Sort each group by finding ID + for findings_list in groups.values(): + findings_list.sort(key=lambda f: f.finding_id) + + return groups + +def generate_poams_from_findings(findings: List[Finding], existing_poam_ids: List[str], current_year: int = None) -> List[Tuple[List[Finding], PoamEntry]]: + """ + Generate new POAMs from a list of findings. + + Args: + findings: List of findings to convert to POAMs + existing_poam_ids: List of existing POAM IDs + current_year: Optional year to use (defaults to current year) + + Returns: + List of tuples containing (findings_list, generated_poam), sorted by first finding ID + """ + # Group findings by weakness + grouped_findings = group_findings_by_weakness(findings) + + # Sort groups by the first finding ID in each group + sorted_groups = sorted( + grouped_findings.values(), + key=lambda findings_list: findings_list[0].finding_id if findings_list else "" + ) + + # Generate POAMs for each group + result = [] + current_year = datetime.now().year if current_year is None else current_year + next_id = get_next_trivy_id(existing_poam_ids, current_year) + + for findings_list in sorted_groups: + poam = findings_to_poam(findings_list, next_id) + result.append((findings_list, poam)) + + # Get next ID for the next group + next_id = get_next_trivy_id([*existing_poam_ids, next_id], current_year) + + return result \ No newline at end of file diff --git a/tools/utils.py b/tools/utils.py new file mode 100644 index 0000000..b2783ef --- /dev/null +++ b/tools/utils.py @@ -0,0 +1,26 @@ +""" +Common utility functions used across tools. +""" +import os +from pathlib import Path + +def ensure_working_dir() -> Path: + """ + Ensure the working directory exists and return its path. + The working directory is used for temporary files and downloads. + + Checks for WORKING environment variable first, then falls back to pwd/working. + + Returns: + Path object for the working directory + """ + # Check WORKING environment variable first + working_env = os.getenv('WORKING') + if working_env: + working_dir = Path(working_env) + else: + # Fall back to pwd/working + working_dir = Path(os.getcwd()) / 'working' + + working_dir.mkdir(parents=True, exist_ok=True) + return working_dir \ No newline at end of file diff --git a/tools/zap/__init__.py b/tools/zap/__init__.py new file mode 100644 index 0000000..f1121f6 --- /dev/null +++ b/tools/zap/__init__.py @@ -0,0 +1,7 @@ +""" +Package for handling ZAP scan reports and findings. +""" + +from .alerts import convert_alerts_to_findings + +__all__ = ['convert_alerts_to_findings'] \ No newline at end of file diff --git a/tools/zap/alerts.py b/tools/zap/alerts.py new file mode 100644 index 0000000..dc82553 --- /dev/null +++ b/tools/zap/alerts.py @@ -0,0 +1,124 @@ +""" +Module for handling ZAP scan reports and converting alerts to findings. +""" +import csv +from datetime import datetime, timedelta, timezone +from typing import List +import json +from pathlib import Path +from ..findings import Finding + +def get_completion_date(severity: str, detection_date: datetime) -> datetime: + """Calculate completion date based on severity.""" + days_map = { + 'Critical': 15, + 'High': 30, + 'Medium': 90, # Using 90 days for medium + 'Low': 180, + 'Informational': 180 + } + days = days_map.get(severity, 180) # Default to 180 days if unknown severity + return detection_date + timedelta(days=days) + +def parse_zap_csv(csv_file: str) -> List[Finding]: + """ + Parse a ZAP CSV report and extract alert findings. + + Args: + csv_file: Path to the ZAP CSV report file + + Returns: + List of Finding objects + """ + findings = [] + + with open(csv_file, 'r') as f: + reader = csv.DictReader(f) + for row in reader: + # Parse dates + detection_date = datetime.strptime(row['Original Detection Date'], '%m/%d/%Y') + detection_date = detection_date.replace(tzinfo=timezone.utc) + + # Parse completion date if available + completion_date = None + if row['Scheduled Completion Date']: + try: + completion_date = datetime.strptime(row['Scheduled Completion Date'], '%Y-%m-%d %H:%M:%S') + completion_date = completion_date.replace(tzinfo=timezone.utc) + except ValueError: + # If parsing fails, calculate based on risk rating + completion_date = get_completion_date(row['Original Risk Rating'], detection_date) + else: + completion_date = get_completion_date(row['Original Risk Rating'], detection_date) + + # Create finding + finding = Finding( + finding_id=f"{row['ids']}", + controls="RA-5", + weakness_name=row['Weakness Name'], + weakness_description=row['Weakness Description'], + weakness_detector_source=row['Weakness Detector Source'], + weakness_source_identifier=row['Weakness Source Identifier'], + asset_identifier=row['Asset Identifier'], + point_of_contact="Chris Llanwarne", + resources_required="None", + overall_remediation_plan="Perform necessary updates to resolve the vulnerability", + original_detection_date=detection_date, + scheduled_completion_date=completion_date, + planned_milestones=f"(1) {completion_date.strftime('%Y-%m-%d')}: Perform necessary updates to resolve the vulnerability", + milestone_changes="", + status_date=datetime.now(timezone.utc), + vendor_dependency="No", + last_vendor_check_in_date=None, + vendor_dependent_product_name="N/A", + original_risk_rating=row['Original Risk Rating'], + adjusted_risk_rating=None, + risk_adjustment="", + false_positive="", + operational_requirement="", + deviation_rationale=None, + supporting_documents=None, + comments=None, + auto_approve="", + binding_operational_directive_22_01_tracking="", + binding_operational_directive_22_01_due_date=None, + cve=None, + service_name="Hail" + ) + findings.append(finding) + + return findings + +def convert_alerts_to_findings(csv_file: str, output_file: str = None) -> str: + """ + Convert ZAP CSV alerts to findings JSON format. + + Args: + csv_file: Path to the ZAP CSV report file + output_file: Optional output file path. If None, uses input file with .findings.json extension + + Returns: + Path to the output JSON file + """ + findings = parse_zap_csv(csv_file) + + # Convert findings to dictionaries + findings_data = [] + for finding in findings: + finding_dict = vars(finding) + # Convert datetime objects to strings + for key, value in finding_dict.items(): + if isinstance(value, datetime): + finding_dict[key] = value.strftime('%Y-%m-%d %H:%M:%S') + findings_data.append(finding_dict) + + # Determine output filename + if output_file is None: + input_path = Path(csv_file) + output_file = str(input_path.with_suffix('.findings.json')) + + # Write findings to JSON file + with open(output_file, 'w') as f: + json.dump(findings_data, f, indent=2) + + return output_file \ No newline at end of file diff --git a/tools/zap/diff.py b/tools/zap/diff.py new file mode 100644 index 0000000..946c2e8 --- /dev/null +++ b/tools/zap/diff.py @@ -0,0 +1,251 @@ +""" +Module for comparing ZAP findings against existing POAMs. +""" +from dataclasses import dataclass +from typing import List, Optional, Tuple, Dict, Any, Union +from pathlib import Path +from datetime import datetime + +from ..findings import Finding +from ..poam import PoamFile, PoamEntry +from ..diff import PoamFileDiff, compare_findings_to_poams +from .poam_generator import generate_poams_from_findings + +@dataclass +class FindingPoamMatch: + """Represents a match between a finding and an existing POAM.""" + finding: Finding + poam: PoamEntry + +@dataclass +class ZapAlertsDiff: + """Represents the difference between current findings and existing POAMs.""" + new_findings: List[Finding] # Findings without corresponding POAMs + existing_matches: List[FindingPoamMatch] # Findings matched to existing POAMs + closed_poams: List[PoamEntry] # POAMs without corresponding findings + reopened_findings: List[FindingPoamMatch] # Findings that match previously closed POAMs + proposed_poams: List[Tuple[List[Finding], PoamEntry]] # Proposed new POAMs with their findings + + def to_json(self) -> Dict[str, Any]: + """ + Convert the diff results to a JSON-serializable dictionary. + + Returns: + Dictionary containing the diff results in a structured format + """ + def format_datetime(dt: Union[datetime, str]) -> str: + if isinstance(dt, str): + return dt + else: + return dt.strftime("%Y-%m-%d") if dt else None + + def poam_to_full_dict(poam: PoamEntry) -> Dict[str, Any]: + """Convert a POAM to a complete dictionary with all fields.""" + return { + "poam_id": poam.poam_id, + "controls": poam.controls, + "weakness_name": poam.weakness_name, + "weakness_description": poam.weakness_description, + "weakness_detector_source": poam.weakness_detector_source, + "weakness_source_identifier": poam.weakness_source_identifier, + "asset_identifier": poam.asset_identifier, + "point_of_contact": poam.point_of_contact, + "resources_required": poam.resources_required, + "overall_remediation_plan": poam.overall_remediation_plan, + "original_detection_date": format_datetime(poam.original_detection_date), + "scheduled_completion_date": format_datetime(poam.scheduled_completion_date), + "planned_milestones": poam.planned_milestones, + "milestone_changes": poam.milestone_changes, + "status_date": format_datetime(poam.status_date), + "vendor_dependency": poam.vendor_dependency, + "last_vendor_check_in_date": format_datetime(poam.last_vendor_check_in_date), + "vendor_dependent_product_name": poam.vendor_dependent_product_name, + "original_risk_rating": poam.original_risk_rating, + "adjusted_risk_rating": poam.adjusted_risk_rating, + "risk_adjustment": poam.risk_adjustment, + "false_positive": poam.false_positive, + "operational_requirement": poam.operational_requirement, + "deviation_rationale": poam.deviation_rationale, + "supporting_documents": poam.supporting_documents, + "comments": poam.comments, + "auto_approve": poam.auto_approve, + "binding_operational_directive_22_01_tracking": poam.binding_operational_directive_22_01_tracking, + "binding_operational_directive_22_01_due_date": format_datetime(poam.binding_operational_directive_22_01_due_date), + "cve": poam.cve, + "service_name": poam.service_name + } + + def finding_to_dict(finding: Finding) -> Dict[str, Any]: + return { + "finding_id": finding.finding_id, + "weakness_name": finding.weakness_name, + "asset_identifier": finding.asset_identifier, + "original_detection_date": format_datetime(finding.original_detection_date), + "original_risk_rating": finding.original_risk_rating, + "cve": finding.cve, + "service_name": finding.service_name + } + + return { + "metadata": { + "new_findings_count": len(self.new_findings), + "existing_matches_count": len(self.existing_matches), + "closed_poams_count": len(self.closed_poams), + "reopened_findings_count": len(self.reopened_findings), + "proposed_poams_count": len(self.proposed_poams) + }, + "new_poams": [ + { + "poam": poam_to_full_dict(poam), + "findings": [finding_to_dict(f) for f in findings], + "finding_ids": [f.finding_id for f in findings] + } + for findings, poam in self.proposed_poams + ], + "reopen_poams": [ + { + "poam_id": match.poam.poam_id, + "finding_id": match.finding.finding_id + } + for match in self.reopened_findings + ], + "close_poams": [poam.poam_id for poam in self.closed_poams] + } + + def print_summary(self, max_preview: int = 10) -> None: + """Print a human-readable summary of the diff.""" + # Print new findings + print("\n=== New Findings ===") + print(f"Count: {len(self.new_findings)}") + if self.new_findings: + finding_ids = [finding.finding_id for finding in self.new_findings] + print(f"Finding IDs: {', '.join(finding_ids)}") + + # Print existing matches + print("\n=== Existing Matches ===") + print(f"Count: {len(self.existing_matches)}") + if self.existing_matches: + matches = [f"{match.finding.finding_id} -> {match.poam.poam_id}" + for match in self.existing_matches[:max_preview]] + print(f"Preview of matches: {', '.join(matches)}") + if len(self.existing_matches) > max_preview: + print(f"... and {len(self.existing_matches) - max_preview} more") + + # Print reopened findings + print("\n=== Reopened Findings ===") + print(f"Count: {len(self.reopened_findings)}") + if self.reopened_findings: + matches = [f"{match.finding.finding_id} -> {match.poam.poam_id}" + for match in self.reopened_findings[:max_preview]] + print(f"Preview of matches: {', '.join(matches)}") + if len(self.reopened_findings) > max_preview: + print(f"... and {len(self.reopened_findings) - max_preview} more") + + # Print closed POAMs + print("\n=== Closed POAMs ===") + print(f"Count: {len(self.closed_poams)}") + if self.closed_poams: + poam_ids = [poam.poam_id for poam in self.closed_poams] + print(f"POAM IDs no longer active: {', '.join(poam_ids)}") + + # Print proposed POAMs + print("\n=== Proposed POAMs ===") + print(f"Count: {len(self.proposed_poams)}") + if self.proposed_poams: + for findings, poam in self.proposed_poams[:max_preview]: + finding_ids = [f.finding_id for f in findings] + print(f"{', '.join(finding_ids)} => {poam.poam_id}") + if len(self.proposed_poams) > max_preview: + print(f"... and {len(self.proposed_poams) - max_preview} more") + + # Show sample of first proposed POAM + print("\nSample new POAM:") + sample_findings, sample_poam = self.proposed_poams[0] + print(f"POAM ID: {sample_poam.poam_id}") + print(f"Weakness Name: {sample_poam.weakness_name}") + print(f"Asset Identifiers: {sample_poam.asset_identifier}") + print(f"Finding IDs: {sample_poam.comments}") + # Handle case where date might already be a string + detection_date = sample_poam.original_detection_date + if isinstance(detection_date, datetime): + detection_date = detection_date.strftime('%Y-%m-%d') + print(f"Detection Date: {detection_date}") + print(f"Risk Rating: {sample_poam.original_risk_rating}") + if sample_poam.cve: + print(f"CVE: {sample_poam.cve}") + +def _is_exact_match(str1: str, str2: str) -> bool: + """Check if two strings match exactly, ignoring case.""" + if not str1 or not str2: + return False + return str1.lower().strip() == str2.lower().strip() + +def _is_asset_covered(finding_asset: str, poam_assets: str) -> bool: + """ + Check if the finding's asset is included in the POAM's asset list. + + Args: + finding_asset: Asset identifier from the finding + poam_assets: Asset identifier field from the POAM (may contain multiple assets) + + Returns: + True if the finding's asset is contained within the POAM's asset list + """ + if not finding_asset or not poam_assets: + return False + return finding_asset.lower().strip() in poam_assets.lower().strip() + +def _find_matching_poam(finding: Finding, poams: List[PoamEntry]) -> Optional[FindingPoamMatch]: + """Find a matching POAM for a given finding based on exact weakness name match and asset coverage.""" + for poam in poams: + # Weakness name must match exactly, and the finding's asset must be included in the POAM's assets + if (_is_exact_match(finding.weakness_name, poam.weakness_name) and + _is_asset_covered(finding.asset_identifier, poam.asset_identifier)): + return FindingPoamMatch(finding=finding, poam=poam) + + return None + +def get_zap_poam_entries(poam_file: PoamFile) -> Tuple[List[PoamEntry], List[PoamEntry]]: + """ + Get ZAP POAMs from a POAM file. + + Args: + poam_file: PoamFile instance + + Returns: + Tuple of (open_poams, closed_poams) + """ + # Pattern matches YYYY-ZAPXXXX where XXXX is 4 or more digits + zap_pattern = r'^\d{4}-ZAP\d{4,}$' + + # Get open POAMs + open_df = poam_file.df[poam_file.df['POAM ID'].str.match(zap_pattern, na=False)] + open_poams = [PoamEntry.from_dict(row) for _, row in open_df.iterrows()] + + # Get closed POAMs if available + closed_poams = [] + if poam_file.closed_df is not None: + closed_df = poam_file.closed_df[poam_file.closed_df['POAM ID'].str.match(zap_pattern, na=False)] + closed_poams = [PoamEntry.from_dict(row) for _, row in closed_df.iterrows()] + + return open_poams, closed_poams + +def compare_findings_to_zap_poams(findings: List[Finding], poam_file: Path) -> PoamFileDiff: + """ + Compare a list of findings against ZAP POAMs. + + Args: + findings: List of current findings from ZAP + poam_file: Path to Excel file containing ZAP POAMs + + Returns: + PoamFileDiff containing new, existing, closed, and reopened findings + """ + # Load ZAP POAMs + poam_file_handler = PoamFile(poam_file) + open_poams, closed_poams = get_zap_poam_entries(poam_file_handler) + + # Get all POAM IDs for generating new ones + all_poam_ids = [p.poam_id for p in [*open_poams, *closed_poams]] + + return compare_findings_to_poams(findings, open_poams, closed_poams, all_poam_ids, generate_poams_from_findings, store_as_configuration_findings=False) diff --git a/tools/zap/poam_generator.py b/tools/zap/poam_generator.py new file mode 100644 index 0000000..e5bbc57 --- /dev/null +++ b/tools/zap/poam_generator.py @@ -0,0 +1,171 @@ +""" +Module for generating POAMs from ZAP findings. +""" +from datetime import datetime +from typing import List, Dict, Tuple +import re +from ..findings import Finding +from ..poam import PoamEntry + +def parse_zap_id(poam_id: str) -> Tuple[int, int]: + """ + Parse a ZAP POAM ID into year and sequence components. + + Args: + poam_id: POAM ID in format YYYY-ZAPXXXX + + Returns: + Tuple of (year, sequence_number) + + Raises: + ValueError: If the ID format is invalid + """ + match = re.match(r'^(\d{4})-ZAP(\d{4,})$', poam_id) + if not match: + raise ValueError(f"Invalid ZAP POAM ID format: {poam_id}") + + year = int(match.group(1)) + sequence = int(match.group(2)) + return year, sequence + +def get_next_zap_id(existing_poam_ids: List[str], current_year: int = None) -> str: + """ + Generate the next available ZAP POAM ID. + + Args: + existing_poam_ids: List of existing POAM IDs + current_year: Optional year to use (defaults to current year) + + Returns: + Next available POAM ID in format YYYY-ZAPXXXX + """ + current_year = datetime.now().year if current_year is None else current_year + + # Find highest sequence number for the current year + max_sequence = 0 + for poam_id in existing_poam_ids: + try: + year, sequence = parse_zap_id(poam_id) + if year == current_year: + max_sequence = max(max_sequence, sequence) + except ValueError: + continue # Skip non-ZAP IDs + + return f"{current_year}-ZAP{max_sequence + 1:04d}" + +def findings_to_poam(findings: List[Finding], poam_id: str) -> PoamEntry: + """ + Convert a list of findings with the same weakness into a single POAM. + + Args: + findings: List of findings with the same weakness + poam_id: POAM ID to use for the new POAM + + Returns: + PoamEntry combining all findings + + Raises: + ValueError: If findings have different weakness names + """ + if not findings: + raise ValueError("Cannot create POAM from empty findings list") + + # Verify all findings have the same weakness name + weakness_name = findings[0].weakness_name + if not all(f.weakness_name == weakness_name for f in findings): + raise ValueError("All findings must have the same weakness name") + + # Combine asset identifiers and finding IDs + asset_identifiers = [f.asset_identifier for f in findings] + finding_ids = [f.finding_id for f in findings] + + # Use the first finding as a template + first = findings[0] + return PoamEntry( + poam_id=poam_id, + controls=first.controls, + weakness_name=first.weakness_name, + weakness_description=first.weakness_description, + weakness_detector_source=first.weakness_detector_source, + weakness_source_identifier=first.weakness_source_identifier, + asset_identifier=", ".join(asset_identifiers), + point_of_contact=first.point_of_contact, + resources_required=first.resources_required, + overall_remediation_plan=first.overall_remediation_plan, + original_detection_date=first.original_detection_date, + scheduled_completion_date=first.scheduled_completion_date, + planned_milestones=first.planned_milestones, + milestone_changes=first.milestone_changes, + status_date=first.status_date, + vendor_dependency=first.vendor_dependency, + last_vendor_check_in_date=first.last_vendor_check_in_date, + vendor_dependent_product_name=first.vendor_dependent_product_name, + original_risk_rating=first.original_risk_rating, + adjusted_risk_rating=first.adjusted_risk_rating, + risk_adjustment=first.risk_adjustment, + false_positive=first.false_positive, + operational_requirement=first.operational_requirement, + deviation_rationale=first.deviation_rationale, + supporting_documents=first.supporting_documents, + comments=", ".join(finding_ids), # Store finding IDs in comments + auto_approve=first.auto_approve, + binding_operational_directive_22_01_tracking=first.binding_operational_directive_22_01_tracking, + binding_operational_directive_22_01_due_date=first.binding_operational_directive_22_01_due_date, + cve=first.cve, + service_name=first.service_name + ) + +def group_findings_by_weakness(findings: List[Finding]) -> Dict[str, List[Finding]]: + """ + Group findings by weakness name. + + Args: + findings: List of findings to group + + Returns: + Dictionary mapping weakness names to lists of findings, with findings sorted by ID + """ + groups: Dict[str, List[Finding]] = {} + for finding in findings: + groups.setdefault(finding.weakness_name, []).append(finding) + + # Sort each group by finding ID + for findings_list in groups.values(): + findings_list.sort(key=lambda f: f.finding_id) + + return groups + +def generate_poams_from_findings(findings: List[Finding], existing_poam_ids: List[str], current_year: int = None) -> List[Tuple[List[Finding], PoamEntry]]: + """ + Generate new POAMs from a list of findings. + + Args: + findings: List of findings to convert to POAMs + existing_poam_ids: List of existing POAM IDs + current_year: Optional year to use (defaults to current year) + + Returns: + List of tuples containing (findings_list, generated_poam), sorted by first finding ID + """ + # Group findings by weakness + grouped_findings = group_findings_by_weakness(findings) + + # Sort groups by the first finding ID in each group + sorted_groups = sorted( + grouped_findings.values(), + key=lambda findings_list: findings_list[0].finding_id if findings_list else "" + ) + + # Generate POAMs for each group + result = [] + current_year = datetime.now().year if current_year is None else current_year + next_id = get_next_zap_id(existing_poam_ids, current_year) + + for findings_list in sorted_groups: + poam = findings_to_poam(findings_list, next_id) + result.append((findings_list, poam)) + + # Get next ID for the next group + next_id = get_next_zap_id([*existing_poam_ids, next_id], current_year) + + return result \ No newline at end of file