regression-test.yaml

name: Regression Test Suite

on:
  push:
    branches: [main, v1beta3]
  pull_request:
    types: [opened, synchronize, reopened]
  workflow_dispatch:
    inputs:
      update_baselines:
        description: 'Update baselines after run (use with caution)'
        type: boolean
        default: false

jobs:
  # Build binaries once (shared by all test jobs)
  build-binaries:
    if: github.actor != 'dependabot[bot]'
    runs-on: ubuntu-22.04
    timeout-minutes: 5

    steps:
      - name: Checkout code
        uses: actions/checkout@v7
        with:
          fetch-depth: 0

      - name: Setup Go
        uses: actions/setup-go@v7
        with:
          go-version-file: 'go.mod'

      - name: Build binaries
        run: |
          echo "Building preflight and support-bundle binaries..."
          make bin/preflight bin/support-bundle
          ./bin/preflight version
          ./bin/support-bundle version

      - name: Upload binaries
        uses: actions/upload-artifact@v7
        with:
          name: binaries-${{ github.run_id }}
          path: |
            bin/preflight
            bin/support-bundle
          retention-days: 1

  # Preflight v1beta3 test (parallel job 1)
  test-preflight-v1beta3:
    needs: [build-binaries]
    if: github.actor != 'dependabot[bot]'
    runs-on: ubuntu-22.04
    timeout-minutes: 15

    steps:
      - name: Checkout code
        uses: actions/checkout@v7

      - name: Create output directory
        run: mkdir -p test/output

      - name: Create k3s cluster
        uses: replicatedhq/action-k3s@main
        with:
          version: v1.31.2-k3s1

      - name: Verify cluster access
        run: kubectl get nodes -o wide

      - name: Wait for all pods to be ready
        run: |
          echo "Waiting for all pods to be running..."
          kubectl get pods --all-namespaces
          kubectl wait --for=condition=Ready pods --all --all-namespaces --timeout=300s || true
          kubectl get pods --all-namespaces

      - name: Download binaries
        uses: actions/download-artifact@v8
        with:
          name: binaries-${{ github.run_id }}
          path: bin/

      - name: Make binaries executable
        run: |
          chmod +x bin/preflight bin/support-bundle
          ./bin/preflight version

      - name: Setup Python for comparison
        uses: actions/setup-python@v7
        with:
          python-version: '3.11'

      - name: Install Python dependencies
        run: pip install pyyaml deepdiff

      - name: Run preflight v1beta3
        continue-on-error: true
        run: |
          echo "Running preflight v1beta3..."
          ./bin/preflight \
            examples/preflight/complex-v1beta3.yaml \
            --values examples/preflight/values-complex-full.yaml \
            --interactive=false \
            --format=json \
            --auto-update=false \
            --output=test/output/preflight-results-v1beta3.json 2>&1 | tee test/output/v1beta3.log || true

          BUNDLE=$(ls -t preflightbundle-*.tar.gz 2>/dev/null | head -1)
          if [ -n "$BUNDLE" ]; then
            mv "$BUNDLE" test/output/preflight-v1beta3-bundle.tar.gz
            echo "✓ v1beta3 bundle saved"
          fi

      - name: Compare preflight v1beta3 bundle
        id: compare
        continue-on-error: true
        run: |
          echo "Comparing v1beta3 preflight bundle against baseline..."
          if [ ! -f test/baselines/preflight-v1beta3/baseline.tar.gz ]; then
            echo "⚠ No baseline found for v1beta3 - skipping comparison"
            echo "baseline_missing=true" >> $GITHUB_OUTPUT
            exit 0
          fi

          python3 scripts/compare_bundles.py \
            --baseline test/baselines/preflight-v1beta3/baseline.tar.gz \
            --current test/output/preflight-v1beta3-bundle.tar.gz \
            --rules scripts/compare_rules.yaml \
            --report test/output/diff-report-v1beta3.json \
            --spec-type preflight

      - name: Upload test artifacts
        if: ${{ !cancelled() }}
        uses: actions/upload-artifact@v7
        with:
          name: test-results-v1beta3-${{ github.run_id }}-${{ github.run_attempt }}
          path: |
            test/output/preflight-v1beta3-bundle.tar.gz
            test/output/preflight-results-v1beta3.json
            test/output/diff-report-v1beta3.json
            test/output/v1beta3.log
          retention-days: 30

      - name: Set job outcome
        if: ${{ !cancelled() }}
        run: |
          if [ "${{ steps.compare.outcome }}" == "failure" ] && [ "${{ steps.compare.outputs.baseline_missing }}" != "true" ]; then
            echo "comparison_failed=true" >> $GITHUB_OUTPUT
            exit 1
          fi

  # Preflight v1beta2 test (parallel job 2)
  test-preflight-v1beta2:
    needs: [build-binaries]
    if: github.actor != 'dependabot[bot]'
    runs-on: ubuntu-22.04
    timeout-minutes: 15

    steps:
      - name: Checkout code
        uses: actions/checkout@v7

      - name: Create output directory
        run: mkdir -p test/output

      - name: Create k3s cluster
        uses: replicatedhq/action-k3s@main
        with:
          version: v1.31.2-k3s1

      - name: Verify cluster access
        run: kubectl get nodes -o wide

      - name: Wait for all pods to be ready
        run: |
          echo "Waiting for all pods to be running..."
          kubectl get pods --all-namespaces
          kubectl wait --for=condition=Ready pods --all --all-namespaces --timeout=300s || true
          kubectl get pods --all-namespaces

      - name: Download binaries
        uses: actions/download-artifact@v8
        with:
          name: binaries-${{ github.run_id }}
          path: bin/

      - name: Make binaries executable
        run: |
          chmod +x bin/preflight bin/support-bundle
          ./bin/preflight version

      - name: Setup Python for comparison
        uses: actions/setup-python@v7
        with:
          python-version: '3.11'

      - name: Install Python dependencies
        run: pip install pyyaml deepdiff

      - name: Run preflight v1beta2
        continue-on-error: true
        run: |
          echo "Running preflight v1beta2..."
          ./bin/preflight \
            examples/preflight/all-analyzers-v1beta2.yaml \
            --interactive=false \
            --format=json \
            --auto-update=false \
            --output=test/output/preflight-results-v1beta2.json 2>&1 | tee test/output/v1beta2.log || true

          BUNDLE=$(ls -t preflightbundle-*.tar.gz 2>/dev/null | head -1)
          if [ -n "$BUNDLE" ]; then
            mv "$BUNDLE" test/output/preflight-v1beta2-bundle.tar.gz
            echo "✓ v1beta2 bundle saved"
          fi

      - name: Compare preflight v1beta2 bundle
        id: compare
        continue-on-error: true
        run: |
          echo "Comparing v1beta2 preflight bundle against baseline..."
          if [ ! -f test/baselines/preflight-v1beta2/baseline.tar.gz ]; then
            echo "⚠ No baseline found for v1beta2 - skipping comparison"
            echo "baseline_missing=true" >> $GITHUB_OUTPUT
            exit 0
          fi

          python3 scripts/compare_bundles.py \
            --baseline test/baselines/preflight-v1beta2/baseline.tar.gz \
            --current test/output/preflight-v1beta2-bundle.tar.gz \
            --rules scripts/compare_rules.yaml \
            --report test/output/diff-report-v1beta2.json \
            --spec-type preflight

      - name: Upload test artifacts
        if: ${{ !cancelled() }}
        uses: actions/upload-artifact@v7
        with:
          name: test-results-v1beta2-${{ github.run_id }}-${{ github.run_attempt }}
          path: |
            test/output/preflight-v1beta2-bundle.tar.gz
            test/output/preflight-results-v1beta2.json
            test/output/diff-report-v1beta2.json
            test/output/v1beta2.log
          retention-days: 30

      - name: Set job outcome
        if: ${{ !cancelled() }}
        run: |
          if [ "${{ steps.compare.outcome }}" == "failure" ] && [ "${{ steps.compare.outputs.baseline_missing }}" != "true" ]; then
            echo "comparison_failed=true" >> $GITHUB_OUTPUT
            exit 1
          fi

  # Support bundle test (parallel job 3)
  test-supportbundle:
    needs: [build-binaries]
    if: github.actor != 'dependabot[bot]'
    runs-on: ubuntu-22.04
    timeout-minutes: 15

    steps:
      - name: Checkout code
        uses: actions/checkout@v7

      - name: Create output directory
        run: mkdir -p test/output

      - name: Create k3s cluster
        uses: replicatedhq/action-k3s@main
        with:
          version: v1.31.2-k3s1

      - name: Verify cluster access
        run: kubectl get nodes -o wide

      - name: Wait for all pods to be ready
        run: |
          echo "Waiting for all pods to be running..."
          kubectl get pods --all-namespaces
          kubectl wait --for=condition=Ready pods --all --all-namespaces --timeout=300s || true
          kubectl get pods --all-namespaces

      - name: Download binaries
        uses: actions/download-artifact@v8
        with:
          name: binaries-${{ github.run_id }}
          path: bin/

      - name: Make binaries executable
        run: |
          chmod +x bin/preflight bin/support-bundle
          ./bin/support-bundle version

      - name: Setup Python for comparison
        uses: actions/setup-python@v7
        with:
          python-version: '3.11'

      - name: Install Python dependencies
        run: pip install pyyaml deepdiff

      - name: Run support bundle
        continue-on-error: true
        run: |
          echo "Running support bundle..."
          ./bin/support-bundle \
            examples/collect/host/all-kubernetes-collectors.yaml \
            --interactive=false \
            --auto-update=false \
            --output=test/output/supportbundle.tar.gz 2>&1 | tee test/output/supportbundle.log || true

          if [ -f test/output/supportbundle.tar.gz ]; then
            echo "✓ Support bundle saved"
          fi

      - name: Compare support bundle
        id: compare
        continue-on-error: true
        run: |
          echo "Comparing support bundle against baseline..."
          if [ ! -f test/baselines/supportbundle/baseline.tar.gz ]; then
            echo "⚠ No baseline found for support bundle - skipping comparison"
            echo "baseline_missing=true" >> $GITHUB_OUTPUT
            exit 0
          fi

          python3 scripts/compare_bundles.py \
            --baseline test/baselines/supportbundle/baseline.tar.gz \
            --current test/output/supportbundle.tar.gz \
            --rules scripts/compare_rules.yaml \
            --report test/output/diff-report-supportbundle.json \
            --spec-type supportbundle

      - name: Upload test artifacts
        if: ${{ !cancelled() }}
        uses: actions/upload-artifact@v7
        with:
          name: test-results-supportbundle-${{ github.run_id }}-${{ github.run_attempt }}
          path: |
            test/output/supportbundle.tar.gz
            test/output/diff-report-supportbundle.json
            test/output/supportbundle.log
          retention-days: 30

      - name: Set job outcome
        if: ${{ !cancelled() }}
        run: |
          if [ "${{ steps.compare.outcome }}" == "failure" ] && [ "${{ steps.compare.outputs.baseline_missing }}" != "true" ]; then
            echo "comparison_failed=true" >> $GITHUB_OUTPUT
            exit 1
          fi

  # Report results (runs after all tests complete)
  report-results:
    needs: [test-preflight-v1beta3, test-preflight-v1beta2, test-supportbundle]
    if: ${{ !cancelled() && github.actor != 'dependabot[bot]' }}
    runs-on: ubuntu-22.04
    timeout-minutes: 5

    steps:
      - name: Checkout code
        uses: actions/checkout@v7

      - name: Setup Python
        uses: actions/setup-python@v7
        with:
          python-version: '3.11'

      - name: Install Python dependencies
        run: pip install pyyaml deepdiff

      - name: Download all test artifacts
        uses: actions/download-artifact@v8
        with:
          path: test/output
          pattern: test-results-*

      - name: Reorganize artifacts
        run: |
          # Move artifacts from nested directories to test/output
          # Use shopt to handle glob patterns that don't match
          shopt -s nullglob
          for dir in test/output/test-results-*; do
            if [ -d "$dir" ]; then
              find "$dir" -type f -exec mv {} test/output/ \;
            fi
          done
          # Clean up empty directories
          for dir in test/output/test-results-*; do
            if [ -d "$dir" ]; then
              find "$dir" -type d -empty -delete || true
            fi
          done
          shopt -u nullglob

      - name: Generate summary report
        run: |
          # Handle case where no reports exist
          shopt -s nullglob
          REPORTS=(test/output/diff-report-*.json)
          shopt -u nullglob
          
          if [ ${#REPORTS[@]} -eq 0 ]; then
            echo "⚠ No comparison reports found - test jobs may have failed before generating reports"
            echo "## Summary Report" >> $GITHUB_STEP_SUMMARY
            echo "No comparison reports available. Check individual test job results." >> $GITHUB_STEP_SUMMARY
          else
            python3 scripts/generate_summary.py \
              --reports test/output/diff-report-*.json \
              --output-file $GITHUB_STEP_SUMMARY \
              --output-console
          fi

      - name: Check for regressions
        run: |
          echo "Checking comparison results..."

          FAILURES=0

          if [ "${{ needs.test-preflight-v1beta3.result }}" == "failure" ] || [ "${{ needs.test-preflight-v1beta3.result }}" == "skipped" ]; then
            if [ "${{ needs.test-preflight-v1beta3.result }}" == "skipped" ]; then
              echo "❌ v1beta3 test was skipped (likely due to build failure)"
            else
              echo "❌ v1beta3 comparison failed"
            fi
            FAILURES=$((FAILURES + 1))
          fi

          if [ "${{ needs.test-preflight-v1beta2.result }}" == "failure" ] || [ "${{ needs.test-preflight-v1beta2.result }}" == "skipped" ]; then
            if [ "${{ needs.test-preflight-v1beta2.result }}" == "skipped" ]; then
              echo "❌ v1beta2 test was skipped (likely due to build failure)"
            else
              echo "❌ v1beta2 comparison failed"
            fi
            FAILURES=$((FAILURES + 1))
          fi

          if [ "${{ needs.test-supportbundle.result }}" == "failure" ] || [ "${{ needs.test-supportbundle.result }}" == "skipped" ]; then
            if [ "${{ needs.test-supportbundle.result }}" == "skipped" ]; then
              echo "❌ Support bundle test was skipped (likely due to build failure)"
            else
              echo "❌ Support bundle comparison failed"
            fi
            FAILURES=$((FAILURES + 1))
          fi

          if [ $FAILURES -gt 0 ]; then
            echo ""
            echo "❌ $FAILURES regression(s) detected!"
            echo "Review the comparison reports in the artifacts."
            exit 1
          else
            echo "✅ All comparisons passed or skipped (no baseline)"
          fi

      - name: Update baselines
        if: ${{ !cancelled() && github.event.inputs.update_baselines == 'true' && github.event_name == 'workflow_dispatch' }}
        run: |
          echo "Updating baselines with current bundles..."

          # Copy new bundles as baselines
          if [ -f test/output/preflight-v1beta3-bundle.tar.gz ]; then
            mkdir -p test/baselines/preflight-v1beta3
            cp test/output/preflight-v1beta3-bundle.tar.gz test/baselines/preflight-v1beta3/baseline.tar.gz
            echo "✓ Updated v1beta3 baseline"
          fi

          if [ -f test/output/preflight-v1beta2-bundle.tar.gz ]; then
            mkdir -p test/baselines/preflight-v1beta2
            cp test/output/preflight-v1beta2-bundle.tar.gz test/baselines/preflight-v1beta2/baseline.tar.gz
            echo "✓ Updated v1beta2 baseline"
          fi

          if [ -f test/output/supportbundle.tar.gz ]; then
            mkdir -p test/baselines/supportbundle
            cp test/output/supportbundle.tar.gz test/baselines/supportbundle/baseline.tar.gz
            echo "✓ Updated support bundle baseline"
          fi

          # Create metadata file
          cat > test/baselines/metadata.json <<EOF
          {
            "updated_at": "$(date -u +%Y-%m-%dT%H:%M:%SZ)",
            "git_sha": "${{ github.sha }}",
            "k8s_version": "v1.31.2-k3s1",
            "workflow_run": "${{ github.run_id }}"
          }
          EOF

          # Commit and push
          git config user.name "github-actions[bot]"
          git config user.email "github-actions[bot]@users.noreply.github.com"
          git add test/baselines/
          git commit -m "chore: update regression test baselines from run ${{ github.run_id }}"
          git push