[deps] Dependency update: troubleshoot #679
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Regression Test Suite | |
| on: | |
| push: | |
| branches: [main, v1beta3] | |
| pull_request: | |
| types: [opened, synchronize, reopened] | |
| workflow_dispatch: | |
| inputs: | |
| update_baselines: | |
| description: 'Update baselines after run (use with caution)' | |
| type: boolean | |
| default: false | |
| jobs: | |
| # Build binaries once (shared by all test jobs) | |
| build-binaries: | |
| if: github.actor != 'dependabot[bot]' | |
| runs-on: ubuntu-22.04 | |
| timeout-minutes: 5 | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v7 | |
| with: | |
| fetch-depth: 0 | |
| - name: Setup Go | |
| uses: actions/setup-go@v7 | |
| with: | |
| go-version-file: 'go.mod' | |
| - name: Build binaries | |
| run: | | |
| echo "Building preflight and support-bundle binaries..." | |
| make bin/preflight bin/support-bundle | |
| ./bin/preflight version | |
| ./bin/support-bundle version | |
| - name: Upload binaries | |
| uses: actions/upload-artifact@v7 | |
| with: | |
| name: binaries-${{ github.run_id }} | |
| path: | | |
| bin/preflight | |
| bin/support-bundle | |
| retention-days: 1 | |
| # Preflight v1beta3 test (parallel job 1) | |
| test-preflight-v1beta3: | |
| needs: [build-binaries] | |
| if: github.actor != 'dependabot[bot]' | |
| runs-on: ubuntu-22.04 | |
| timeout-minutes: 15 | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v7 | |
| - name: Create output directory | |
| run: mkdir -p test/output | |
| - name: Create k3s cluster | |
| uses: replicatedhq/action-k3s@main | |
| with: | |
| version: v1.31.2-k3s1 | |
| - name: Verify cluster access | |
| run: kubectl get nodes -o wide | |
| - name: Wait for all pods to be ready | |
| run: | | |
| echo "Waiting for all pods to be running..." | |
| kubectl get pods --all-namespaces | |
| kubectl wait --for=condition=Ready pods --all --all-namespaces --timeout=300s || true | |
| kubectl get pods --all-namespaces | |
| - name: Download binaries | |
| uses: actions/download-artifact@v8 | |
| with: | |
| name: binaries-${{ github.run_id }} | |
| path: bin/ | |
| - name: Make binaries executable | |
| run: | | |
| chmod +x bin/preflight bin/support-bundle | |
| ./bin/preflight version | |
| - name: Setup Python for comparison | |
| uses: actions/setup-python@v7 | |
| with: | |
| python-version: '3.11' | |
| - name: Install Python dependencies | |
| run: pip install pyyaml deepdiff | |
| - name: Run preflight v1beta3 | |
| continue-on-error: true | |
| run: | | |
| echo "Running preflight v1beta3..." | |
| ./bin/preflight \ | |
| examples/preflight/complex-v1beta3.yaml \ | |
| --values examples/preflight/values-complex-full.yaml \ | |
| --interactive=false \ | |
| --format=json \ | |
| --auto-update=false \ | |
| --output=test/output/preflight-results-v1beta3.json 2>&1 | tee test/output/v1beta3.log || true | |
| BUNDLE=$(ls -t preflightbundle-*.tar.gz 2>/dev/null | head -1) | |
| if [ -n "$BUNDLE" ]; then | |
| mv "$BUNDLE" test/output/preflight-v1beta3-bundle.tar.gz | |
| echo "✓ v1beta3 bundle saved" | |
| fi | |
| - name: Compare preflight v1beta3 bundle | |
| id: compare | |
| continue-on-error: true | |
| run: | | |
| echo "Comparing v1beta3 preflight bundle against baseline..." | |
| if [ ! -f test/baselines/preflight-v1beta3/baseline.tar.gz ]; then | |
| echo "⚠ No baseline found for v1beta3 - skipping comparison" | |
| echo "baseline_missing=true" >> $GITHUB_OUTPUT | |
| exit 0 | |
| fi | |
| python3 scripts/compare_bundles.py \ | |
| --baseline test/baselines/preflight-v1beta3/baseline.tar.gz \ | |
| --current test/output/preflight-v1beta3-bundle.tar.gz \ | |
| --rules scripts/compare_rules.yaml \ | |
| --report test/output/diff-report-v1beta3.json \ | |
| --spec-type preflight | |
| - name: Upload test artifacts | |
| if: ${{ !cancelled() }} | |
| uses: actions/upload-artifact@v7 | |
| with: | |
| name: test-results-v1beta3-${{ github.run_id }}-${{ github.run_attempt }} | |
| path: | | |
| test/output/preflight-v1beta3-bundle.tar.gz | |
| test/output/preflight-results-v1beta3.json | |
| test/output/diff-report-v1beta3.json | |
| test/output/v1beta3.log | |
| retention-days: 30 | |
| - name: Set job outcome | |
| if: ${{ !cancelled() }} | |
| run: | | |
| if [ "${{ steps.compare.outcome }}" == "failure" ] && [ "${{ steps.compare.outputs.baseline_missing }}" != "true" ]; then | |
| echo "comparison_failed=true" >> $GITHUB_OUTPUT | |
| exit 1 | |
| fi | |
| # Preflight v1beta2 test (parallel job 2) | |
| test-preflight-v1beta2: | |
| needs: [build-binaries] | |
| if: github.actor != 'dependabot[bot]' | |
| runs-on: ubuntu-22.04 | |
| timeout-minutes: 15 | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v7 | |
| - name: Create output directory | |
| run: mkdir -p test/output | |
| - name: Create k3s cluster | |
| uses: replicatedhq/action-k3s@main | |
| with: | |
| version: v1.31.2-k3s1 | |
| - name: Verify cluster access | |
| run: kubectl get nodes -o wide | |
| - name: Wait for all pods to be ready | |
| run: | | |
| echo "Waiting for all pods to be running..." | |
| kubectl get pods --all-namespaces | |
| kubectl wait --for=condition=Ready pods --all --all-namespaces --timeout=300s || true | |
| kubectl get pods --all-namespaces | |
| - name: Download binaries | |
| uses: actions/download-artifact@v8 | |
| with: | |
| name: binaries-${{ github.run_id }} | |
| path: bin/ | |
| - name: Make binaries executable | |
| run: | | |
| chmod +x bin/preflight bin/support-bundle | |
| ./bin/preflight version | |
| - name: Setup Python for comparison | |
| uses: actions/setup-python@v7 | |
| with: | |
| python-version: '3.11' | |
| - name: Install Python dependencies | |
| run: pip install pyyaml deepdiff | |
| - name: Run preflight v1beta2 | |
| continue-on-error: true | |
| run: | | |
| echo "Running preflight v1beta2..." | |
| ./bin/preflight \ | |
| examples/preflight/all-analyzers-v1beta2.yaml \ | |
| --interactive=false \ | |
| --format=json \ | |
| --auto-update=false \ | |
| --output=test/output/preflight-results-v1beta2.json 2>&1 | tee test/output/v1beta2.log || true | |
| BUNDLE=$(ls -t preflightbundle-*.tar.gz 2>/dev/null | head -1) | |
| if [ -n "$BUNDLE" ]; then | |
| mv "$BUNDLE" test/output/preflight-v1beta2-bundle.tar.gz | |
| echo "✓ v1beta2 bundle saved" | |
| fi | |
| - name: Compare preflight v1beta2 bundle | |
| id: compare | |
| continue-on-error: true | |
| run: | | |
| echo "Comparing v1beta2 preflight bundle against baseline..." | |
| if [ ! -f test/baselines/preflight-v1beta2/baseline.tar.gz ]; then | |
| echo "⚠ No baseline found for v1beta2 - skipping comparison" | |
| echo "baseline_missing=true" >> $GITHUB_OUTPUT | |
| exit 0 | |
| fi | |
| python3 scripts/compare_bundles.py \ | |
| --baseline test/baselines/preflight-v1beta2/baseline.tar.gz \ | |
| --current test/output/preflight-v1beta2-bundle.tar.gz \ | |
| --rules scripts/compare_rules.yaml \ | |
| --report test/output/diff-report-v1beta2.json \ | |
| --spec-type preflight | |
| - name: Upload test artifacts | |
| if: ${{ !cancelled() }} | |
| uses: actions/upload-artifact@v7 | |
| with: | |
| name: test-results-v1beta2-${{ github.run_id }}-${{ github.run_attempt }} | |
| path: | | |
| test/output/preflight-v1beta2-bundle.tar.gz | |
| test/output/preflight-results-v1beta2.json | |
| test/output/diff-report-v1beta2.json | |
| test/output/v1beta2.log | |
| retention-days: 30 | |
| - name: Set job outcome | |
| if: ${{ !cancelled() }} | |
| run: | | |
| if [ "${{ steps.compare.outcome }}" == "failure" ] && [ "${{ steps.compare.outputs.baseline_missing }}" != "true" ]; then | |
| echo "comparison_failed=true" >> $GITHUB_OUTPUT | |
| exit 1 | |
| fi | |
| # Support bundle test (parallel job 3) | |
| test-supportbundle: | |
| needs: [build-binaries] | |
| if: github.actor != 'dependabot[bot]' | |
| runs-on: ubuntu-22.04 | |
| timeout-minutes: 15 | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v7 | |
| - name: Create output directory | |
| run: mkdir -p test/output | |
| - name: Create k3s cluster | |
| uses: replicatedhq/action-k3s@main | |
| with: | |
| version: v1.31.2-k3s1 | |
| - name: Verify cluster access | |
| run: kubectl get nodes -o wide | |
| - name: Wait for all pods to be ready | |
| run: | | |
| echo "Waiting for all pods to be running..." | |
| kubectl get pods --all-namespaces | |
| kubectl wait --for=condition=Ready pods --all --all-namespaces --timeout=300s || true | |
| kubectl get pods --all-namespaces | |
| - name: Download binaries | |
| uses: actions/download-artifact@v8 | |
| with: | |
| name: binaries-${{ github.run_id }} | |
| path: bin/ | |
| - name: Make binaries executable | |
| run: | | |
| chmod +x bin/preflight bin/support-bundle | |
| ./bin/support-bundle version | |
| - name: Setup Python for comparison | |
| uses: actions/setup-python@v7 | |
| with: | |
| python-version: '3.11' | |
| - name: Install Python dependencies | |
| run: pip install pyyaml deepdiff | |
| - name: Run support bundle | |
| continue-on-error: true | |
| run: | | |
| echo "Running support bundle..." | |
| ./bin/support-bundle \ | |
| examples/collect/host/all-kubernetes-collectors.yaml \ | |
| --interactive=false \ | |
| --auto-update=false \ | |
| --output=test/output/supportbundle.tar.gz 2>&1 | tee test/output/supportbundle.log || true | |
| if [ -f test/output/supportbundle.tar.gz ]; then | |
| echo "✓ Support bundle saved" | |
| fi | |
| - name: Compare support bundle | |
| id: compare | |
| continue-on-error: true | |
| run: | | |
| echo "Comparing support bundle against baseline..." | |
| if [ ! -f test/baselines/supportbundle/baseline.tar.gz ]; then | |
| echo "⚠ No baseline found for support bundle - skipping comparison" | |
| echo "baseline_missing=true" >> $GITHUB_OUTPUT | |
| exit 0 | |
| fi | |
| python3 scripts/compare_bundles.py \ | |
| --baseline test/baselines/supportbundle/baseline.tar.gz \ | |
| --current test/output/supportbundle.tar.gz \ | |
| --rules scripts/compare_rules.yaml \ | |
| --report test/output/diff-report-supportbundle.json \ | |
| --spec-type supportbundle | |
| - name: Upload test artifacts | |
| if: ${{ !cancelled() }} | |
| uses: actions/upload-artifact@v7 | |
| with: | |
| name: test-results-supportbundle-${{ github.run_id }}-${{ github.run_attempt }} | |
| path: | | |
| test/output/supportbundle.tar.gz | |
| test/output/diff-report-supportbundle.json | |
| test/output/supportbundle.log | |
| retention-days: 30 | |
| - name: Set job outcome | |
| if: ${{ !cancelled() }} | |
| run: | | |
| if [ "${{ steps.compare.outcome }}" == "failure" ] && [ "${{ steps.compare.outputs.baseline_missing }}" != "true" ]; then | |
| echo "comparison_failed=true" >> $GITHUB_OUTPUT | |
| exit 1 | |
| fi | |
| # Report results (runs after all tests complete) | |
| report-results: | |
| needs: [test-preflight-v1beta3, test-preflight-v1beta2, test-supportbundle] | |
| if: ${{ !cancelled() && github.actor != 'dependabot[bot]' }} | |
| runs-on: ubuntu-22.04 | |
| timeout-minutes: 5 | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v7 | |
| - name: Setup Python | |
| uses: actions/setup-python@v7 | |
| with: | |
| python-version: '3.11' | |
| - name: Install Python dependencies | |
| run: pip install pyyaml deepdiff | |
| - name: Download all test artifacts | |
| uses: actions/download-artifact@v8 | |
| with: | |
| path: test/output | |
| pattern: test-results-* | |
| - name: Reorganize artifacts | |
| run: | | |
| # Move artifacts from nested directories to test/output | |
| # Use shopt to handle glob patterns that don't match | |
| shopt -s nullglob | |
| for dir in test/output/test-results-*; do | |
| if [ -d "$dir" ]; then | |
| find "$dir" -type f -exec mv {} test/output/ \; | |
| fi | |
| done | |
| # Clean up empty directories | |
| for dir in test/output/test-results-*; do | |
| if [ -d "$dir" ]; then | |
| find "$dir" -type d -empty -delete || true | |
| fi | |
| done | |
| shopt -u nullglob | |
| - name: Generate summary report | |
| run: | | |
| # Handle case where no reports exist | |
| shopt -s nullglob | |
| REPORTS=(test/output/diff-report-*.json) | |
| shopt -u nullglob | |
| if [ ${#REPORTS[@]} -eq 0 ]; then | |
| echo "⚠ No comparison reports found - test jobs may have failed before generating reports" | |
| echo "## Summary Report" >> $GITHUB_STEP_SUMMARY | |
| echo "No comparison reports available. Check individual test job results." >> $GITHUB_STEP_SUMMARY | |
| else | |
| python3 scripts/generate_summary.py \ | |
| --reports test/output/diff-report-*.json \ | |
| --output-file $GITHUB_STEP_SUMMARY \ | |
| --output-console | |
| fi | |
| - name: Check for regressions | |
| run: | | |
| echo "Checking comparison results..." | |
| FAILURES=0 | |
| if [ "${{ needs.test-preflight-v1beta3.result }}" == "failure" ] || [ "${{ needs.test-preflight-v1beta3.result }}" == "skipped" ]; then | |
| if [ "${{ needs.test-preflight-v1beta3.result }}" == "skipped" ]; then | |
| echo "❌ v1beta3 test was skipped (likely due to build failure)" | |
| else | |
| echo "❌ v1beta3 comparison failed" | |
| fi | |
| FAILURES=$((FAILURES + 1)) | |
| fi | |
| if [ "${{ needs.test-preflight-v1beta2.result }}" == "failure" ] || [ "${{ needs.test-preflight-v1beta2.result }}" == "skipped" ]; then | |
| if [ "${{ needs.test-preflight-v1beta2.result }}" == "skipped" ]; then | |
| echo "❌ v1beta2 test was skipped (likely due to build failure)" | |
| else | |
| echo "❌ v1beta2 comparison failed" | |
| fi | |
| FAILURES=$((FAILURES + 1)) | |
| fi | |
| if [ "${{ needs.test-supportbundle.result }}" == "failure" ] || [ "${{ needs.test-supportbundle.result }}" == "skipped" ]; then | |
| if [ "${{ needs.test-supportbundle.result }}" == "skipped" ]; then | |
| echo "❌ Support bundle test was skipped (likely due to build failure)" | |
| else | |
| echo "❌ Support bundle comparison failed" | |
| fi | |
| FAILURES=$((FAILURES + 1)) | |
| fi | |
| if [ $FAILURES -gt 0 ]; then | |
| echo "" | |
| echo "❌ $FAILURES regression(s) detected!" | |
| echo "Review the comparison reports in the artifacts." | |
| exit 1 | |
| else | |
| echo "✅ All comparisons passed or skipped (no baseline)" | |
| fi | |
| - name: Update baselines | |
| if: ${{ !cancelled() && github.event.inputs.update_baselines == 'true' && github.event_name == 'workflow_dispatch' }} | |
| run: | | |
| echo "Updating baselines with current bundles..." | |
| # Copy new bundles as baselines | |
| if [ -f test/output/preflight-v1beta3-bundle.tar.gz ]; then | |
| mkdir -p test/baselines/preflight-v1beta3 | |
| cp test/output/preflight-v1beta3-bundle.tar.gz test/baselines/preflight-v1beta3/baseline.tar.gz | |
| echo "✓ Updated v1beta3 baseline" | |
| fi | |
| if [ -f test/output/preflight-v1beta2-bundle.tar.gz ]; then | |
| mkdir -p test/baselines/preflight-v1beta2 | |
| cp test/output/preflight-v1beta2-bundle.tar.gz test/baselines/preflight-v1beta2/baseline.tar.gz | |
| echo "✓ Updated v1beta2 baseline" | |
| fi | |
| if [ -f test/output/supportbundle.tar.gz ]; then | |
| mkdir -p test/baselines/supportbundle | |
| cp test/output/supportbundle.tar.gz test/baselines/supportbundle/baseline.tar.gz | |
| echo "✓ Updated support bundle baseline" | |
| fi | |
| # Create metadata file | |
| cat > test/baselines/metadata.json <<EOF | |
| { | |
| "updated_at": "$(date -u +%Y-%m-%dT%H:%M:%SZ)", | |
| "git_sha": "${{ github.sha }}", | |
| "k8s_version": "v1.31.2-k3s1", | |
| "workflow_run": "${{ github.run_id }}" | |
| } | |
| EOF | |
| # Commit and push | |
| git config user.name "github-actions[bot]" | |
| git config user.email "github-actions[bot]@users.noreply.github.com" | |
| git add test/baselines/ | |
| git commit -m "chore: update regression test baselines from run ${{ github.run_id }}" | |
| git push |