Merge pull request #100 from multimindlab/feature-fix-API #270
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: CI | |
| on: | |
| push: | |
| branches: [main, develop] | |
| pull_request: | |
| branches: [main, develop] | |
| workflow_dispatch: | |
| # Cancel in-progress runs on the same ref when a new commit lands. | |
| concurrency: | |
| group: ci-${{ github.workflow }}-${{ github.ref }} | |
| cancel-in-progress: true | |
| env: | |
| PIP_DISABLE_PIP_VERSION_CHECK: "1" | |
| jobs: | |
| # --------------------------------------------------------------------------- | |
| # Lint (fast — runs once, on one Python version) | |
| # --------------------------------------------------------------------------- | |
| lint: | |
| name: Lint (ruff check + ruff format) | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: actions/setup-python@v5 | |
| with: | |
| python-version: "3.11" | |
| - name: Install lint tools | |
| run: pip install "ruff>=0.4" | |
| - name: ruff check (lints) | |
| run: ruff check multimind/ | |
| - name: ruff format --check (formatter) | |
| run: ruff format --check multimind/ | |
| # --------------------------------------------------------------------------- | |
| # Security scanning. | |
| # | |
| # bandit — static analysis of our own code. HIGH-severity findings | |
| # fail the job (0 today). Medium/low are surfaced in the log | |
| # but don't block (the codebase has ~190 medium findings, | |
| # mostly B615 HuggingFace revision-pinning noise — tracked | |
| # for a future cleanup, see SECURITY.md). | |
| # pip-audit — dependency CVE scan. Advisory (continue-on-error) because | |
| # several CVEs live in transitive deps without a fixed | |
| # release yet; we still want them visible on every PR. | |
| # --------------------------------------------------------------------------- | |
| security: | |
| name: Security (bandit + pip-audit) | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: actions/setup-python@v5 | |
| with: | |
| python-version: "3.11" | |
| - name: Install security tools | |
| run: pip install bandit pip-audit | |
| - name: bandit (high-severity gate) | |
| run: bandit -r multimind/ -lll | |
| - name: bandit (full report — informational) | |
| if: always() | |
| run: bandit -r multimind/ -ll || true | |
| - name: pip-audit (dependency CVEs — advisory) | |
| if: always() | |
| continue-on-error: true | |
| run: | | |
| pip install -e ".[dev]" | |
| pip-audit --progress-spinner off | |
| # --------------------------------------------------------------------------- | |
| # Core tests across the supported Python matrix. | |
| # | |
| # Installs only the [dev] extra so we exercise the lazy-imports surface | |
| # (Phase 2). Tests that require heavy extras are gated by markers and | |
| # filtered out here, then run in dedicated jobs below. | |
| # --------------------------------------------------------------------------- | |
| test-core: | |
| name: test-core (py${{ matrix.python-version }}) | |
| runs-on: ubuntu-latest | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| python-version: ["3.9", "3.10", "3.11", "3.12", "3.13"] | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: actions/setup-python@v5 | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| cache: pip | |
| - name: Install core + dev | |
| run: | | |
| python -m pip install --upgrade pip | |
| pip install -e ".[dev]" | |
| - name: Run core tests | |
| run: | | |
| pytest tests/ -v \ | |
| --tb=short \ | |
| --junitxml=test-results-core.xml \ | |
| -m "not integration and not slow and not requires_api_key" | |
| - name: Upload test results | |
| if: always() | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: test-results-core-py${{ matrix.python-version }} | |
| path: test-results-core.xml | |
| # --------------------------------------------------------------------------- | |
| # RAG extras (faiss-cpu, chromadb, sentence-transformers, etc.) | |
| # --------------------------------------------------------------------------- | |
| test-rag: | |
| name: test-rag | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: actions/setup-python@v5 | |
| with: | |
| python-version: "3.11" | |
| cache: pip | |
| - name: Install rag + vector-stores + dev | |
| run: | | |
| python -m pip install --upgrade pip | |
| pip install -e ".[rag,vector-stores,dev]" | |
| - name: Run RAG-related tests | |
| run: | | |
| pytest tests/ -v \ | |
| --tb=short \ | |
| --junitxml=test-results-rag.xml \ | |
| -m "not integration and not slow and not requires_api_key" \ | |
| tests/test_retrieval.py \ | |
| tests/test_vector_store.py \ | |
| tests/test_document_loader.py | |
| # --------------------------------------------------------------------------- | |
| # Compliance extras (cryptography, plotly, dash, pandas) | |
| # --------------------------------------------------------------------------- | |
| test-compliance: | |
| name: test-compliance | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: actions/setup-python@v5 | |
| with: | |
| python-version: "3.11" | |
| cache: pip | |
| - name: Install compliance + dev | |
| run: | | |
| python -m pip install --upgrade pip | |
| pip install -e ".[compliance,dev]" | |
| - name: Run compliance tests | |
| run: | | |
| pytest tests/ -v \ | |
| --tb=short \ | |
| --junitxml=test-results-compliance.xml \ | |
| -m "not integration and not slow and not requires_api_key" \ | |
| tests/test_compliance_legacy_imports.py \ | |
| tests/test_compliance_controls.py \ | |
| tests/examples/compliance/ | |
| # --------------------------------------------------------------------------- | |
| # Fine-tuning extras (torch, transformers, peft, datasets, optuna) | |
| # Heavy install — limited to one Python version. | |
| # --------------------------------------------------------------------------- | |
| test-finetune: | |
| name: test-finetune | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: actions/setup-python@v5 | |
| with: | |
| python-version: "3.11" | |
| cache: pip | |
| - name: Install finetune + dev (CPU torch) | |
| run: | | |
| python -m pip install --upgrade pip | |
| # CPU-only torch wheel keeps the install size manageable on | |
| # GitHub-hosted runners. | |
| pip install "torch>=2.0.0" --index-url https://download.pytorch.org/whl/cpu | |
| pip install -e ".[finetune,dev]" | |
| - name: Run fine-tuning tests | |
| run: | | |
| pytest -v \ | |
| --tb=short \ | |
| --junitxml=test-results-finetune.xml \ | |
| -m "not integration and not slow and not requires_api_key" \ | |
| tests/test_model_client.py \ | |
| tests/test_llm.py \ | |
| tests/test_llm_wrappers.py | |
| # --------------------------------------------------------------------------- | |
| # Gateway / API server extras (fastapi, uvicorn, redis, jose) | |
| # --------------------------------------------------------------------------- | |
| test-gateway: | |
| name: test-gateway | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: actions/setup-python@v5 | |
| with: | |
| python-version: "3.11" | |
| cache: pip | |
| - name: Install gateway + dev | |
| run: | | |
| python -m pip install --upgrade pip | |
| pip install -e ".[gateway,dev]" | |
| - name: Run gateway tests | |
| run: | | |
| pytest tests/ -v \ | |
| --tb=short \ | |
| --junitxml=test-results-gateway.xml \ | |
| -m "not integration and not slow and not requires_api_key" \ | |
| -k "gateway or api or mcp" | |
| # --------------------------------------------------------------------------- | |
| # Full suite with everything installed, including the framework-interop | |
| # extras (langchain-core, llama-index-core) and the MCP SDK (already part | |
| # of `all`). This is the only job where tests that `importorskip` those | |
| # packages actually execute instead of skipping. Enforces two gates: a | |
| # 95% pass-rate threshold (see the Python step below) and a 20% line | |
| # coverage floor (--cov-fail-under=20) — NOT a 95% coverage gate. | |
| # --------------------------------------------------------------------------- | |
| test-full: | |
| name: test-full (95% pass-rate gate, 20% coverage floor) | |
| runs-on: ubuntu-latest | |
| needs: [test-core] | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: actions/setup-python@v5 | |
| with: | |
| python-version: "3.11" | |
| cache: pip | |
| - name: Install all extras + dev | |
| run: | | |
| python -m pip install --upgrade pip | |
| pip install "torch>=2.0.0" --index-url https://download.pytorch.org/whl/cpu | |
| pip install -e ".[all,dev,langchain,llamaindex]" | |
| - name: Run full suite with coverage | |
| run: | | |
| pytest tests/ \ | |
| --cov=multimind \ | |
| --cov-report=term-missing \ | |
| --cov-report=xml \ | |
| --cov-fail-under=20 \ | |
| --junitxml=test-results-full.xml \ | |
| -v --tb=short \ | |
| | tee pytest-summary.txt | |
| - name: Enforce 95% pass-rate threshold | |
| run: | | |
| python - <<'PY' | |
| import re, sys | |
| with open("pytest-summary.txt") as f: | |
| out = f.read() | |
| passed = int((re.search(r"(\d+) passed", out) or [0,"0"])[1]) | |
| failed = int((re.search(r"(\d+) failed", out) or [0,"0"])[1]) | |
| skipped = int((re.search(r"(\d+) skipped", out) or [0,"0"])[1]) | |
| total = passed + failed + skipped | |
| rate = (passed / total * 100) if total else 0.0 | |
| print(f"Results: {passed} passed, {failed} failed, {skipped} skipped") | |
| print(f"Pass rate: {rate:.1f}%") | |
| if rate < 95.0: | |
| sys.exit(f"FAIL: pass rate {rate:.1f}% < 95%") | |
| print("PASS: meets 95% threshold") | |
| PY | |
| - name: Upload coverage | |
| if: always() | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: coverage-xml | |
| path: coverage.xml | |
| - name: Upload test results | |
| if: always() | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: test-results-full | |
| path: test-results-full.xml |