python-package.yml

# This workflow will install Python dependencies, run tests and lint with a variety of Python versions
# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions

name: MSTICPy CI build and check

on:
  push:
    branches: [main]
  pull_request:
    branches: [main, release/*]
  merge_group:
  schedule:
    - cron: "0 0 * * 0,2,4"

jobs:
  build:
    runs-on: ubuntu-latest
    permissions: read-all
    strategy:
      matrix:
        python-version: ["3.10", "3.11", "3.12", "3.13", "3.14"]
    steps:
      # Print out details about the run
      - name: Dump GitHub context
        env:
          GITHUB_CONTEXT: ${{ toJSON(github) }}
        run: echo "$GITHUB_CONTEXT"
      - name: Dump job context
        env:
          JOB_CONTEXT: ${{ toJSON(job) }}
        run: echo "$JOB_CONTEXT"
      # end print details
      - uses: actions/checkout@v4
      - name: Free up disk space
        run: |
          sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc
          sudo rm -rf /usr/share/locale/*
          sudo rm -rf /usr/share/i18n/*
      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python-version }}
      - name: Cache pip
        uses: actions/cache@v4
        with:
          # This path is specific to Ubuntu
          path: ~/.cache/pip
          # Look to see if there is a cache hit for the corresponding requirements file
          key: ${{ runner.os }}-pip-${{ hashFiles('setup.py', 'requirements.txt', 'requirements-dev.txt') }}
          restore-keys: |
            ${{ runner.os }}-pip-${{ hashFiles('setup.py', 'requirements.txt', 'requirements-dev.txt') }}
            ${{ runner.os }}-pip
      - name: Install dependencies
        # Installs core install + all extras ("all") + dev/test dependencies ("dev")
        # via the combined "test" extra defined in setup.py.
        run: |
          python -m pip install --upgrade pip wheel setuptools
          python -m pip install --use-pep517 -e ".[test]"
      - name: Install test dependencies
        # ToDo - remove pip install xgboost when flaml is fixed
        run: |
          python -m pip install "pandas>=1.3.0" "pygeohash>=1.2.0"
          python -m pip install "xgboost"
      - name: Prepare test dummy data
        run: |
          mkdir ~/.msticpy
          mkdir ~/.msticpy/mordor
          cp ./tests/testdata/geolite/GeoLite2-City.mmdb ~/.msticpy
          touch ~/.msticpy/GeoLite2-City.mmdb
          cp -r ./tests/testdata/mordor/* ~/.msticpy/mordor
          touch ~/.msticpy/mordor/mitre_tact_cache.pkl
          touch ~/.msticpy/mordor/mitre_tech_cache.pkl
          touch ~/.msticpy/mordor/mordor_cache.pkl
      - name: Pytest
        env:
          MAXMIND_AUTH: ${{ secrets.MAXMIND_AUTH }}
          IPSTACK_AUTH: ${{ secrets.IPSTACK_AUTH }}
          MSTICPYCONFIG: ./tests/msticpyconfig-test.yaml
          MSTICPY_BUILD_SOURCE: fork
          JUPYTER_PLATFORM_DIRS: 1
        run: |
          jupyter --paths
          pytest tests -n auto --junitxml=junit/test-${{ matrix.python-version }}-results.xml --cov=msticpy --cov-report=xml
        if: ${{ always() }}
      - name: Upload pytest test results
        uses: actions/upload-artifact@v4
        with:
          name: pytest-results-${{ matrix.python-version }}
          path: junit/test-${{ matrix.python-version }}-results.xml
        # Use always() to always run this step to publish test results when there are test failures
        if: ${{ always() }}
  docs:
    runs-on: ubuntu-latest
    permissions: read-all
    strategy:
      matrix:
        python-version: ["3.12"]
    steps:
      - uses: actions/checkout@v4
      - name: Free up disk space
        run: |
          sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc
          sudo rm -rf /usr/share/locale/*
          sudo rm -rf /usr/share/i18n/*
      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python-version }}
      - name: Cache pip
        uses: actions/cache@v4
        with:
          # This path is specific to Ubuntu
          path: ~/.cache/pip
          # Look to see if there is a cache hit for the corresponding requirements file
          key: ${{ runner.os }}-pip-docs-${{ hashFiles('docs/requirements.txt') }}
          restore-keys: |
            ${{ runner.os }}-pip-docs-${{ hashFiles('docs/requirements.txt') }}
            ${{ runner.os }}-pip-docs
            ${{ runner.os }}-pip
      - name: Sphinx Read the Docs build
        working-directory: docs
        run: |
          echo "Building docs from ${{ github.workspace }}/docs"
          python -m pip install sphinx readthedocs-sphinx-ext>=2.1.4 sphinx-rtd-theme>=1.0.0
          python -m pip install -r ${{ github.workspace }}/docs/requirements.txt
          make html
        env:
          SPHINX_NOGEN: "true"
  lint:
    runs-on: ubuntu-latest
    permissions: read-all
    strategy:
      matrix:
        python-version: ["3.12"]
    steps:
      - uses: actions/checkout@v4
      - name: Free up disk space
        run: |
          sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc
          sudo rm -rf /usr/share/locale/*
          sudo rm -rf /usr/share/i18n/*
      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python-version }}
      - name: Cache pip
        uses: actions/cache@v4
        with:
          # This path is specific to Ubuntu
          path: ~/.cache/pip
          # Look to see if there is a cache hit for the corresponding requirements file
          key: ${{ runner.os }}-pip-lint-${{ hashFiles('setup.py', 'requirements.txt', 'requirements-dev.txt') }}
          restore-keys: |
            ${{ runner.os }}-pip-lint-${{ hashFiles('setup.py', 'requirements.txt', 'requirements-dev.txt') }}
            ${{ runner.os }}-pip-lint
            ${{ runner.os }}-pip
      - name: Install dependencies
        # Installs core install + all extras ("all") + dev/test dependencies ("dev")
        # via the combined "test" extra defined in setup.py.
        run: |
          python -m pip install --upgrade pip wheel setuptools
          python -m pip install -e ".[test]"
      - name: ruff
        run: |
          ruff check msticpy --ignore PLW0603
        if: ${{ always() }}
      - name: pylint
        run: |
          pylint msticpy --disable=duplicate-code --disable=E1135,E1101,E1133,W0101
        if: ${{ always() }}
      - name: Cache/restore MyPy data
        id: cache-mypy
        uses: actions/cache@v4
        with:
          # MyPy cache files are stored in `~/.mypy_cache`
          path: .mypy_cache
          key: ${{ runner.os }}-build-mypy-${{ github.ref }}-${{ github.sha }}
          restore-keys: |
            ${{ runner.os }}-build-mypy-${{ github.ref }}-${{ github.sha }}
            ${{ runner.os }}-build-mypy-${{ github.ref }}
            ${{ runner.os }}-build-mypy
      - name: mypy
        # --python-version is set explicitly to the job's Python version because mypy
        # otherwise infers its syntax target from setup.cfg's python_requires (the
        # package's minimum supported version), which can cause false positives on
        # newer-syntax stubs (e.g. PEP 695 `type` statements in recent numpy releases).
        run: |
          mypy --ignore-missing-imports --follow-imports=silent --show-column-numbers --show-error-end --show-error-context --disable-error-code annotation-unchecked --python-version ${{ matrix.python-version }} --junit-xml junit/mypy-test-${{ matrix.python-version }}-results.xml msticpy
        if: ${{ always() }}
      - name: Upload mypy test results
        uses: actions/upload-artifact@v4
        with:
          name: Mypy results ${{ matrix.python-version }}
          path: junit/mypy-test-${{ matrix.python-version }}-results.xml
        # Use always() to always run this step to publish test results when there are test failures
        if: ${{ always() }}
      - name: bandit
        run: |
          bandit -x tests -r -s B303,B404,B603,B607,B608,B113 msticpy
        if: ${{ always() }}
      - name: pyroma
        run: |
          pyroma --min 10 .
        if: ${{ always() }}
  check_status:
    runs-on: ubuntu-latest
    permissions: read-all
    needs: [build, lint, docs]
    steps:
      - name: File build fail issue
        if: ${{ env.GITHUB_REF_NAME == 'main' && ( needs.build.result == 'failure' || needs.lint.result == 'failure' ) }}
        uses: dacbd/create-issue-action@v1
        with:
          token: ${{ github.token }}
          title: "Build failed for main branch"
          body: The build failed on branch ${{ github.ref }}. Please investigate
          labels: build_break, bug, high_severity