Merge pull request #916 from microsoft/chore/remove-conda-and-reqs-all

Remove conda reqs and requirements-all.txt; simplify CI installs via setup.py extras

Author: ianhelle

.devcontainer/Dockerfile

@@ -3,12 +3,8 @@ FROM mcr.microsoft.com/vscode/devcontainers/anaconda
 LABEL maintainer="Msticpy Dev Team <msticpy@microsoft.com>"
 
 #installing Msticpy requirements and dependencies
-COPY ./requirements-all.txt /tmp/
-COPY ./requirements-dev.txt /tmp/
-RUN pip install --requirement /tmp/requirements-all.txt & \
-pip install --requirement /tmp/requirements-dev.txt & \
-pip install -e git+https://github.com/microsoft/msticpy
 COPY . /tmp/
+RUN pip install -e "git+https://github.com/microsoft/msticpy#egg=msticpy[test]"
 
 # Activate ipywidgets extension in the environment that runs the notebook server
 RUN jupyter nbextension enable --py widgetsnbextension --sys-prefix && \

.github/workflows/python-package.yml

@@ -46,29 +46,19 @@ jobs:
           # This path is specific to Ubuntu
           path: ~/.cache/pip
           # Look to see if there is a cache hit for the corresponding requirements file
-          key: ${{ runner.os }}-pip-${{ hashFiles('requirements-all.txt') }}
+          key: ${{ runner.os }}-pip-${{ hashFiles('setup.py', 'requirements.txt', 'requirements-dev.txt') }}
           restore-keys: |
-            ${{ runner.os }}-pip-${{ hashFiles('requirements-all.txt') }}
+            ${{ runner.os }}-pip-${{ hashFiles('setup.py', 'requirements.txt', 'requirements-dev.txt') }}
             ${{ runner.os }}-pip
       - name: Install dependencies
+        # Installs core install + all extras ("all") + dev/test dependencies ("dev")
+        # via the combined "test" extra defined in setup.py.
         run: |
           python -m pip install --upgrade pip wheel setuptools
-          if [ -f requirements-all.txt ]; then
-            python -m pip install --use-pep517 -r requirements-all.txt
-          elif [ -f requirements.txt ]; then
-            python -m pip install --use-pep517 -r requirements.txt;
-          fi
-          python -m pip install -e .
+          python -m pip install --use-pep517 -e ".[test]"
       - name: Install test dependencies
         # ToDo - remove pip install xgboost when flaml is fixed
         run: |
-          if [ -f requirements-dev.txt ]; then
-            python -m pip install -r requirements-dev.txt
-          else
-            echo "Missing requirements-dev.txt. Installing minimal requirements for testing."
-            python -m pip install pytest pytest-cov pytest-xdist pytest-check aiohttp nbconvert jupyter_contrib_nbextensions
-            python -m pip install Pygments respx pytest-xdist markdown beautifulsoup4 Pillow async-cache lxml
-          fi
           python -m pip install "pandas>=1.3.0" "pygeohash>=1.2.0"
           python -m pip install "xgboost"
       - name: Prepare test dummy data
@@ -122,9 +112,9 @@ jobs:
           # This path is specific to Ubuntu
           path: ~/.cache/pip
           # Look to see if there is a cache hit for the corresponding requirements file
-          key: ${{ runner.os }}-pip-docs-${{ hashFiles('doc/requirements-all.txt') }}
+          key: ${{ runner.os }}-pip-docs-${{ hashFiles('docs/requirements.txt') }}
           restore-keys: |
-            ${{ runner.os }}-pip-docs-${{ hashFiles('requirements-all.txt') }}
+            ${{ runner.os }}-pip-docs-${{ hashFiles('docs/requirements.txt') }}
             ${{ runner.os }}-pip-docs
             ${{ runner.os }}-pip
       - name: Sphinx Read the Docs build
@@ -159,28 +149,17 @@ jobs:
           # This path is specific to Ubuntu
           path: ~/.cache/pip
           # Look to see if there is a cache hit for the corresponding requirements file
-          key: ${{ runner.os }}-pip-lint-${{ hashFiles('requirements-all.txt') }}
+          key: ${{ runner.os }}-pip-lint-${{ hashFiles('setup.py', 'requirements.txt', 'requirements-dev.txt') }}
           restore-keys: |
-            ${{ runner.os }}-pip-lint-${{ hashFiles('requirements-all.txt') }}
+            ${{ runner.os }}-pip-lint-${{ hashFiles('setup.py', 'requirements.txt', 'requirements-dev.txt') }}
             ${{ runner.os }}-pip-lint
             ${{ runner.os }}-pip
       - name: Install dependencies
+        # Installs core install + all extras ("all") + dev/test dependencies ("dev")
+        # via the combined "test" extra defined in setup.py.
         run: |
           python -m pip install --upgrade pip wheel setuptools
-          if [ -f requirements-all.txt ]; then
-            python -m pip install -r requirements-all.txt
-          elif [ -f requirements.txt ]; then
-            python -m pip install -r requirements.txt;
-          fi
-          python -m pip install -e .
-      - name: Install test dependencies
-        run: |
-          if [ -f requirements-dev.txt ]; then
-            python -m pip install -r requirements-dev.txt
-          else
-            echo "Missing requirements-dev.txt. Installing minimal requirements for testing."
-            python -m pip install bandit mypy types-attrs pyroma
-          fi
+          python -m pip install -e ".[test]"
       - name: ruff
         run: |
           ruff check msticpy --ignore PLW0603

.pre-commit-config.yaml

@@ -33,12 +33,3 @@ repos:
       - id: ruff-format
         args:
         - msticpy
-  - repo: local
-    hooks:
-      - id: check_reqs_all
-        name: check_reqs_all
-        entry: python -m tools.create_reqs_all
-        pass_filenames: False
-        language: python
-        types: [python]
-        additional_dependencies: ['packaging>=24.0', 'setuptools>=42']

MANIFEST.in

@@ -3,6 +3,5 @@ include msticpy/msticpyconfig.yaml
 include msticpy/resources/*
 include requirements.txt
 include requirements-dev.txt
-include requirements-all.txt
 include README.md
 recursive-exclude tests *

azure-pipelines.yml

@@ -68,8 +68,7 @@ stages:
         restoreKeys: python
     - script: |
         python -m pip install --upgrade pip wheel setuptools==56.0.0
-        python -m pip install -r requirements-all.txt
-        python -m pip install -e .
+        python -m pip install -e ".[test]"
       displayName: 'Install package and dependencies'
     - script: |
         mkdir ~/.msticpy
@@ -94,7 +93,6 @@ stages:
         echo Env $MSTICPYCONFIG or %MSTICPYCONFIG%
         echo Build source: $(prSource)
         echo Env $MSTICPY_BUILD_SOURCE or %MSTICPY_BUILD_SOURCE%
-        python -m pip install -r requirements-dev.txt
         python -m pip install "pandas>=1.3.0"
         python -m pip install seaborn
         pytest tests --junitxml=junit/test-$(variables.imageName)-$(variables.python.version)-results.xml -n auto --cov=msticpy --cov-report=xml