chore: ci workflows v3 (#2238)

chore(ci): update CI workflow

- separate environment for functional tests requiring external API secrets
- updated Makefile and workflow
- updated test markers in packages to enable functional testing

Author: mattbit

.github/workflows/ci.yml

@@ -1,9 +1,8 @@
-name: Full CI
+name: CI
+
 on:
   push:
-    branches:
-      - main
-      - dev
+    branches: [main]
   pull_request:
   workflow_dispatch:
 
@@ -12,89 +11,57 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
-  detect-changes:
+  lint:
     runs-on: ubuntu-latest
-    outputs:
-      giskard-core: ${{ steps.filter.outputs.giskard-core }}
-      giskard-agents: ${{ steps.filter.outputs.giskard-agents }}
-      giskard-checks: ${{ steps.filter.outputs.giskard-checks }}
+    timeout-minutes: 10
     steps:
-      - name: Checkout code
-        uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6
-
-      - name: Detect changed libraries
-        uses: dorny/paths-filter@v3
-        id: filter
+      - uses: actions/checkout@v4
+      - uses: astral-sh/setup-uv@v7
         with:
-          filters: |
-            giskard-core:
-              - 'libs/giskard-core/**'
-            giskard-agents:
-              - 'libs/giskard-agents/**'
-            giskard-checks:
-              - 'libs/giskard-checks/**'
+          enable-cache: true
+          python-version: "3.12"
+      - run: make install install-tools
+      - run: make check
 
-  pre_checks:
+  test-unit:
+    needs: lint
     runs-on: ubuntu-latest
-    timeout-minutes: 30
+    timeout-minutes: 15
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ["3.12", "3.13"]
+        package: [giskard-core, giskard-agents, giskard-checks]
+    name: test-unit / ${{ matrix.package }} / ${{ matrix.python-version }}
     steps:
-      - name: Checkout code
-        uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6
-
-      - name: Install uv
-        uses: astral-sh/setup-uv@681c641aba71e4a1c380be3ab5e12ad51f415867 # v7
-
-      - name: Setup development environment
-        run: make setup
-
-      - name: Run all checks
-        run: make check
+      - uses: actions/checkout@v4
+      - uses: astral-sh/setup-uv@v7
+        with:
+          enable-cache: true
+          python-version: ${{ matrix.python-version }}
+      - run: make install
+      - run: make test-unit PACKAGE=${{ matrix.package }}
 
-  test:
+  test-functional:
+    needs: lint
     runs-on: ubuntu-latest
-    needs: [detect-changes]
-    if: |
-      github.ref == 'refs/heads/main' ||
-      needs.detect-changes.outputs.giskard-core == 'true' ||
-      needs.detect-changes.outputs.giskard-agents == 'true' ||
-      needs.detect-changes.outputs.giskard-checks == 'true'
-    env:
-      TEST_MODEL: "gemini/gemini-2.0-flash"
-      GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }}
+    environment: CI
     timeout-minutes: 30
     strategy:
-      fail-fast: true
+      fail-fast: false
       matrix:
         python-version: ["3.12", "3.13"]
+        package: [giskard-core, giskard-agents, giskard-checks]
+    name: test-functional / ${{ matrix.package }} / ${{ matrix.python-version }}
+    env:
+      GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }}
+      OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
+      TEST_MODEL: "gemini/gemini-2.0-flash"
     steps:
-      - name: Checkout code
-        uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6
-
-      - name: Install uv and set the python version
-        uses: astral-sh/setup-uv@681c641aba71e4a1c380be3ab5e12ad51f415867 # v7
+      - uses: actions/checkout@v4
+      - uses: astral-sh/setup-uv@v7
         with:
+          enable-cache: true
           python-version: ${{ matrix.python-version }}
-
-      - name: Setup development environment
-        run: make setup
-
-      - name: Run tests for giskard-core
-        if: |
-          github.ref == 'refs/heads/main' ||
-          needs.detect-changes.outputs.giskard-core == 'true'
-        run: make test-giskard-core
-
-      - name: Run tests for giskard-agents
-        if: |
-          github.ref == 'refs/heads/main' ||
-          needs.detect-changes.outputs.giskard-core == 'true' ||
-          needs.detect-changes.outputs.giskard-agents == 'true'
-        run: make test-giskard-agents
-
-      - name: Run tests for giskard-checks
-        if: |
-          github.ref == 'refs/heads/main' ||
-          needs.detect-changes.outputs.giskard-core == 'true' ||
-          needs.detect-changes.outputs.giskard-agents == 'true' ||
-          needs.detect-changes.outputs.giskard-checks == 'true'
-        run: make test-giskard-checks
+      - run: make install
+      - run: make test-functional PACKAGE=${{ matrix.package }}

Makefile

@@ -1,17 +1,16 @@
 # Variables
 LIBS := giskard-core giskard-agents giskard-checks
-TEST_TARGETS := $(addprefix test-,$(LIBS))
+PACKAGE ?= # Optional package to test (e.g., giskard-core, giskard-agents, giskard-checks)
 
 # Default target
 help: ## Show this help message
 	@echo "Available targets:"
 	@grep -E '^[a-zA-Z0-9_%-]+:.*?## .*$$' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "  \033[36m%-20s\033[0m %s\n", $$1, $$2}'
+
 # Installation targets
 install: ## Install project dependencies
 	uv sync
 
-sync: install ## Alias for install
-
 install-tools: ## Install development tools
 	uv tool install ruff
 	uv tool install vermin
@@ -23,13 +22,26 @@ pre-commit-install: ## Setup pre-commit hooks
 
 setup: install install-tools pre-commit-install ## Complete development setup (install deps + tools)
 
-test: $(TEST_TARGETS) ## Run all tests
-
-$(TEST_TARGETS): test-%: ## Run tests for a specific library (e.g., make test-giskard-core)
-	@echo "------------------------------------------------"
-	@echo "Running tests for $*"
-	@echo "------------------------------------------------"
-	uv run pytest libs/$*
+test: ## Run all tests (unit + functional), optional PACKAGE=<name>
+ifdef PACKAGE
+	uv run pytest libs/$(PACKAGE)
+else
+	uv run pytest libs/
+endif
+
+test-unit: ## Run unit tests only (excludes functional), optional PACKAGE=<name>
+ifdef PACKAGE
+	uv run pytest libs/$(PACKAGE) -m "not functional"
+else
+	$(foreach lib,$(LIBS),uv run pytest libs/$(lib) -m "not functional" &&) true
+endif
+
+test-functional: ## Run functional tests only (requires API keys), optional PACKAGE=<name>
+ifdef PACKAGE
+	uv run pytest libs/$(PACKAGE) -m "functional"
+else
+	$(foreach lib,$(LIBS),uv run pytest libs/$(lib) -m "functional" &&) true
+endif
 
 test-package-conflict: ## Test package conflict with giskard legacy package installed
 	@echo "Testing package conflict..."
@@ -67,7 +79,7 @@ typecheck: ## Run type checking with basedpyright
 	uv tool run basedpyright --level error .
 
 security: ## Check for security vulnerabilities
-	uv run pip-audit
+	uv run pip-audit --skip-editable
 
 generate-licenses: ## Generate licenses
 	uv tool run licensecheck --license MIT \
@@ -79,13 +91,7 @@ check-licenses: ## Check for licenses
 		--show-only-failing --zero \
 		--skip-dependencies giskard-agents giskard-checks giskard-core
 
-# Combined targets
-check: lint check-format check-compat typecheck security check-licenses ## Run all checks (lint, format, compatibility, typecheck)
-
-all: format check test ## Format, check, and test
-
-# CI simulation
-ci: check test ## Run the same checks as CI
+check: lint check-format check-compat typecheck security check-licenses ## Run all checks
 
 clean: ## Clean up build artifacts and caches
 	find . -type d -name "__pycache__" -exec rm -rf {} + 2>/dev/null || true

libs/giskard-agents/pyproject.toml

@@ -33,6 +33,9 @@ build-backend = "hatchling.build"
 [tool.pytest.ini_options]
 asyncio_mode = "auto"
 testpaths = ["tests"]
+markers = [
+    "functional: marks tests that depend on external APIs (e.g., Gemini, OpenAI)",
+]
 
 [tool.ruff.lint.isort.sections]
 "my_packages" = ["giskard"]

libs/giskard-agents/tests/test_embedding_model.py

@@ -49,6 +49,7 @@ async def test_litellm_embedding_model_embed_with_mock(
         assert np.isclose(embeddings[1], np.array([0.4, 0.5, 0.6])).all()
 
 
+@pytest.mark.functional
 async def test_embedding_model_real_embedding(
     embedding_model: LitellmEmbeddingModel,
 ) -> None:

libs/giskard-agents/tests/test_generator.py

@@ -39,6 +39,7 @@ async def test_litellm_generator_completion_with_mock(
         assert response.finish_reason == "stop"
 
 
+@pytest.mark.functional
 async def test_generator_completion(generator: LiteLLMGenerator):
     response = await generator.complete(
         messages=[
@@ -57,6 +58,7 @@ async def test_generator_completion(generator: LiteLLMGenerator):
     assert response.finish_reason == "stop"
 
 
+@pytest.mark.functional
 async def test_generator_chat(generator: LiteLLMGenerator):
     test_message = "Hello, world!"
     pipeline = generator.chat(test_message)

libs/giskard-agents/tests/test_tools.py

@@ -80,6 +80,7 @@ async def get_weather(self, city: str) -> str:
     assert await weather.get_weather(city="Tokyo") == "It's sunny in Tokyo."
 
 
+@pytest.mark.functional
 async def test_tool_run(generator):
     """Test that the tool runs correctly."""
 

libs/giskard-agents/tests/test_workflow.py

@@ -1,12 +1,14 @@
 from pathlib import Path
 
+import pytest
 from giskard import agents
 from giskard.agents.chat import Chat
 from giskard.agents.generators.litellm_generator import LiteLLMGenerator
 from giskard.agents.templates.prompts_manager import PromptsManager
 from pydantic import BaseModel
 
 
+@pytest.mark.functional
 async def test_single_run(generator):
     workflow = agents.ChatWorkflow(generator=generator)
 
@@ -20,6 +22,7 @@ async def test_single_run(generator):
     assert "testbot" in chat.last.content.lower()
 
 
+@pytest.mark.functional
 async def test_run_many(generator):
     """Test that the workflow runs correctly."""
 
@@ -30,6 +33,7 @@ async def test_run_many(generator):
     assert len(chats) == 3
 
 
+@pytest.mark.functional
 async def test_run_batch(generator):
     """Test that the workflow runs correctly."""
 
@@ -50,6 +54,7 @@ async def test_run_batch(generator):
     assert len(chats) == 3
 
 
+@pytest.mark.functional
 async def test_stream_many(generator):
     workflow = agents.ChatWorkflow(generator=generator).chat("Hello!", role="user")
 
@@ -61,6 +66,7 @@ async def test_stream_many(generator):
     assert len(chats) == 3
 
 
+@pytest.mark.functional
 async def test_stream_batch(generator):
     workflow = agents.ChatWorkflow(generator=generator).chat("Hello!", role="user")
 
@@ -77,6 +83,7 @@ async def test_stream_batch(generator):
     assert len(chats) == 2
 
 
+@pytest.mark.functional
 async def test_workflow_with_mixed_templates(generator: LiteLLMGenerator):
     workflow = agents.ChatWorkflow(
         generator=generator,
@@ -123,6 +130,7 @@ async def test_workflow_with_mixed_templates(generator: LiteLLMGenerator):
     assert chat.messages[4].role == "assistant"
 
 
+@pytest.mark.functional
 async def test_output_format(generator):
     workflow = agents.ChatWorkflow(generator=generator)
 

libs/giskard-checks/tests/conftest.py

@@ -24,3 +24,10 @@ def pytest_collection_modifyitems(
     for item in items:
         if "integration" in item.keywords:
             item.add_marker(skip_integration)
+
+
+def pytest_sessionfinish(session, exitstatus):
+    # If no tests were collected, set the exit status to 0 to avoid failure.
+    # This is a workaround for packages not having any functional tests.
+    if exitstatus == 5:
+        session.exitstatus = 0

libs/giskard-core/tests/conftest.py

@@ -0,0 +1,5 @@
+def pytest_sessionfinish(session, exitstatus):
+    # If no tests were collected, set the exit status to 0 to avoid failure.
+    # This is a workaround for packages not having any functional tests.
+    if exitstatus == 5:
+        session.exitstatus = 0

pyproject.toml

@@ -46,6 +46,11 @@ giskard-core = { workspace = true }
 giskard-agents = { workspace = true }
 giskard-checks = { workspace = true }
 
+[tool.pytest.ini_options]
+markers = [
+    "functional: tests that require external API calls (deselect with '-m \"not functional\"')",
+]
+
 [dependency-groups]
 dev = [
     "ipykernel==7.1.0",