feat(telemetry): measure scan and hub export

libs/giskard-checks/README.md

@@ -20,7 +20,7 @@ pip install giskard-checks
 
 Requires Python >= 3.12.
 
-**Telemetry:** This package depends on `giskard-core`, which may send **optional, aggregated usage analytics** when you run scenarios, suites, or test cases (no prompts, outputs, or scenario text). See **[Telemetry](../giskard-core/README.md#telemetry)** in the `giskard-core` README for what is collected and how to opt out (`DO_NOT_TRACK`, `GISKARD_TELEMETRY_DISABLED`, `GISKARD_TELEMETRY_DISABLE_GEOIP`, or `disable_telemetry()`).
+**Telemetry:** This package depends on `giskard-core`, which may send **optional, aggregated usage analytics** when you run scenarios, suites, or test cases, or export a result to the Hub format (no prompts, outputs, or scenario text). See **[Telemetry](../giskard-core/README.md#telemetry)** in the `giskard-core` README for what is collected and how to opt out (`DO_NOT_TRACK`, `GISKARD_TELEMETRY_DISABLED`, `GISKARD_TELEMETRY_DISABLE_GEOIP`, or `disable_telemetry()`).
 
 **Dependencies:**
 - `pydantic>=2.12` - Core data validation and serialization

libs/giskard-checks/src/giskard/checks/export/hub.py

@@ -2,9 +2,12 @@
 
 from typing import Any
 
+from giskard.core import scoped_telemetry, telemetry_capture, telemetry_tag
+
 from ..core.result import SuiteResult
 
 
+@scoped_telemetry
 def to_hub_format(result: SuiteResult) -> dict[str, Any]:
     """Convert a SuiteResult into a JSON-serializable Giskard Hub payload.
 
@@ -20,4 +23,19 @@ def to_hub_format(result: SuiteResult) -> dict[str, Any]:
     dict[str, Any]
         JSON-serializable representation of the suite result
     """
-    return result.model_dump(mode="json", fallback=str)
+    telemetry_tag("giskard_component", "export")
+    telemetry_tag("giskard_operation", "to_hub_format")
+    payload = result.model_dump(mode="json", fallback=str)
+    telemetry_capture(
+        "checks_hub_exported",
+        properties={
+            "integration": "giskard-checks",
+            "scenario_count": len(result.results),
+            "passed_count": result.passed_count,
+            "failed_count": result.failed_count,
+            "errored_count": result.errored_count,
+            "skipped_count": result.skipped_count,
+            "has_recommendation": bool(result.recommendation),
+        },
+    )
+    return payload

libs/giskard-checks/tests/export/test_hub.py

@@ -1,6 +1,7 @@
 """Tests for Hub format export."""
 
 from giskard.checks.core.result import SuiteResult
+from giskard.checks.export import hub as hub_module
 from giskard.checks.export.hub import to_hub_format
 
 
@@ -13,3 +14,55 @@ def test_to_hub_format_pass_rate_null_when_empty() -> None:
     payload = to_hub_format(SuiteResult(results=[], duration_ms=0))
     assert "pass_rate" in payload
     assert payload["pass_rate"] is None
+
+
+def test_to_hub_format_emits_aggregate_paid_intent_telemetry(monkeypatch) -> None:
+    events: list[tuple[str, dict[str, object]]] = []
+    tags: list[tuple[str, str]] = []
+    call_order: list[str] = []
+    original_model_dump = SuiteResult.model_dump
+
+    def model_dump_spy(self, *args, **kwargs):
+        call_order.append("model_dump")
+        return original_model_dump(self, *args, **kwargs)
+
+    monkeypatch.setattr(SuiteResult, "model_dump", model_dump_spy)
+    monkeypatch.setattr(
+        hub_module,
+        "telemetry_capture",
+        lambda event, *, properties: (
+            call_order.append("telemetry_capture"),
+            events.append((event, properties)),
+        ),
+    )
+    monkeypatch.setattr(
+        hub_module,
+        "telemetry_tag",
+        lambda key, value: tags.append((key, value)),
+    )
+
+    result = SuiteResult(
+        results=[], duration_ms=123, recommendation="private recommendation"
+    )
+    to_hub_format(result)
+
+    assert call_order == ["model_dump", "telemetry_capture"]
+    assert tags == [
+        ("giskard_component", "export"),
+        ("giskard_operation", "to_hub_format"),
+    ]
+    assert events == [
+        (
+            "checks_hub_exported",
+            {
+                "integration": "giskard-checks",
+                "scenario_count": 0,
+                "passed_count": 0,
+                "failed_count": 0,
+                "errored_count": 0,
+                "skipped_count": 0,
+                "has_recommendation": True,
+            },
+        )
+    ]
+    assert "private recommendation" not in repr(events)

libs/giskard-core/README.md

@@ -21,7 +21,7 @@ pip install giskard-core
 
 ### What we collect
 
-Installed versions of `giskard-core`, `giskard-checks`, and `giskard-agents` (each the package version or `not_installed`), a coarse **environment** label (`ci`, `colab`, `kaggle`, or `local`), and when you run **giskard-checks** flows, aggregated non-content metrics such as step counts, counts of checks by `kind`, booleans (e.g. custom trace type or target present), durations, and pass/fail/skip-style outcomes. **Scenario names, prompts, model outputs, trace content, and exception messages are not sent.**
+Installed versions of Giskard libraries (each the package version or `not_installed`), a coarse **environment** label (`ci`, `colab`, `kaggle`, or `local`), and aggregated non-content metrics for Checks runs, Scan runs, and Hub-format exports. These metrics include counts, counts of checks by `kind` (including custom kind identifiers you register), fixed feature modes, booleans, durations, and pass/fail/error/skip-style outcomes. **Descriptions, language values, scenario names, prompts, model outputs, recommendations, trace content, file paths, and exception messages are not sent.**
 
 If an error propagates through the telemetry context, a single event may record **`exception_type`** (the Python class name only), not the exception string or traceback.
 

libs/giskard-scan/README.md

@@ -2,6 +2,12 @@
 
 Agent vulnerability scanner — red teaming, prompt injection, adversarial scenario generation.
 
+**Telemetry:** This package depends on `giskard-core`, which may send optional,
+aggregated scan lifecycle metrics (scan type, fixed execution modes, counts,
+durations, and outcomes). Descriptions, language values, generated scenarios,
+prompts, outputs, recommendations, and other user content are never included.
+See [Telemetry](../giskard-core/README.md#telemetry) for details and opt-outs.
+
 ## Scan entrypoints
 
 `quality_scan` and `vulnerability_scan` share the same explicit execution options.

libs/giskard-scan/src/giskard/scan/_telemetry.py

@@ -0,0 +1,32 @@
+"""Privacy-boundary helpers for scan analytics."""
+
+from typing import Any
+
+
+def safe_target_mode(value: Any) -> str:
+    """Return only a fixed target-mode label."""
+    return (
+        value
+        if type(value) is str and value in ("singleturn", "multiturn")
+        else "unknown"
+    )
+
+
+def safe_bool(value: Any) -> bool | None:
+    """Return booleans without coercing arbitrary caller-owned objects."""
+    return value if type(value) is bool else None
+
+
+def scenario_budget(value: Any) -> str:
+    """Bucket a requested scenario limit so it cannot carry identifying numbers."""
+    if value is None:
+        return "default"
+    if type(value) is not int or value < 0:
+        return "unknown"
+    if value == 0:
+        return "none"
+    if value <= 10:
+        return "small"
+    if value <= 100:
+        return "medium"
+    return "large"

libs/giskard-scan/src/giskard/scan/quality.py

@@ -4,7 +4,9 @@
 import warnings
 
 from giskard.checks import SuiteResult, Target, Trace
+from giskard.core import scoped_telemetry, telemetry_capture, telemetry_tag
 
+from ._telemetry import safe_bool, safe_target_mode, scenario_budget
 from .catalog import generate_suite
 from .generators.base import DEFAULT_TARGET_MODE, TargetMode
 from .generators.knowledge_base import (
@@ -35,6 +37,7 @@
     quality_suite_generator_registry.register(generator_type)
 
 
+@scoped_telemetry
 async def quality_scan[InputType, OutputType, TraceType: Trace](  # pyright: ignore[reportMissingTypeArgument]
     target: Target[InputType, OutputType, TraceType],
     description: str,
@@ -111,29 +114,61 @@ async def quality_scan[InputType, OutputType, TraceType: Trace](  # pyright: ign
     knowledge_base = normalize_knowledge_base(
         _warn_if_missing_knowledge_base(knowledge_base)
     )
+    telemetry_tag("giskard_component", "scan")
+    telemetry_tag("giskard_operation", "quality_scan")
+    telemetry_properties: dict[str, object] = {
+        "integration": "giskard-scan",
+        "scan_type": "quality",
+        "target_mode": safe_target_mode(target_mode),
+        "language_count": len(languages),
+        "scenario_budget": scenario_budget(opts["max_scenarios"]),
+        "parallel": safe_bool(opts["parallel"]),
+        "has_knowledge_base": knowledge_base is not None,
+    }
+    telemetry_capture("scan_run_started", properties=telemetry_properties)
 
-    suite = await generate_suite(
-        description=description,
-        languages=languages,
-        generators=quality_suite_generator_registry.generators(),
-        max_scenarios=opts["max_scenarios"],
-        seed=opts["seed"],
-        target_mode=target_mode,
-        knowledge_base=knowledge_base,
-    )
+    try:
+        suite = await generate_suite(
+            description=description,
+            languages=languages,
+            generators=quality_suite_generator_registry.generators(),
+            max_scenarios=opts["max_scenarios"],
+            seed=opts["seed"],
+            target_mode=target_mode,
+            knowledge_base=knowledge_base,
+        )
 
-    result: SuiteResult = await suite.run(
-        target,
-        parallel=opts["parallel"],
-        max_concurrency=opts["max_concurrency"],
-        return_exception=opts["return_exception"],
-    )
+        result: SuiteResult = await suite.run(
+            target,
+            parallel=opts["parallel"],
+            max_concurrency=opts["max_concurrency"],
+            return_exception=opts["return_exception"],
+        )
+    except Exception:
+        telemetry_capture(
+            "scan_run_finished",
+            properties={**telemetry_properties, "outcome": "error"},
+        )
+        raise
     try:
         recommendation = await generate_quality_recommendation(result)
     except Exception:
         logger.exception("Quality recommendation generation failed")
         recommendation = ""
     quality_result = result.model_copy(update={"recommendation": recommendation})
+    telemetry_capture(
+        "scan_run_finished",
+        properties={
+            **telemetry_properties,
+            "outcome": "completed",
+            "duration_ms": quality_result.duration_ms,
+            "scenario_count": len(quality_result.results),
+            "passed_count": quality_result.passed_count,
+            "failed_count": quality_result.failed_count,
+            "errored_count": quality_result.errored_count,
+            "skipped_count": quality_result.skipped_count,
+        },
+    )
     quality_result.print_report(group_by=opts["group_by"])
     return quality_result
 

libs/giskard-scan/src/giskard/scan/vulnerability.py

@@ -1,5 +1,7 @@
 from giskard.checks import SuiteResult, Target, Trace
+from giskard.core import scoped_telemetry, telemetry_capture, telemetry_tag
 
+from ._telemetry import safe_bool, safe_target_mode, scenario_budget
 from .catalog import generate_suite
 from .generators.adversarial import AdversarialScenarioGenerator
 from .generators.base import DEFAULT_TARGET_MODE, TargetMode
@@ -32,6 +34,7 @@
     vulnerability_suite_generator_registry.register(generator)
 
 
+@scoped_telemetry
 async def vulnerability_scan[InputType, OutputType, TraceType: Trace](  # pyright: ignore[reportMissingTypeArgument]
     target: Target[InputType, OutputType, TraceType],
     description: str,
@@ -104,23 +107,55 @@ async def vulnerability_scan[InputType, OutputType, TraceType: Trace](  # pyrigh
         return_exception=return_exception,
         commercial_use=commercial_use,
     )
+    telemetry_tag("giskard_component", "scan")
+    telemetry_tag("giskard_operation", "vulnerability_scan")
+    telemetry_properties: dict[str, object] = {
+        "integration": "giskard-scan",
+        "scan_type": "vulnerability",
+        "target_mode": safe_target_mode(target_mode),
+        "language_count": len(languages),
+        "scenario_budget": scenario_budget(opts["max_scenarios"]),
+        "parallel": safe_bool(opts["parallel"]),
+        "commercial_use": safe_bool(opts["commercial_use"]),
+    }
+    telemetry_capture("scan_run_started", properties=telemetry_properties)
 
-    suite = await generate_suite(
-        description=description,
-        languages=languages,
-        generators=vulnerability_suite_generator_registry.generators(
-            commercial_use=opts["commercial_use"]
-        ),
-        max_scenarios=opts["max_scenarios"],
-        seed=opts["seed"],
-        target_mode=target_mode,
-    )
+    try:
+        suite = await generate_suite(
+            description=description,
+            languages=languages,
+            generators=vulnerability_suite_generator_registry.generators(
+                commercial_use=opts["commercial_use"]
+            ),
+            max_scenarios=opts["max_scenarios"],
+            seed=opts["seed"],
+            target_mode=target_mode,
+        )
 
-    result = await suite.run(
-        target,
-        parallel=opts["parallel"],
-        max_concurrency=opts["max_concurrency"],
-        return_exception=opts["return_exception"],
+        result = await suite.run(
+            target,
+            parallel=opts["parallel"],
+            max_concurrency=opts["max_concurrency"],
+            return_exception=opts["return_exception"],
+        )
+    except Exception:
+        telemetry_capture(
+            "scan_run_finished",
+            properties={**telemetry_properties, "outcome": "error"},
+        )
+        raise
+    telemetry_capture(
+        "scan_run_finished",
+        properties={
+            **telemetry_properties,
+            "outcome": "completed",
+            "duration_ms": result.duration_ms,
+            "scenario_count": len(result.results),
+            "passed_count": result.passed_count,
+            "failed_count": result.failed_count,
+            "errored_count": result.errored_count,
+            "skipped_count": result.skipped_count,
+        },
     )
     result.print_report(group_by=opts["group_by"])
     return result