Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
34 changes: 32 additions & 2 deletions docs/how-to/dashboards_and_quality_gates.rst
Original file line number Diff line number Diff line change
Expand Up @@ -60,9 +60,39 @@ explicit override in ``docs/conf.py``:
When this setting is provided, the explicit list is used instead of
autodiscovery.

Use ``score_metrics_include_external_needs = True`` only in repositories that
Use ``score_metrics_include_external_needs = True`` in repositories that
intentionally aggregate requirements across module dependencies, such as
integration repositories.
integration repositories, or combine it with the optional feature scope below.

Optional External Feature Scope
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

A module repository can report on feature requirements maintained in another
repository, such as ``score``, without counting unrelated external requirements.
First import that repository's needs data through the existing external-needs
configuration, then configure the metrics in ``docs/conf.py``:
Comment thread
a-zw marked this conversation as resolved.
Resolved

.. code-block:: python

score_metrics_include_external_needs = True
score_metrics_requirement_types = "feat_req,comp_req,aou_req"
score_metrics_external_feature_ids = ["feat__lifecycle"]

Replace the example feature ID with the exact ID referenced by the imported
feature requirements' ``satisfied_by`` links. Multiple feature IDs are supported.
All local requirements of the selected requirement types remain included.
External requirements are included only if they are ``feat_req`` needs with a
``satisfied_by`` link to at least one selected feature. Requirements without
tests are still counted; unrelated external requirements are excluded.
Local tests can link to the imported requirements through ``fully_verifies``
or ``partially_verifies`` as usual. The metrics selection does not remove
imported needs from the documentation or from test-reference resolution.

This feature is opt-in: ``score_metrics_external_feature_ids`` defaults to
``[]``, preserving the existing local-only or full-aggregation behavior.
The scope has no effect while ``score_metrics_include_external_needs`` is
``False``. It does not import external needs automatically or change the
configured requirement types.

Building the Dashboard
----------------------
Expand Down
11 changes: 11 additions & 0 deletions src/extensions/score_metrics/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -71,6 +71,17 @@ def setup(app: Sphinx) -> dict[str, str | bool]:
"Default is False so each repo gates only its own needs."
),
)
app.add_config_value(
"score_metrics_external_feature_ids",
[],
rebuild="env",
types=[list],
description=(
"Optional feature IDs used to select external feat_req requirements "
"by satisfied_by links when external needs are enabled. Local "
"requirements remain included. Empty preserves full aggregation."
),
)
app.setup_extension("score_metamodel")

# Calculates the metrics & sets global var for access
Expand Down
95 changes: 93 additions & 2 deletions src/extensions/score_metrics/tests/test_traceability_metrics.py
Original file line number Diff line number Diff line change
Expand Up @@ -11,15 +11,18 @@
# SPDX-License-Identifier: Apache-2.0
# *******************************************************************************

from types import SimpleNamespace
from typing import Any, cast
from unittest.mock import MagicMock

import pytest
import score_metrics.traceability_metrics as metrics
from score_metamodel import ScoreNeedType
from score_metamodel.tests import need as test_need
from score_pytest.attribute_plugin import add_test_properties
from sphinx_needs.data import NeedsView
from sphinx_needs.need_item import NeedItem
from sphinx.environment import BuildEnvironment
from sphinx_needs.data import NeedsView, SphinxNeedsData
from sphinx_needs.need_item import NeedItem, NeedItemSourceExternal


@add_test_properties(
Expand Down Expand Up @@ -266,3 +269,91 @@ def test_calculate_test_metrics_counts_linked_tests_and_broken_refs() -> None:
assert result["broken_references"] == [
{"testcase": "TC_2", "missing_need": "REQ_404"}
]


@add_test_properties(
partially_verifies=["tool_req__docs_test_linkage_metrics"],
test_type="requirements-based",
derivation_technique="equivalence-classes",
)
@pytest.mark.parametrize(
("include_external", "feature_ids", "feature_total", "component_total"),
[
(False, None, 1, 1),
(True, None, 6, 2),
(True, [], 6, 2),
(False, ["feat__lifecycle"], 1, 1),
(True, ["feat__lifecycle"], 4, 1),
(True, ["feat__lifecycle", "feat__other"], 5, 1),
(True, ["feat__missing"], 1, 1),
],
)
def test_calculate_full_need_metrics_optional_external_feature_scope(
monkeypatch: pytest.MonkeyPatch,
include_external: bool,
feature_ids: list[str] | None,
feature_total: int,
component_total: int,
) -> None:
"""Scope external requirements by feature without hiding untested requirements."""
external_requirements = [
test_need(
id="feat_req__linked",
type="feat_req",
satisfied_by=["feat__lifecycle"],
testlink="TC_1",
),
test_need(
id="feat_req__untested",
type="feat_req",
satisfied_by=["feat__lifecycle"],
),
test_need(
id="feat_req__unrelated",
type="feat_req",
satisfied_by=["feat__other"],
),
test_need(
id="feat_req__shared",
type="feat_req",
satisfied_by=["feat__lifecycle", "feat__other"],
),
test_need(id="feat_req__unassigned", type="feat_req"),
test_need(id="comp_req__external", type="comp_req"),
]
for need in external_requirements:
monkeypatch.setattr(
need,
"_source",
NeedItemSourceExternal(url=f"https://example.org/{need.id}"),
)
needs = [
*external_requirements,
test_need(id="feat_req__local", type="feat_req"),
test_need(id="comp_req__local", type="comp_req"),
test_need(id="TC_1", type="testcase", fully_verifies=["feat_req__linked"]),
]
monkeypatch.setattr(metrics, "CALCULATED_METRICS", {})
app = MagicMock()
app.env = MagicMock(spec=BuildEnvironment)
needs_data = SphinxNeedsData(app.env)
for need in needs:
needs_data.add_need(need)
needs_data.needs_is_post_processed = True
app.config = SimpleNamespace(
score_metamodel_yaml=None,
score_metrics_requirement_types="feat_req,comp_req",
)
if feature_ids is not None:
app.config.score_metrics_external_feature_ids = feature_ids

metrics.calculate_full_need_metrics(app, include_external)

result = metrics.CALCULATED_METRICS
assert result["metrics_by_type"]["feat_req"]["total"] == feature_total
assert result["metrics_by_type"]["comp_req"]["total"] == component_total
assert result["overall_metrics"]["total"] == feature_total + component_total
expected_linked = int(include_external and feature_ids != ["feat__missing"])
assert result["metrics_by_type"]["feat_req"]["with_test_link"] == expected_linked
assert result["tests"]["linked_to_requirements"] == 1
assert result["tests"]["broken_references"] == []
18 changes: 16 additions & 2 deletions src/extensions/score_metrics/traceability_metrics.py
Original file line number Diff line number Diff line change
Expand Up @@ -139,6 +139,9 @@ def calculate_full_need_metrics(app: Sphinx, include_external: bool):
filter_reqs = [t.strip() for t in raw.split(",") if t.strip()]
if not filter_reqs:
filter_reqs = get_need_types_by_tags(metamodel.needs_types, {"requirement"})
external_feature_ids = set(
getattr(app.config, "score_metrics_external_feature_ids", [])
)
# ──────────────────[ Calculate Test Metrics ]──────────────────

test_needs = list(all_needs.filter_types(["testcase"]).values())
Expand All @@ -162,10 +165,21 @@ def calculate_full_need_metrics(app: Sphinx, include_external: bool):
needs_of_req_type = all_needs.filter_types([req_type]).filter_is_external(
False
)
selected_requirements = list(needs_of_req_type.values())
if include_external and external_feature_ids:
selected_requirements = [
need
for need in selected_requirements
if not need.get("is_external")
or (
req_type == "feat_req"
and external_feature_ids.intersection(need.get("satisfied_by", []))
)
]
# We do not care if there is no requirements of this type.
if not list(needs_of_req_type.values()):
if not selected_requirements:
continue
req_metrics = calculate_requirement_metrics(list(needs_of_req_type.values()))
req_metrics = calculate_requirement_metrics(selected_requirements)
overall_metrics["total"] += req_metrics["total"]
overall_metrics["with_code_link"] += req_metrics["with_code_link"]
overall_metrics["with_test_link"] += req_metrics["with_test_link"]
Expand Down
Loading