From a7411690713f446755db73580caa9d71f1f94041 Mon Sep 17 00:00:00 2001 From: Shubhangi-Nikam Date: Tue, 6 Oct 2026 13:27:23 +0530 Subject: [PATCH] [rules score] add test_specifications() for TestSpec LOBSTER traceability Introduces a test_specifications() macro so TRLC TestSpec records can feed LOBSTER traceability, tracing up to Feature Requirements via TestSpec.verifies, and wires python test-case lobster output into dependable_element()'s tests attribute. - providers.bzl: add TestSpecificationsInfo - requirements.bzl: add req_kind = "test_spec" - private/test_specifications.bzl: new public macro - rules_score.bzl: export test_specifications - private/dependable_element.bzl: - test_specifications attribute + Test Specifications LOBSTER block (intentionally no trace-to Feature Requirements, so elements can adopt it incrementally) - Test Cases LOBSTER block sourced from `tests` targets that expose an OutputGroupInfo "lobster" output group - fix multi-page RST report source-root depth (off by one directory, broke every Source: link) - lobster/config/lobster_de.conf.tpl: add block placeholders - tools/lobster_rst_report/_renderers.py: - render TestSpec items as separate Attributes/Description sections instead of one pull-quote blob - label TestSpec's upward trace "Verifies:" instead of "Derived from:" --- .../lobster/config/lobster_de.conf.tpl | 4 + .../private/dependable_element.bzl | 59 +++++++++- .../rules_score/private/requirements.bzl | 10 +- .../private/test_specifications.bzl | 107 ++++++++++++++++++ bazel/rules/rules_score/providers.bzl | 8 ++ bazel/rules/rules_score/rules_score.bzl | 5 + tools/lobster_rst_report/_renderers.py | 85 ++++++++++++-- 7 files changed, 264 insertions(+), 14 deletions(-) create mode 100644 bazel/rules/rules_score/private/test_specifications.bzl diff --git a/bazel/rules/rules_score/lobster/config/lobster_de.conf.tpl b/bazel/rules/rules_score/lobster/config/lobster_de.conf.tpl index 14cedbd6..94e66c93 100644 --- a/bazel/rules/rules_score/lobster/config/lobster_de.conf.tpl +++ b/bazel/rules/rules_score/lobster/config/lobster_de.conf.tpl @@ -1,5 +1,9 @@ {FEAT_REQ_BLOCK} +{TEST_SPEC_BLOCK} + +{TEST_CASE_BLOCK} + {RECEIVED_AOU_BLOCK} {FORWARDED_AOU_BLOCK} diff --git a/bazel/rules/rules_score/private/dependable_element.bzl b/bazel/rules/rules_score/private/dependable_element.bzl index 0d131002..eacdff85 100644 --- a/bazel/rules/rules_score/private/dependable_element.bzl +++ b/bazel/rules/rules_score/private/dependable_element.bzl @@ -44,6 +44,7 @@ load( "SphinxModuleInfo", "SphinxNeedsInfo", "SphinxSourcesInfo", + "TestSpecificationsInfo", "UnitInfo", ) load( @@ -1405,6 +1406,26 @@ def _dependable_element_index_impl(ctx): feat_req_lobster_depset = depset(transitive = feat_req_lobster_files) + # Collect test specification .lobster files from test_specifications + # targets (TestSpecificationsInfo). These trace up to Feature Requirements + # via the TestSpec.verifies field. + test_spec_lobster_files = [] + for req_target in ctx.attr.test_specifications: + if TestSpecificationsInfo in req_target: + test_spec_lobster_files.append(req_target[TestSpecificationsInfo].srcs) + + test_spec_lobster_depset = depset(transitive = test_spec_lobster_files) + + # Collect python test-case .lobster files from `tests` targets exposing an + # OutputGroupInfo "lobster" output group (py_lobster_trace()). These trace + # up to Test Specifications via the pytest verifies-mark. + py_test_lobster_files = [] + for test_target in ctx.attr.tests: + if OutputGroupInfo in test_target and hasattr(test_target[OutputGroupInfo], "lobster"): + py_test_lobster_files.append(test_target[OutputGroupInfo].lobster) + + py_test_lobster_depset = depset(transitive = py_test_lobster_files) + comp_req_lobster_files = [] comp_test_lobster_files = [] comp_arch_lobster_files = [] @@ -1505,6 +1526,8 @@ def _dependable_element_index_impl(ctx): # Build the DE-level lobster report if feature and component traces exist feat_req_list = feat_req_lobster_depset.to_list() + test_spec_list = test_spec_lobster_depset.to_list() + py_test_list = py_test_lobster_depset.to_list() comp_req_list = comp_req_lobster_depset.to_list() comp_test_list = comp_test_lobster_depset.to_list() @@ -1579,6 +1602,8 @@ def _dependable_element_index_impl(ctx): # reference" issues for levels a given dependable element does not use. has_any_lobster_input = any([ feat_req_list, + test_spec_list, + py_test_list, received_aou_list, comp_req_list, comp_test_list, @@ -1637,6 +1662,22 @@ def _dependable_element_index_impl(ctx): output = lobster_config, substitutions = { "{FEAT_REQ_BLOCK}": format_lobster_block("requirements", "Feature Requirements", feat_req_list), + # Test Specifications intentionally does not `trace to:` Feature + # Requirements: doing so would require every Feature Requirement + # to be covered by a test specification as soon as a single one + # exists. Each TestSpec.verifies reference is instead resolved + # individually against Feature Requirements via the normal + # per-item up-reference check. + "{TEST_SPEC_BLOCK}": format_lobster_block( + "requirements", + "Test Specifications", + test_spec_list, + ), + "{TEST_CASE_BLOCK}": format_lobster_block( + "activity", + "Test Cases", + py_test_list, + ), # Target level: every item here must be covered, either by a # Component Requirement (`derived_from`) or by being # further chain-forwarded (Forwarded AoUs, below). Without the @@ -1736,7 +1777,7 @@ def _dependable_element_index_impl(ctx): }, ) - all_lobster_inputs = feat_req_list + comp_req_list + comp_arch_list + comp_test_list + interface_req_list + fm_list + rc_list + safetymeasures_list + received_aou_list + forwarded_aou_markers_list + coverage_lobster_files + all_lobster_inputs = feat_req_list + test_spec_list + py_test_list + comp_req_list + comp_arch_list + comp_test_list + interface_req_list + fm_list + rc_list + safetymeasures_list + received_aou_list + forwarded_aou_markers_list + coverage_lobster_files lobster_report_file = subrule_lobster_report(all_lobster_inputs, lobster_config) lobster_files = [lobster_config, lobster_report_file] @@ -1758,7 +1799,11 @@ def _dependable_element_index_impl(ctx): if not source_root: package = ctx.label.package package_depth = len(package.split("/")) if package else 0 - source_root = "/".join([".." for _ in range(package_depth + 2)]) + "/" + # +3: escape "traceability_report/", "html/" and "_doc/" + # (the multi-page report nests item pages one level deeper than + # "_doc/html/", inside "_doc/html/traceability_report/"). + # +1: escape the "bazel-bin" convenience symlink itself. + source_root = "/".join([".." for _ in range(package_depth + 4)]) + "/" rst_args = ctx.actions.args() rst_args.add(lobster_report_file.path) rst_args.add_all(["--out-dir", lobster_rst_dir.path]) @@ -1852,6 +1897,11 @@ def _dependable_element_index_attrs(): providers = [[FeatureRequirementsInfo], [AssumedSystemRequirementsInfo]], doc = "Feature or assumed system requirements targets.", ), + "test_specifications": attr.label_list( + default = [], + providers = [[TestSpecificationsInfo]], + doc = "Test specification targets tracing to feature requirements via TestSpec.verifies, used for LOBSTER traceability.", + ), "architectural_design": attr.label_list( mandatory = True, doc = "Architectural design targets or files.", @@ -2096,6 +2146,7 @@ def dependable_element( checklists = [], glossary = [], deps = [], + test_specifications = [], aou_forwarding = None, maturity = "release", generate_html_report = False, @@ -2136,6 +2187,9 @@ def dependable_element( terminology and definitions for the dependable element. deps: Optional list of other module targets this element depends on. Cross-references will work automatically. + test_specifications: Optional list of labels to test_specifications + targets tracing to feature requirements via TestSpec.verifies, + used for LOBSTER traceability. aou_forwarding: Optional label to a YAML file listing received AoU IDs to further-forward to this element's own dependees. Only needed for chain-forwarding received AoUs that this element cannot handle. @@ -2193,6 +2247,7 @@ def dependable_element( glossary = glossary, tests = tests, deps = deps, + test_specifications = test_specifications, processed_deps = processed_deps, aou_forwarding = aou_forwarding, integrity_level = integrity_level, diff --git a/bazel/rules/rules_score/private/requirements.bzl b/bazel/rules/rules_score/private/requirements.bzl index 7e4240ed..46cc1870 100644 --- a/bazel/rules/rules_score/private/requirements.bzl +++ b/bazel/rules/rules_score/private/requirements.bzl @@ -21,7 +21,7 @@ public-facing macros. load("@lobster//:lobster.bzl", "subrule_lobster_trlc") load("@trlc//:trlc.bzl", "TrlcProviderInfo", "subrule_trlc_image_stage") -load("//bazel/rules/rules_score:providers.bzl", "AssumedSystemRequirementsInfo", "AssumptionsOfUseInfo", "ComponentRequirementsInfo", "FeatureRequirementsInfo", "SafetyAnalysisProviderInfo", "SphinxSourcesInfo") +load("//bazel/rules/rules_score:providers.bzl", "AssumedSystemRequirementsInfo", "AssumptionsOfUseInfo", "ComponentRequirementsInfo", "FeatureRequirementsInfo", "SafetyAnalysisProviderInfo", "SphinxSourcesInfo", "TestSpecificationsInfo") load("//bazel/rules/rules_score/private:rst_to_trlc.bzl", "rst_to_trlc") _DEFAULT_SPEC = Label("//bazel/rules/rules_score/trlc/config:score_requirements_model") @@ -34,6 +34,7 @@ _REQ_KIND_TITLES = { "component": "Component Requirements", "assumed_system": "Assumed System Requirements", "aou": "Assumptions of Use", + "test_spec": "Test Specifications", } # ============================================================================ @@ -117,6 +118,11 @@ def _requirements_impl(ctx): aou_lobster = depset([lobster_file]), name = ctx.label.name, ) + elif ctx.attr.req_kind == "test_spec": + req_provider = TestSpecificationsInfo( + srcs = depset([lobster_file]), + name = ctx.label.name, + ) else: # assumed_system req_provider = AssumedSystemRequirementsInfo( srcs = depset([lobster_file]), @@ -173,7 +179,7 @@ _score_requirements_rule = rule( doc = "Other requirement targets whose TRLC records are needed for cross-reference parsing.", ), "req_kind": attr.string( - values = ["feature", "component", "assumed_system", "aou"], + values = ["feature", "component", "assumed_system", "aou", "test_spec"], mandatory = True, doc = "Kind of requirements; determines which domain provider is emitted.", ), diff --git a/bazel/rules/rules_score/private/test_specifications.bzl b/bazel/rules/rules_score/private/test_specifications.bzl new file mode 100644 index 00000000..4c00f5fd --- /dev/null +++ b/bazel/rules/rules_score/private/test_specifications.bzl @@ -0,0 +1,107 @@ +# ******************************************************************************* +# Copyright (c) 2026 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) distributed with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made available under the +# terms of the Apache License Version 2.0 which is available at +# https://www.apache.org/licenses/LICENSE-2.0 +# +# SPDX-License-Identifier: Apache-2.0 +# ******************************************************************************* + +""" +Test Specification build rules for S-CORE projects. + +Test specifications describe verification test cases (preconditions, test +steps, expected results) that trace to feature requirements via the +TestSpec.verifies field, enabling LOBSTER traceability from requirements +down to test design. +""" + +load("@trlc//:trlc.bzl", "trlc_requirements_test") +load("//bazel/rules/rules_score/private:requirements.bzl", "score_requirements_rule") + +# ============================================================================ +# Public Macro +# ============================================================================ + +def test_specifications( + name, + srcs, + lobster_config, + deps = [], + spec = Label("//bazel/rules/rules_score/trlc/config:score_requirements_model"), + ref_package = "", + image_srcs = [], + **kwargs): + """Define test specifications following S-CORE process guidelines. + + Creates a target providing TestSpecificationsInfo, TrlcProviderInfo, and + SphinxSourcesInfo, plus a validation test target ``_test``. + + Because this target emits TrlcProviderInfo, it can reference + feature_requirements targets directly in its ``deps`` without any + intermediate trlc_requirements wrapper. + + Args: + name: The name of the target. + srcs: List of .trlc source files containing TestSpec records as + defined in the requirements model in use (the base S-CORE + requirements model does not define TestSpec; it is provided by + extension models such as SweAsCodeReq). + lobster_config: Lobster extraction config label. There is no default + because the TestSpec record type is not part of the base S-CORE + requirements model (e.g. use + ``@swe_as_code//tools/trlc/config:lobster_test_specification``). + deps: Optional list of requirement targets (e.g. feature_requirements) + whose TRLC records are needed for cross-reference parsing since + TestSpec.verifies references FeatReqId. These targets must + provide TrlcProviderInfo. + spec: Optional TRLC specification target providing RSL type definitions. + Defaults to the S-CORE requirements model + (``@score_tooling//bazel/rules/rules_score/trlc/config:score_requirements_model``). + Override this when using a custom requirements model that defines TestSpec. + ref_package: TRLC package prefix used for verifies cross-references + when converting RST sources. + image_srcs: Image and diagram files (.svg, .png, or .puml) to stage + next to the rendered RST. + visibility: Bazel visibility specification for the generated targets. + + Generated Targets: + : Main target providing TestSpecificationsInfo, TrlcProviderInfo, + and SphinxSourcesInfo. + _test: TRLC validation test (runs ``trlc --verify``). + + Example: + ```starlark + feature_requirements( + name = "feat_req", + srcs = ["feature_requirements.trlc"], + ) + + test_specifications( + name = "test_spec", + srcs = ["test_specifications.trlc"], + deps = [":feat_req"], + lobster_config = "@swe_as_code//tools/trlc/config:lobster_test_specification", + ) + ``` + """ + score_requirements_rule( + name = name, + srcs = srcs, + deps = deps, + req_kind = "test_spec", + lobster_config = lobster_config, + spec = spec, + ref_package = ref_package, + image_srcs = image_srcs, + **kwargs + ) + trlc_requirements_test( + name = name + "_test", + reqs = [":" + name], + **kwargs + ) diff --git a/bazel/rules/rules_score/providers.bzl b/bazel/rules/rules_score/providers.bzl index 196f299b..2f877afb 100644 --- a/bazel/rules/rules_score/providers.bzl +++ b/bazel/rules/rules_score/providers.bzl @@ -105,6 +105,14 @@ ComponentRequirementsInfo = provider( }, ) +TestSpecificationsInfo = provider( + doc = "Provider for test specification artifacts.", + fields = { + "srcs": "Depset of .lobster traceability files generated from TRLC test specification sources.", + "name": "Name of the test specifications target.", + }, +) + AssumedSystemRequirementsInfo = provider( doc = "Provider for assumed system requirements artifacts.", fields = { diff --git a/bazel/rules/rules_score/rules_score.bzl b/bazel/rules/rules_score/rules_score.bzl index 85b0ae42..c3ee892d 100644 --- a/bazel/rules/rules_score/rules_score.bzl +++ b/bazel/rules/rules_score/rules_score.bzl @@ -65,6 +65,10 @@ load( "//bazel/rules/rules_score/private:sphinx_module.bzl", _sphinx_module = "sphinx_module", ) +load( + "//bazel/rules/rules_score/private:test_specifications.bzl", + _test_specifications = "test_specifications", +) load( "//bazel/rules/rules_score/private:unit.bzl", _unit = "unit", @@ -84,6 +88,7 @@ glossary = _glossary filter_execpath = _filter_execpath safety_analysis = _safety_analysis sphinx_module = _sphinx_module +test_specifications = _test_specifications unit = _unit unit_design = _unit_design component = _component diff --git a/tools/lobster_rst_report/_renderers.py b/tools/lobster_rst_report/_renderers.py index 6973c8eb..50c6e11c 100644 --- a/tools/lobster_rst_report/_renderers.py +++ b/tools/lobster_rst_report/_renderers.py @@ -101,6 +101,65 @@ def __init__(self, item: Item, report: Report, source_root: str = ""): self._report = report self._source_root = source_root + def _render_test_spec_text(self, item, body, sep, has_content): + """Render a TestSpec item's text as separate Attributes/Description sections. + + TestSpec.text is a flattened "field: value" blob (one paragraph per + field, joined by blank lines). Short scalar fields (priority, status, + etc.) are listed under "Attributes", one per line. The narrative + fields (preconditions, test_steps, ...) are listed under + "Description", each under its own sub-heading. RST line blocks + ("| ...") keep every line (e.g. each numbered step) on its own line + instead of being merged into a single wrapped paragraph. + + Returns: + The updated ``has_content`` flag. + """ + e = RstUtils.escape + metadata_fields = ( + "derivation_technique", "security", "test_type", + "testspec_status", "priority", "test_environment", "author", + ) + metadata_lines = [] + narrative_paragraphs = [] + for para in item.text.split("\n\n"): + field, sep_found, _ = para.partition(":") + if sep_found and field.strip() in metadata_fields: + metadata_lines.append(para) + else: + narrative_paragraphs.append(para) + + if metadata_lines: + body("**Attributes:**") + body("") + for line in metadata_lines: + field, _, value = line.partition(":") + body(f"| {e(field.strip())}: {e(value.strip())}") + body("") + has_content = True + + if narrative_paragraphs: + if has_content: + sep() + body("**Description:**") + body("") + for para in narrative_paragraphs: + lines = [line.strip() for line in para.splitlines()] + if not lines: + continue + field, sep_found, first_value = lines[0].partition(":") + if sep_found: + body(f"**{e(field.strip())}:**") + body("") + lines = [first_value.strip()] + lines[1:] + for text_line in lines: + if text_line: + body(f"| {e(text_line)}") + body("") + has_content = True + + return has_content + def build(self) -> list: """Return RST lines for the item dropdown. @@ -171,15 +230,18 @@ def issue_box(msgs): # Description / specification text if isinstance(item, (Requirement, Activity)) and item.text: - body(".. pull-quote::") - out.append("") - for text_line in item.text.splitlines(): - # Requirement text originates from external sources — escape RST - # special chars. Activity text is author-controlled and may - # contain intentional RST markup — render as-is. - nested(e(text_line) if isinstance(item, Requirement) else text_line) - out.append("") - has_content = True + if getattr(item, "kind", None) == "TestSpec": + has_content = self._render_test_spec_text(item, body, sep, has_content) + else: + body(".. pull-quote::") + out.append("") + for text_line in item.text.splitlines(): + # Requirement text originates from external sources — escape + # RST special chars. Activity text is author-controlled and + # may contain intentional RST markup — render as-is. + nested(e(text_line) if isinstance(item, Requirement) else text_line) + out.append("") + has_content = True # lobster-trace: UseCases.List_Requirements_to_Tests # lobster-trace: UseCases.List_Tests_to_Requirements @@ -205,7 +267,10 @@ def issue_box(msgs): if has_up: if has_content: sep() - body("**Derived from:**") + # TestSpec.verifies traces up to Feature Requirements; every other + # requirement kind keeps the generic "Derived from" wording. + up_label = "Verifies" if getattr(item, "kind", None) == "TestSpec" else "Derived from" + body(f"**{up_label}:**") out.append("") for ref_str in self._resolve_refs(item.ref_up): body(f"* {ref_str}")