Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
### Added
- **Model Zoo Benchmark**: Introduced a comprehensive scientific validation notebook (`examples/trustlens_model_zoo_benchmark.ipynb`) that systematically evaluates TrustLens across 6 model architectures and multiple data corruption scenarios with statistical aggregation.
- **Deployment Recommendation Explanations**: Added `TrustReport.deployment_explanation` and `TrustReport.deployment_summary` to provide structured deployment verdict explanations, identify primary risks, and surface actionable recommendations based on Trust Score penalties and sub-scores.
- **Deployment Recommendation UX**: Deployment recommendations are now surfaced directly in `TrustReport.show()`, text exports, and HTML report views. Users now receive deployment verdicts, primary risk identification, and actionable recommendations without needing to access `deployment_summary` manually.

### Fixed
- Fixed incorrect `top_mistake_indices` in `misclassification_summary()` to return **global dataset indices** instead of local filtered subset positions, improving downstream EDA and debugging workflows for high-confidence model errors. (PR #104) Thanks @dicnunz 🙌
Expand Down
135 changes: 108 additions & 27 deletions examples/trustlens_demo.ipynb

Large diffs are not rendered by default.

54 changes: 54 additions & 0 deletions tests/test_deployment_explanation.py
Original file line number Diff line number Diff line change
Expand Up @@ -139,3 +139,57 @@ def test_serialization_includes_explanation(mock_report):
assert "deployment_verdict" in flat_dict
assert flat_dict["deployment_verdict"] == "PASS"
assert "deployment_primary_risk_metric" in flat_dict


def test_show_contains_deployment_summary(mock_report, capsys):
mock_report.trust_score = TrustScoreResult(
score=75,
grade="C",
verdict="Moderate Trust",
sub_scores={"calibration": 90.0, "bias": 60.0},
penalties_applied={"Fairness": 15.0},
is_blocked=False,
)

mock_report.show()
captured = capsys.readouterr()
assert "Deployment Verdict: CAUTION" in captured.out
assert "Primary Risk:" in captured.out
assert "Investigate subgroup performance disparities" in captured.out


def test_deployment_summary_in_text_report(mock_report):
mock_report.trust_score = TrustScoreResult(
score=75,
grade="C",
verdict="Moderate Trust",
sub_scores={"calibration": 90.0, "bias": 60.0},
penalties_applied={"Fairness": 15.0},
is_blocked=False,
)

text = mock_report._generate_text_report()
assert "Deployment Verdict: CAUTION" in text
assert "Primary Risk:" in text
assert "Investigate subgroup performance disparities" in text


def test_deployment_summary_in_html_report(mock_report):
mock_report.trust_score = TrustScoreResult(
score=75,
grade="C",
verdict="Moderate Trust",
sub_scores={"calibration": 90.0, "bias": 60.0},
penalties_applied={"Fairness": 15.0},
is_blocked=False,
)

import matplotlib.pyplot as plt

fig, ax = plt.subplots()
mock_report.summary_plot = lambda show=False: fig

html = mock_report._repr_html_()
assert "Deployment Verdict:" in html
assert "Fairness" in html
assert "Investigate subgroup performance disparities" in html
45 changes: 45 additions & 0 deletions trustlens/report.py
Original file line number Diff line number Diff line change
Expand Up @@ -297,6 +297,8 @@ def show(self, verbose: bool = False) -> None:
for line in explanation:
print(line)

print(f"\n{self.deployment_summary}")

# Print Key Observations/Insights
print("\nKey Observations:")
insights = self._generate_insights()
Expand Down Expand Up @@ -356,6 +358,9 @@ def _generate_text_report(self, verbose: bool = False) -> str:
lines.append("")
lines.extend(explanation)

lines.append("")
lines.extend(self.deployment_summary.split("\n"))

lines.append("\nKey Observations:")
insights = self._generate_insights()
if not insights:
Expand Down Expand Up @@ -1501,6 +1506,46 @@ def _repr_html_(self) -> str:
<div style="font-size: 15px; font-weight: 600; color: {_C["dark"]}; margin-bottom: 5px;">Overall Assessment</div>
<div style="font-size: 14px; color: #444;">{ts.verdict}</div>
</div>
"""

exp = self.deployment_explanation
fail_icon = "<span style='color: #d32f2f; font-weight: bold;'>✗</span>"
pass_icon = "<span style='color: #2e7d32; font-weight: bold;'>✓</span>"

reasons_html = "".join(
[
f'<li style="margin-bottom: 4px;">{fail_icon if r["status"] == "fail" else pass_icon} {r["message"]}</li>'
for r in exp["reasons"]
]
)
pr_html = (
f'<div style="font-size: 13px; font-weight: 700; color: {_C["gray"]}; margin-top: 15px; margin-bottom: 4px; text-transform: uppercase;">Primary Risk</div><div style="font-size: 14px; color: #d32f2f; font-weight: 600;">{exp["primary_risk"]["metric"]}</div>'
if exp["primary_risk"]
else ""
)
recs_html = "".join(
[f'<li style="margin-bottom: 4px;">{r}</li>' for r in exp["recommendations"]]
)

html += f"""
<div style="background-color: #ffffff; border: 1px solid #e0e0e0; border-radius: 12px; padding: 15px; margin-bottom: 25px;">
<div style="font-size: 15px; font-weight: 600; color: {_C["dark"]}; margin-bottom: 8px;">Deployment Verdict: <span style="font-weight: 700;">{exp["verdict"]}</span></div>
{pr_html}
<div style="display: flex; gap: 30px; margin-top: 15px; flex-wrap: wrap;">
<div style="flex: 1; min-width: 200px;">
<div style="font-size: 13px; font-weight: 700; color: {_C["gray"]}; margin-bottom: 8px; text-transform: uppercase;">Reasons</div>
<ul style="margin: 0; padding-left: 0; list-style-type: none; font-size: 13.5px; color: #333; line-height: 1.5;">
{reasons_html}
</ul>
</div>
<div style="flex: 1; min-width: 250px;">
<div style="font-size: 13px; font-weight: 700; color: {_C["gray"]}; margin-bottom: 8px; text-transform: uppercase;">Recommendations</div>
<ul style="margin: 0; padding-left: 20px; font-size: 13.5px; color: #333; line-height: 1.5;">
{recs_html}
</ul>
</div>
</div>
</div>
Comment thread
Khanz9664 marked this conversation as resolved.

<div style="margin-bottom: 20px;">
<img src="data:image/png;base64,{data}" style="width: 100%; border-radius: 8px; border: 1px solid #f0f0f0;" />
Expand Down
Loading