Skip to content

Commit 17ea7e0

Browse files
andrcunsandrcuns
authored
report global errors on feature exit event (via #1444)
1 parent 4bf7b4b commit 17ea7e0

14 files changed

Lines changed: 362 additions & 0 deletions

‎allure-karate/src/main/java/io/qameta/allure/karate/AllureKarate.java‎

Lines changed: 52 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,8 @@
1616
package io.qameta.allure.karate;
1717

1818
import io.karatelabs.common.Json;
19+
import io.karatelabs.core.FeatureRunEvent;
20+
import io.karatelabs.core.FeatureRuntime;
1921
import io.karatelabs.core.HttpRunEvent;
2022
import io.karatelabs.core.RunEvent;
2123
import io.karatelabs.core.RunListener;
@@ -67,6 +69,7 @@
6769
import java.util.stream.Collectors;
6870
import java.util.stream.Stream;
6971

72+
import static io.qameta.allure.util.ResultsUtils.createGlobalError;
7073
import static io.qameta.allure.util.ResultsUtils.createLabel;
7174
import static io.qameta.allure.util.ResultsUtils.createLink;
7275
import static io.qameta.allure.util.ResultsUtils.createParameter;
@@ -94,6 +97,7 @@ public class AllureKarate implements RunListener {
9497

9598
private final AllureLifecycle lifecycle;
9699

100+
private final Map<FeatureRuntime, Map<ScenarioResult, ReportedScenario>> reportedScenarios = new ConcurrentHashMap<>();
97101
private final Map<ScenarioRuntime, ScenarioContext> scenarioContexts = new ConcurrentHashMap<>();
98102
private final Map<ScenarioRuntime, AllureExternalKey> activeStepKeys = new ConcurrentHashMap<>();
99103
private final Set<AllureExternalKey> redactedStepKeys = ConcurrentHashMap.newKeySet();
@@ -121,6 +125,9 @@ public AllureKarate(final AllureLifecycle lifecycle) {
121125
@Override
122126
public boolean onEvent(final RunEvent event) {
123127
switch (event.getType()) {
128+
case FEATURE_EXIT:
129+
afterFeature((FeatureRunEvent) event);
130+
return true;
124131
case SCENARIO_ENTER:
125132
return beforeScenario(((ScenarioRunEvent) event).source());
126133
case SCENARIO_EXIT:
@@ -294,6 +301,13 @@ private void afterScenario(final ScenarioRunEvent event) {
294301
return;
295302
}
296303

304+
// Shared cleanup hooks run after SCENARIO_EXIT, so remember which steps are already reported.
305+
maybeResult.ifPresent(
306+
result -> reportedScenarios
307+
.computeIfAbsent(sr.getFeatureRuntime(), key -> new ConcurrentHashMap<>())
308+
.put(result, new ReportedScenario(result.getStepResults().size(), redactFailure))
309+
);
310+
297311
final List<Parameter> list = new ArrayList<>();
298312
if (!context.reportDisabled()
299313
&& event.result() != null
@@ -315,6 +329,41 @@ private void afterScenario(final ScenarioRunEvent event) {
315329
lifecycle.writeTest(testKey);
316330
}
317331

332+
private void afterFeature(final FeatureRunEvent event) {
333+
final Map<ScenarioResult, ReportedScenario> reported = reportedScenarios.remove(event.source());
334+
if (event.source().isCalled() || Objects.isNull(event.result())) {
335+
return;
336+
}
337+
338+
for (ScenarioResult result : event.result().getScenarioResults()) {
339+
final ReportedScenario snapshot = Objects.isNull(reported) ? null : reported.get(result);
340+
reportFeatureErrors(result, snapshot);
341+
}
342+
}
343+
344+
private void reportFeatureErrors(final ScenarioResult result, final ReportedScenario snapshot) {
345+
// Preparation errors may occur before any scenario emits lifecycle events.
346+
final int reportedStepCount = Objects.isNull(snapshot) ? 0 : snapshot.stepCount();
347+
final boolean redact = result.isReportDisabled()
348+
|| getTagTexts(result.getScenario()).contains("report=false")
349+
|| Objects.nonNull(snapshot) && snapshot.redactFailure();
350+
final String context = "Karate feature " + getFeatureNameQualified(result.getScenario().getFeature())
351+
+ " failed";
352+
final List<StepResult> steps = result.getStepResults();
353+
for (int i = reportedStepCount; i < steps.size(); i++) {
354+
final StepResult step = steps.get(i);
355+
// beforeScenario and afterScenario hooks belong to scenario results, not globals.
356+
if (step.isFailed() && !step.isHook()) {
357+
lifecycle.writeGlobals(
358+
createGlobalError(
359+
redact ? context + ": " + ScenarioResult.SUPPRESSED_FAILURE_MESSAGE : context,
360+
redact ? null : step.getError()
361+
)
362+
);
363+
}
364+
}
365+
}
366+
318367
private void finishCalledScenario(final ScenarioContext context,
319368
final boolean failed,
320369
final boolean redactFailure,
@@ -696,6 +745,9 @@ private List<Link> getLinks(final List<String> labels) {
696745
return allureLinks;
697746
}
698747

748+
private record ReportedScenario(int stepCount, boolean redactFailure) {
749+
}
750+
699751
private record ScenarioContext(
700752
String testUuid,
701753
AllureExternalKey ownerKey,

‎allure-karate/src/test/java/io/qameta/allure/karate/AllureKarateAdvancedTest.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -93,6 +93,7 @@ void shouldRedactReportDisabledCalledFailureFromCaller() {
9393
.containsExactly(ScenarioResult.SUPPRESSED_FAILURE_MESSAGE, null);
9494
assertThat(call.getSteps()).isEmpty();
9595
assertThat(results.getAttachments()).isEmpty();
96+
assertThat(results.getGlobals()).isEmpty();
9697
}
9798

9899
/**

‎allure-karate/src/test/java/io/qameta/allure/karate/AllureKarateGlobalErrorsTest.java‎

Lines changed: 222 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,222 @@
1+
/*
2+
* Copyright 2016-2026 Qameta Software Inc
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
package io.qameta.allure.karate;
17+
18+
import io.karatelabs.core.ScenarioResult;
19+
import io.qameta.allure.Description;
20+
import io.qameta.allure.model.GlobalError;
21+
import io.qameta.allure.model.TestResult;
22+
import io.qameta.allure.test.AllureResults;
23+
import org.junit.jupiter.api.Test;
24+
25+
import java.util.List;
26+
27+
import static io.qameta.allure.model.Status.BROKEN;
28+
import static io.qameta.allure.model.Status.FAILED;
29+
import static org.assertj.core.api.Assertions.assertThat;
30+
31+
@SuppressWarnings({"MultipleStringLiterals", "PMD.AvoidDuplicateLiterals"})
32+
class AllureKarateGlobalErrorsTest extends TestRunner {
33+
34+
/**
35+
* A feature cleanup failure produces a global error identifying the feature, exception, and occurrence time.
36+
*/
37+
@Test
38+
@Description
39+
void shouldReportAfterFeatureFailureAsGlobalError() {
40+
final long started = System.currentTimeMillis();
41+
final AllureResults results = run("classpath:testdata/global-errors/after-feature-failure.feature");
42+
43+
final List<GlobalError> errors = globalErrors(results);
44+
assertThat(errors).hasSize(1);
45+
final GlobalError error = errors.get(0);
46+
assertThat(error.getMessage()).contains("after-feature-failure.feature", "feature cleanup failed");
47+
assertThat(error.getTrace()).contains("feature cleanup failed");
48+
assertThat(error.getTimestamp()).isBetween(started, System.currentTimeMillis());
49+
}
50+
51+
/**
52+
* A scenario failure does not prevent the feature's cleanup failure from being reported as a global error.
53+
*/
54+
@Test
55+
@Description
56+
void shouldReportFeatureCleanupAfterScenarioFailure() {
57+
final AllureResults results = run("classpath:testdata/global-errors/after-failed-scenario.feature");
58+
59+
final List<GlobalError> errors = globalErrors(results);
60+
assertThat(errors).hasSize(1);
61+
final GlobalError error = errors.get(0);
62+
assertThat(error.getMessage()).contains("after-failed-scenario.feature", "cleanup after failure");
63+
assertThat(error.getTrace()).contains("cleanup after failure");
64+
}
65+
66+
/**
67+
* Concurrently executed features each report their own cleanup error once, with the correct feature identity.
68+
*/
69+
@Test
70+
@Description
71+
void shouldKeepParallelFeatureErrorsIsolated() {
72+
final AllureResults results = run(
73+
4,
74+
"classpath:testdata/global-errors/after-feature-failure.feature",
75+
"classpath:testdata/global-errors/after-failed-scenario.feature"
76+
);
77+
78+
final List<GlobalError> errors = globalErrors(results);
79+
assertThat(errors).hasSize(2);
80+
assertThat(errors).anySatisfy(
81+
error -> assertThat(error.getMessage()).contains("after-feature-failure.feature", "feature cleanup failed")
82+
);
83+
assertThat(errors).anySatisfy(
84+
error -> assertThat(error.getMessage()).contains("after-failed-scenario.feature", "cleanup after failure")
85+
);
86+
}
87+
88+
/**
89+
* Scenario-outline cleanup and feature cleanup failures are both reported as global errors.
90+
*/
91+
@Test
92+
@Description
93+
void shouldReportEachSharedHookFailure() {
94+
final AllureResults results = run("classpath:testdata/global-errors/shared-hooks-failure.feature");
95+
96+
final List<GlobalError> errors = globalErrors(results);
97+
assertThat(errors).hasSize(2);
98+
assertThat(errors).allSatisfy(error -> {
99+
assertThat(error.getMessage()).contains("shared-hooks-failure.feature");
100+
assertThat(error.getTimestamp()).isPositive();
101+
});
102+
assertThat(errors).anySatisfy(error -> {
103+
assertThat(error.getMessage()).contains("outline cleanup failed");
104+
assertThat(error.getTrace()).contains("outline cleanup failed");
105+
});
106+
assertThat(errors).anySatisfy(error -> {
107+
assertThat(error.getMessage()).contains("feature cleanup failed");
108+
assertThat(error.getTrace()).contains("feature cleanup failed");
109+
});
110+
}
111+
112+
/**
113+
* Feature cleanup errors honor report suppression: safe feature identity remains, but exception details are hidden.
114+
*/
115+
@Test
116+
@Description
117+
void shouldRedactSuppressedFeatureCleanupFailure() {
118+
final AllureResults results = run("classpath:testdata/global-errors/suppressed-cleanup-failure.feature");
119+
120+
final List<GlobalError> errors = globalErrors(results);
121+
assertThat(errors).hasSize(1);
122+
final GlobalError error = errors.get(0);
123+
assertThat(error.getMessage())
124+
.contains("suppressed-cleanup-failure.feature", ScenarioResult.SUPPRESSED_FAILURE_MESSAGE)
125+
.doesNotContain("private-cleanup-value");
126+
assertThat(error.getTrace()).isNull();
127+
assertThat(error.getActual()).isNull();
128+
assertThat(error.getExpected()).isNull();
129+
assertThat(error.getTimestamp()).isPositive();
130+
}
131+
132+
/**
133+
* A visible caller with a suppressed callee failure keeps its global cleanup error details redacted.
134+
*/
135+
@Test
136+
@Description
137+
void shouldRedactFeatureCleanupAfterSuppressedCalledFailure() {
138+
final AllureResults results = run("classpath:testdata/global-errors/suppressed-called-cleanup-failure.feature");
139+
140+
final List<GlobalError> errors = globalErrors(results);
141+
assertThat(errors).hasSize(1);
142+
final GlobalError error = errors.get(0);
143+
assertThat(error.getMessage())
144+
.contains("suppressed-called-cleanup-failure.feature", ScenarioResult.SUPPRESSED_FAILURE_MESSAGE)
145+
.doesNotContain("private-caller-cleanup-value");
146+
assertThat(error.getTrace()).isNull();
147+
assertThat(error.getActual()).isNull();
148+
assertThat(error.getExpected()).isNull();
149+
}
150+
151+
/**
152+
* Example preparation failures produce a global error with the feature identity and original exception details.
153+
*/
154+
@Test
155+
@Description
156+
void shouldReportFeaturePreparationFailureAsGlobalError() {
157+
final AllureResults results = run("classpath:testdata/global-errors/example-preparation-failure.feature");
158+
159+
final List<GlobalError> errors = globalErrors(results);
160+
assertThat(errors).hasSize(1);
161+
final GlobalError error = errors.get(0);
162+
assertThat(error.getMessage()).contains("example-preparation-failure.feature", "example preparation failed");
163+
assertThat(error.getTrace()).contains("example preparation failed");
164+
assertThat(error.getTimestamp()).isPositive();
165+
}
166+
167+
/**
168+
* Report suppression also protects errors raised while preparing examples before scenario execution begins.
169+
*/
170+
@Test
171+
@Description
172+
void shouldRedactSuppressedFeaturePreparationFailure() {
173+
final AllureResults results = run("classpath:testdata/global-errors/suppressed-preparation-failure.feature");
174+
175+
final List<GlobalError> errors = globalErrors(results);
176+
assertThat(errors).hasSize(1);
177+
final GlobalError error = errors.get(0);
178+
assertThat(error.getMessage())
179+
.contains("suppressed-preparation-failure.feature", ScenarioResult.SUPPRESSED_FAILURE_MESSAGE)
180+
.doesNotContain("private-preparation-value");
181+
assertThat(error.getTrace()).isNull();
182+
assertThat(error.getActual()).isNull();
183+
assertThat(error.getExpected()).isNull();
184+
}
185+
186+
/**
187+
* Failures in callonce, callSingle, and setup scenarios remain scenario failures, not global errors.
188+
*/
189+
@Test
190+
@Description
191+
void shouldKeepSharedSetupFailuresOnScenarioResults() {
192+
final AllureResults results = run("classpath:testdata/global-errors/shared-setup-failures.feature");
193+
194+
final TestResult callonce = results.getTestResultByName("Callonce setup failure");
195+
final TestResult callSingle = results.getTestResultByName("CallSingle setup failure");
196+
final TestResult setup = results.getTestResultByName("Prepare shared data");
197+
assertThat(List.of(callonce, callSingle, setup)).allSatisfy(result -> {
198+
assertThat(result.getStatus()).isIn(FAILED, BROKEN);
199+
assertThat(result.getStatusDetails().getMessage()).contains("shared setup failed");
200+
});
201+
assertThat(results.getGlobals()).isEmpty();
202+
}
203+
204+
/**
205+
* A feature cleanup error is global even when scenario cleanup also fails; per-scenario hooks are not global errors.
206+
*/
207+
@Test
208+
@Description
209+
void shouldReportOnlyFeatureHookFailuresAsGlobals() {
210+
final AllureResults results = run("classpath:testdata/global-errors/scenario-and-feature-hooks-failure.feature");
211+
212+
final List<GlobalError> errors = globalErrors(results);
213+
assertThat(errors).hasSize(1);
214+
final GlobalError error = errors.get(0);
215+
assertThat(error.getMessage()).contains("scenario-and-feature-hooks-failure.feature", "feature cleanup failed");
216+
assertThat(error.getTrace()).contains("feature cleanup failed");
217+
}
218+
219+
private List<GlobalError> globalErrors(final AllureResults results) {
220+
return results.getGlobals().stream().flatMap(globals -> globals.getErrors().stream()).toList();
221+
}
222+
}

‎allure-karate/src/test/java/io/qameta/allure/karate/AllureKarateTest.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -134,6 +134,7 @@ void shouldCreateStatusDetailsIfTestFailed() {
134134
"java.lang.AssertionError: expected status: 200, actual: 401"
135135
)
136136
);
137+
assertThat(results.getGlobals()).isEmpty();
137138
}
138139

139140
@Test

‎allure-karate/src/test/resources/testdata/global-errors/after-failed-scenario.feature‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
Feature: Cleanup after a failed scenario
2+
3+
Background:
4+
* configure afterFeature = function(){ throw new Error('cleanup after failure') }
5+
6+
Scenario: Failing scenario
7+
* match 1 == 2

‎allure-karate/src/test/resources/testdata/global-errors/after-feature-failure.feature‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
Feature: Feature cleanup failure
2+
3+
Background:
4+
* configure afterFeature = function(){ throw new Error('feature cleanup failed') }
5+
6+
Scenario: First scenario
7+
* match 1 == 1
8+
9+
Scenario: Second scenario
10+
* match 2 == 2

‎allure-karate/src/test/resources/testdata/global-errors/example-preparation-failure.feature‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
Feature: Example preparation failure
2+
3+
Scenario Outline: Prepared example
4+
* match value == 1
5+
6+
Examples:
7+
| (function(){ throw new Error('example preparation failed') })() |

‎allure-karate/src/test/resources/testdata/global-errors/scenario-and-feature-hooks-failure.feature‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
Feature: Scenario and feature cleanup failures
2+
3+
Background:
4+
* configure afterScenario = function(){ throw new Error('scenario cleanup failed') }
5+
* configure afterFeature = function(){ throw new Error('feature cleanup failed') }
6+
7+
Scenario: Scenario with both cleanup hooks
8+
* match 1 == 1

‎allure-karate/src/test/resources/testdata/global-errors/setup-failure-target.feature‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
Feature: Failing shared setup
2+
3+
Scenario: Shared setup action
4+
* karate.fail('shared setup failed')

‎allure-karate/src/test/resources/testdata/global-errors/shared-hooks-failure.feature‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
Feature: Shared cleanup hooks
2+
3+
Background:
4+
* configure afterScenarioOutline = function(){ throw new Error('outline cleanup failed') }
5+
* configure afterFeature = function(){ throw new Error('feature cleanup failed') }
6+
7+
Scenario Outline: Example <value>
8+
* match value == '#number'
9+
10+
Examples:
11+
| value |
12+
| 1 |
13+
| 2 |

0 commit comments

Comments
 (0)