Skip to content

Commit 37bebdd

Browse files
andrcunsandrcuns
authored
enrich global errors with failed or broken status (via #1454)
1 parent 8c5aad6 commit 37bebdd

33 files changed

Lines changed: 1002 additions & 23 deletions

File tree

‎allure-citrus/src/test/java/io/qameta/allure/citrus/AllureCitrusTest.java‎

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -46,11 +46,15 @@
4646
import io.qameta.allure.test.IsolatedLifecycle;
4747
import io.qameta.allure.test.RunUtils;
4848
import org.junit.jupiter.api.Test;
49+
import org.junit.jupiter.params.ParameterizedTest;
50+
import org.junit.jupiter.params.provider.Arguments;
51+
import org.junit.jupiter.params.provider.MethodSource;
4952
import org.opentest4j.AssertionFailedError;
5053

5154
import java.time.Instant;
5255
import java.util.List;
5356
import java.util.function.Consumer;
57+
import java.util.stream.Stream;
5458

5559
import static io.qameta.allure.test.AllureTestCommonsUtils.attach;
5660
import static io.qameta.allure.util.ResultsUtils.md5;
@@ -411,6 +415,7 @@ void shouldReportSuiteFailuresWithoutThrowable() {
411415
"Citrus suite teardown failed"
412416
);
413417
assertThat(errors).allSatisfy(error -> {
418+
assertThat(error.getStatus()).isEqualTo(Status.BROKEN);
414419
assertThat(error.getTrace()).isNull();
415420
assertThat(error.getTimestamp()).isPositive();
416421
});
@@ -435,10 +440,38 @@ void shouldReportSuiteFailureComparisonDetails() {
435440
final GlobalError error = errors.get(0);
436441
assertThat(error.getMessage()).contains("Citrus suite teardown failed", "suite comparison failed");
437442
assertThat(error.getTrace()).contains("org.opentest4j.AssertionFailedError: suite comparison failed");
443+
assertThat(error.getStatus()).isEqualTo(Status.FAILED);
438444
assertThat(error.getExpected()).isEqualTo(cause.getExpected().toString());
439445
assertThat(error.getActual()).isEqualTo(cause.getActual().toString());
440446
}
441447

448+
@ParameterizedTest
449+
@MethodSource("suiteGlobalStatuses")
450+
void shouldClassifyBothSuiteFailureCallbacks(final Throwable throwable, final Status status) {
451+
final AllureResults results = reportSuiteFailures(adapter -> {
452+
adapter.onStartFailure(throwable);
453+
adapter.onFinishFailure(throwable);
454+
});
455+
final List<GlobalError> errors = results.getGlobals().stream().flatMap(globals -> globals.getErrors().stream())
456+
.toList();
457+
assertThat(errors).hasSize(2).allSatisfy(error -> {
458+
assertThat(error.getStatus()).isEqualTo(status);
459+
if (throwable != null) {
460+
assertThat(error.getTrace()).contains(throwable.getClass().getName());
461+
}
462+
});
463+
assertThat(results.getTestResults()).isEmpty();
464+
}
465+
466+
static Stream<Arguments> suiteGlobalStatuses() {
467+
return Stream.of(
468+
Arguments.of(new AssertionError("assertion"), Status.FAILED),
469+
Arguments.of(new AssertionFailedError("comparison", "expected", "actual"), Status.FAILED),
470+
Arguments.of(new IllegalStateException("runtime"), Status.BROKEN),
471+
Arguments.of(null, Status.BROKEN)
472+
);
473+
}
474+
442475
private CitrusContext.Builder suiteContext() {
443476
// Only Allure output is needed; disable Citrus HTML and JUnit report generation.
444477
return new CitrusContext.Builder().testReporters(new TestReporters());

‎allure-cucumber7-jvm/build.gradle.kts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@ dependencies {
1717
testImplementation("io.github.glytching:junit-extensions")
1818
testImplementation("org.assertj:assertj-core")
1919
testImplementation("org.junit.jupiter:junit-jupiter-api")
20+
testImplementation("org.junit.jupiter:junit-jupiter-params")
2021
testImplementation("org.slf4j:slf4j-simple")
2122
testImplementation(project(":allure-assertj"))
2223
testImplementation(project(":allure-java-commons-test"))

‎allure-cucumber7-jvm/src/main/java/io/qameta/allure/cucumber7jvm/AllureCucumber7Jvm.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -246,7 +246,7 @@ private void handleTestRunFinished(final TestRunFinished event) {
246246
final String context = scenarioStarted.get()
247247
? "Cucumber test run failed"
248248
: "Cucumber test run failed, scenarios did not run";
249-
lifecycle.writeGlobals(createGlobalError(context, result.getError()));
249+
lifecycle.writeGlobals(createGlobalError(context, result.getError(), translateTestCaseStatus(result)));
250250
}
251251

252252
private void handleTestCaseFinished(final TestCaseFinished event) {

‎allure-cucumber7-jvm/src/test/java/io/qameta/allure/cucumber7jvm/AllureCucumber7JvmTest.java‎

Lines changed: 62 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,8 @@
2424
import io.cucumber.core.runtime.FeaturePathFeatureSupplier;
2525
import io.cucumber.core.runtime.Runtime;
2626
import io.cucumber.core.runtime.TimeServiceEventBus;
27+
import io.cucumber.plugin.event.Result;
28+
import io.cucumber.plugin.event.TestRunFinished;
2729
import io.github.glytching.junit.extension.system.SystemProperty;
2830
import io.github.glytching.junit.extension.system.SystemPropertyExtension;
2931
import io.qameta.allure.Description;
@@ -47,8 +49,13 @@
4749
import org.junit.jupiter.api.Test;
4850
import org.junit.jupiter.api.extension.ExtendWith;
4951
import org.junit.jupiter.api.parallel.ResourceLock;
52+
import org.junit.jupiter.params.ParameterizedTest;
53+
import org.junit.jupiter.params.provider.Arguments;
54+
import org.junit.jupiter.params.provider.MethodSource;
55+
import org.opentest4j.AssertionFailedError;
5056

5157
import java.time.Clock;
58+
import java.time.Duration;
5259
import java.time.Instant;
5360
import java.util.ArrayList;
5461
import java.util.Arrays;
@@ -57,6 +64,7 @@
5764
import java.util.function.Consumer;
5865
import java.util.function.Supplier;
5966
import java.util.stream.Collectors;
67+
import java.util.stream.Stream;
6068

6169
import static io.qameta.allure.util.ResultsUtils.PACKAGE_LABEL_NAME;
6270
import static io.qameta.allure.util.ResultsUtils.PARENT_SUITE_LABEL_NAME;
@@ -1057,6 +1065,60 @@ private AllureResults runFeature(final String featureResource,
10571065
});
10581066
}
10591067

1068+
@ParameterizedTest
1069+
@MethodSource("runErrorStatuses")
1070+
void shouldClassifyRunErrorsUsingCucumberOutcome(final io.cucumber.plugin.event.Status outcome,
1071+
final Throwable throwable, final Status status) {
1072+
final AllureResults results = runFinished(new Result(outcome, Duration.ZERO, throwable));
1073+
assertThat(getGlobalErrors(results)).singleElement().satisfies(error -> {
1074+
assertThat(error.getStatus()).isEqualTo(status);
1075+
assertThat(error.getTrace()).contains(throwable.getClass().getName());
1076+
if (throwable instanceof AssertionFailedError) {
1077+
assertThat(error.getActual()).startsWith("actual (");
1078+
assertThat(error.getExpected()).startsWith("expected (");
1079+
}
1080+
});
1081+
assertThat(results.getTestResults()).isEmpty();
1082+
}
1083+
1084+
static Stream<Arguments> runErrorStatuses() {
1085+
return Stream.of(
1086+
Arguments.of(
1087+
io.cucumber.plugin.event.Status.FAILED,
1088+
new AssertionFailedError("comparison", "expected", "actual"), Status.FAILED
1089+
),
1090+
Arguments.of(
1091+
io.cucumber.plugin.event.Status.FAILED,
1092+
new IllegalStateException("runtime"), Status.BROKEN
1093+
),
1094+
Arguments.of(io.cucumber.plugin.event.Status.SKIPPED, new AssertionError("skip"), Status.BROKEN),
1095+
Arguments.of(io.cucumber.plugin.event.Status.PENDING, new AssertionError("pending"), Status.BROKEN),
1096+
Arguments.of(io.cucumber.plugin.event.Status.AMBIGUOUS, new AssertionError("ambiguous"), Status.BROKEN),
1097+
Arguments.of(io.cucumber.plugin.event.Status.UNDEFINED, new AssertionError("undefined"), Status.BROKEN),
1098+
Arguments.of(io.cucumber.plugin.event.Status.PASSED, new AssertionError("runtime signal"), Status.BROKEN)
1099+
);
1100+
}
1101+
1102+
@Test
1103+
void shouldNotCreateGlobalWhenRunResultHasNoThrowable() {
1104+
assertThat(runFinished(new Result(io.cucumber.plugin.event.Status.FAILED, Duration.ZERO, null)).getGlobals())
1105+
.isEmpty();
1106+
}
1107+
1108+
@Test
1109+
void shouldNotCreateGlobalWhenRunResultIsAbsent() {
1110+
assertThat(runFinished(null).getGlobals()).isEmpty();
1111+
}
1112+
1113+
private static AllureResults runFinished(final Result result) {
1114+
return RunUtils.runTests(lifecycle -> {
1115+
final EventBus bus = new TimeServiceEventBus(Clock.systemUTC(), UUID::randomUUID);
1116+
final AllureCucumber7Jvm adapter = new AllureCucumber7Jvm(lifecycle);
1117+
adapter.setEventPublisher(bus);
1118+
bus.send(result == null ? new TestRunFinished(Instant.now()) : new TestRunFinished(Instant.now(), result));
1119+
});
1120+
}
1121+
10601122
private static List<GlobalError> getGlobalErrors(final AllureResults results) {
10611123
return results.getGlobals().stream()
10621124
.flatMap(globals -> globals.getErrors().stream())

‎allure-java-commons/src/main/java/io/qameta/allure/Allure.java‎

Lines changed: 22 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -46,8 +46,9 @@
4646

4747
/**
4848
* The class contains some useful methods to work with {@link AllureLifecycle}.
49+
* This facade intentionally exposes the runtime API through a single entry point.
4950
*/
50-
@SuppressWarnings("PMD.TooManyMethods")
51+
@SuppressWarnings({"PMD.TooManyMethods", "PMD.GodClass"})
5152
public final class Allure {
5253

5354
private static final String TEXT_PLAIN = "text/plain";
@@ -422,21 +423,39 @@ public static void descriptionHtml(final String descriptionHtml) {
422423
public static void globalError(final Throwable throwable) {
423424
Objects.requireNonNull(throwable, "throwable");
424425
final StatusDetails statusDetails = getStatusDetails(throwable).orElseThrow();
425-
globalError(statusDetails);
426+
globalError(getStatus(throwable).orElse(Status.BROKEN), statusDetails);
426427
}
427428

428429
/**
429430
* Adds an error with the given status details that belongs to the test run rather than to a test or fixture.
430431
* The error is passed synchronously to the configured results writer before this method returns.
431432
* Writers that support globals store it in a new globals artifact.
432433
* The status detail fields are copied without normalization, and the timestamp is assigned when this method
433-
* is called.
434+
* is called. The error defaults to broken; use the explicit status overload to report failed.
434435
*
435436
* @param statusDetails the error details to add
436437
*/
437438
public static void globalError(final StatusDetails statusDetails) {
439+
globalError(Status.BROKEN, statusDetails);
440+
}
441+
442+
/**
443+
* Adds a run-level error with an explicit failed or broken status.
444+
* Details are copied without mutation and the error is written synchronously with a fresh timestamp.
445+
*
446+
* @param status the error status, must be failed or broken
447+
* @param statusDetails the error details to add
448+
* @throws NullPointerException if either argument is null
449+
* @throws IllegalArgumentException if the status is neither failed nor broken
450+
*/
451+
public static void globalError(final Status status, final StatusDetails statusDetails) {
452+
Objects.requireNonNull(status, "status");
438453
Objects.requireNonNull(statusDetails, "statusDetails");
454+
if (status != Status.FAILED && status != Status.BROKEN) {
455+
throw new IllegalArgumentException("Global error status must be failed or broken");
456+
}
439457
final GlobalError globalError = toGlobalError(statusDetails)
458+
.setStatus(status)
440459
.setTimestamp(System.currentTimeMillis());
441460
final Globals globals = new Globals();
442461
globals.getErrors().add(globalError);

‎allure-java-commons/src/main/java/io/qameta/allure/internal/Allure2ModelJackson.java‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,7 @@
2929
import java.util.Locale;
3030

3131
import static com.fasterxml.jackson.annotation.JsonInclude.Include.NON_NULL;
32+
import static com.fasterxml.jackson.databind.MapperFeature.ACCEPT_CASE_INSENSITIVE_ENUMS;
3233
import static com.fasterxml.jackson.databind.MapperFeature.USE_WRAPPER_NAME_AS_PROPERTY_NAME;
3334
import static com.fasterxml.jackson.databind.SerializationFeature.INDENT_OUTPUT;
3435

@@ -56,6 +57,7 @@ public static ObjectMapper createMapper() {
5657
return JsonMapper
5758
.builder()
5859
.configure(USE_WRAPPER_NAME_AS_PROPERTY_NAME, true)
60+
.configure(ACCEPT_CASE_INSENSITIVE_ENUMS, true)
5961
.serializationInclusion(NON_NULL)
6062
.configure(INDENT_OUTPUT, Boolean.getBoolean(INDENT_OUTPUT_PROPERTY_NAME))
6163
.build()

‎allure-java-commons/src/main/java/io/qameta/allure/util/ResultsUtils.java‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -748,7 +748,22 @@ public static Optional<StatusDetails> getStatusDetails(final Throwable e) {
748748
* @return the globals artifact holding a single error
749749
*/
750750
public static Globals createGlobalError(final String context, final Throwable throwable) {
751+
return createGlobalError(context, throwable, getStatus(throwable).orElse(Status.BROKEN));
752+
}
753+
754+
/**
755+
* Returns a globals artifact with an adapter-resolved error status.
756+
* Only failed is preserved; any other status is normalized to broken.
757+
* Resolve the status before passing a redacted or otherwise transformed throwable.
758+
*
759+
* @param context the description of what failed
760+
* @param throwable the failure details, may be null
761+
* @param status the adapter-resolved status, may be null
762+
* @return the globals artifact holding a single error
763+
*/
764+
public static Globals createGlobalError(final String context, final Throwable throwable, final Status status) {
751765
final GlobalError error = new GlobalError()
766+
.setStatus(status == Status.FAILED ? Status.FAILED : Status.BROKEN)
752767
.setTimestamp(System.currentTimeMillis())
753768
.setMessage(context);
754769

0 commit comments

Comments
 (0)