Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,8 @@
import io.cucumber.plugin.event.TestCase;
import io.cucumber.plugin.event.TestCaseFinished;
import io.cucumber.plugin.event.TestCaseStarted;
import io.cucumber.plugin.event.TestRunFinished;
import io.cucumber.plugin.event.TestRunStarted;
import io.cucumber.plugin.event.TestSourceRead;
import io.cucumber.plugin.event.TestStepFinished;
import io.cucumber.plugin.event.TestStepStarted;
Expand Down Expand Up @@ -62,10 +64,12 @@
import java.util.Optional;
import java.util.UUID;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.stream.Collectors;
import java.util.stream.IntStream;
import java.util.stream.Stream;

import static io.qameta.allure.util.ResultsUtils.createGlobalError;
import static io.qameta.allure.util.ResultsUtils.createParameter;
import static io.qameta.allure.util.ResultsUtils.createTitlePath;
import static io.qameta.allure.util.ResultsUtils.createTitlePathFromSourcePath;
Expand Down Expand Up @@ -99,13 +103,17 @@ public class AllureCucumber7Jvm implements ConcurrentEventListener {

private final TestSourcesModelProxy testSources = new TestSourcesModelProxy();

private final AtomicBoolean scenarioStarted = new AtomicBoolean();

private final EventHandler<TestRunStarted> runStartedHandler = event -> scenarioStarted.set(false);
private final EventHandler<TestSourceRead> featureStartedHandler = this::handleFeatureStartedHandler;
private final EventHandler<TestCaseStarted> caseStartedHandler = this::handleTestCaseStarted;
private final EventHandler<TestCaseFinished> caseFinishedHandler = this::handleTestCaseFinished;
private final EventHandler<TestStepStarted> stepStartedHandler = this::handleTestStepStarted;
private final EventHandler<TestStepFinished> stepFinishedHandler = this::handleTestStepFinished;
private final EventHandler<WriteEvent> writeEventHandler = this::handleWriteEvent;
private final EventHandler<EmbedEvent> embedEventHandler = this::handleEmbedEvent;
private final EventHandler<TestRunFinished> runFinishedHandler = this::handleTestRunFinished;

private final Map<UUID, String> hookStepContainerUuid = new ConcurrentHashMap<>();

Expand Down Expand Up @@ -134,6 +142,7 @@ public AllureCucumber7Jvm(final AllureLifecycle lifecycle) {
*/
@Override
public void setEventPublisher(final EventPublisher publisher) {
publisher.registerHandlerFor(TestRunStarted.class, runStartedHandler);
publisher.registerHandlerFor(TestSourceRead.class, featureStartedHandler);

publisher.registerHandlerFor(TestCaseStarted.class, caseStartedHandler);
Expand All @@ -144,6 +153,7 @@ public void setEventPublisher(final EventPublisher publisher) {

publisher.registerHandlerFor(WriteEvent.class, writeEventHandler);
publisher.registerHandlerFor(EmbedEvent.class, embedEventHandler);
publisher.registerHandlerFor(TestRunFinished.class, runFinishedHandler);
}

private static AllureExternalKey scopeKey(final String uuid) {
Expand All @@ -167,6 +177,7 @@ private void handleFeatureStartedHandler(final TestSourceRead event) {
}

private void handleTestCaseStarted(final TestCaseStarted event) {
scenarioStarted.set(true);
final TestCase testCase = event.getTestCase();
final Feature feature = testSources.getFeature(testCase.getUri());

Expand Down Expand Up @@ -218,6 +229,25 @@ private void handleTestCaseStarted(final TestCaseStarted event) {
lifecycle.startTest(testKey);
}

/**
* Reports an error outside an individual scenario as an error of the run.
*
* <p>A failed scenario also makes the run-finished result unsuccessful, but its error stays on the
* {@link TestCaseFinished} event and the run-finished result has no throwable. Only a throwable here represents
* a failure such as a run-level hook that has no scenario result of its own.</p>
*/
private void handleTestRunFinished(final TestRunFinished event) {
final Result result = event.getResult();
if (Objects.isNull(result) || Objects.isNull(result.getError())) {
return;
}

final String context = scenarioStarted.get()
? "Cucumber test run failed"
: "Cucumber test run failed, scenarios did not run";
lifecycle.writeGlobals(createGlobalError(context, result.getError()));
}

private void handleTestCaseFinished(final TestCaseFinished event) {
final TestCase testCase = event.getTestCase();
final Feature feature = testSources.getFeature(testCase.getUri());
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -26,9 +26,11 @@
import io.cucumber.core.runtime.TimeServiceEventBus;
import io.github.glytching.junit.extension.system.SystemProperty;
import io.github.glytching.junit.extension.system.SystemPropertyExtension;
import io.qameta.allure.Description;
import io.qameta.allure.Step;
import io.qameta.allure.model.Attachment;
import io.qameta.allure.model.FixtureResult;
import io.qameta.allure.model.GlobalError;
import io.qameta.allure.model.Label;
import io.qameta.allure.model.Link;
import io.qameta.allure.model.Parameter;
Expand All @@ -52,14 +54,17 @@
import java.util.Arrays;
import java.util.List;
import java.util.UUID;
import java.util.function.Consumer;
import java.util.function.Supplier;
import java.util.stream.Collectors;

import static io.qameta.allure.util.ResultsUtils.PACKAGE_LABEL_NAME;
import static io.qameta.allure.util.ResultsUtils.SUITE_LABEL_NAME;
import static io.qameta.allure.util.ResultsUtils.TEST_CLASS_LABEL_NAME;
import static io.qameta.allure.util.ResultsUtils.TEST_METHOD_LABEL_NAME;
import static io.qameta.allure.util.ResultsUtils.md5;
import static org.assertj.core.api.Assertions.assertThat;
import static org.assertj.core.api.Assertions.assertThatThrownBy;
import static org.assertj.core.api.Assertions.tuple;
import static org.junit.jupiter.api.parallel.ResourceAccessMode.READ_WRITE;
import static org.junit.jupiter.api.parallel.Resources.SYSTEM_PROPERTIES;
Expand Down Expand Up @@ -92,15 +97,22 @@ void shouldSetStatus() {
);
}

/**
* A failed scenario keeps the failure on its own result and is not duplicated as an error of the test run.
*/
@AllureFeatures.FailedTests
@Test
@Description
void shouldSetFailedStatus() {
final AllureResults results = runFeature("features/failed.feature");

final List<TestResult> testResults = results.getTestResults();
assertThat(testResults)
.extracting(TestResult::getStatus)
.containsExactlyInAnyOrder(Status.FAILED);
assertThat(getGlobalErrors(results))
.as("the scenario result already represents this failure")
.isEmpty();
}

@AllureFeatures.FailedTests
Expand Down Expand Up @@ -725,6 +737,56 @@ void shouldDisplayHooksAsStages() {

}

/**
* A failed Cucumber {@code @BeforeAll} hook is an error of the run. No scenario result is invented because the
* hook prevents every selected scenario from starting.
*/
@AllureFeatures.Fixtures
@AllureFeatures.BrokenTests
@Test
@Description
void shouldReportBrokenBeforeAllAsGlobalError() {
final AllureResults results = runFeatureExpectingFailure(
"features/simple.feature",
"Exception in @BeforeAll",
"--glue", "io.qameta.allure.cucumber7jvm.runhooks.beforeall"
);

assertThat(results.getTestResults())
.as("the run-level hook failed before a scenario could start")
.isEmpty();
assertThat(getGlobalErrors(results))
.extracting(GlobalError::getMessage)
.containsExactly(
"Cucumber test run failed, scenarios did not run: Exception in @BeforeAll"
);
}

/**
* A failed Cucumber {@code @AfterAll} hook is an error of the run, while scenarios completed before the hook
* keep their own results.
*/
@AllureFeatures.Fixtures
@AllureFeatures.BrokenTests
@Test
@Description
void shouldReportBrokenAfterAllAsGlobalError() {
final AllureResults results = runFeatureExpectingFailure(
"features/simple.feature",
"Exception in @AfterAll",
"--glue", "io.qameta.allure.cucumber7jvm.runhooks.afterall"
);

assertThat(results.getTestResults())
.extracting(TestResult::getName, TestResult::getStatus)
.containsExactly(tuple("Add a to b", Status.PASSED));
assertThat(getGlobalErrors(results))
.extracting(GlobalError::getMessage)
.containsExactly(
"Cucumber test run failed: Exception in @AfterAll"
);
}

@AllureFeatures.BrokenTests
@Test
void shouldHandleAmbigiousStepsExceptions() {
Expand Down Expand Up @@ -834,6 +896,26 @@ void shouldCalculateIdsFromRuntimeParametersAtTestEnd() {
private AllureResults runFeature(final String featureResource,
final String... moreOptions) {

return runFeature(featureResource, Runtime::run, moreOptions);
}

@Step
private AllureResults runFeatureExpectingFailure(final String featureResource,
final String expectedMessage,
final String... moreOptions) {

return runFeature(
featureResource,
runtime -> assertThatThrownBy(runtime::run)
.hasMessageContaining(expectedMessage),
moreOptions
);
}

private AllureResults runFeature(final String featureResource,
final Consumer<Runtime> run,
final String... moreOptions) {

return RunUtils.runTests(lifecycle -> {
final AllureCucumber7Jvm cucumber7jvm = new AllureCucumber7Jvm(lifecycle);
final Supplier<ClassLoader> classLoader = ClassLoaders::getDefaultClassLoader;
Expand All @@ -859,8 +941,14 @@ private AllureResults runFeature(final String featureResource,
.withFeatureSupplier(supplier)
.build();

runtime.run();
run.accept(runtime);
});
}

private static List<GlobalError> getGlobalErrors(final AllureResults results) {
return results.getGlobals().stream()
.flatMap(globals -> globals.getErrors().stream())
.collect(Collectors.toList());
}

}
Original file line number Diff line number Diff line change
@@ -0,0 +1,29 @@
/*
* Copyright 2016-2026 Qameta Software Inc
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.qameta.allure.cucumber7jvm.runhooks.afterall;

import io.cucumber.java.AfterAll;

public final class BrokenAfterAllHooks {

private BrokenAfterAllHooks() {
}

@AfterAll
public static void afterAll() {
throw new RuntimeException("Exception in @AfterAll");
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,29 @@
/*
* Copyright 2016-2026 Qameta Software Inc
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.qameta.allure.cucumber7jvm.runhooks.beforeall;

import io.cucumber.java.BeforeAll;

public final class BrokenBeforeAllHooks {

private BrokenBeforeAllHooks() {
}

@BeforeAll
public static void beforeAll() {
throw new RuntimeException("Exception in @BeforeAll");
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,8 @@
import io.qameta.allure.Severity;
import io.qameta.allure.SeverityLevel;
import io.qameta.allure.Story;
import io.qameta.allure.model.GlobalError;
import io.qameta.allure.model.Globals;
import io.qameta.allure.model.Label;
import io.qameta.allure.model.Link;
import io.qameta.allure.model.Parameter;
Expand Down Expand Up @@ -729,6 +731,41 @@ public static Optional<StatusDetails> getStatusDetails(final Throwable e) {
});
}

/**
* Returns the globals artifact for a failure that belongs to the test run rather than to any test result.
*
* <p>Adapters use this for failures they cannot attribute to a test without inventing one: a configuration
* method that failed outside any test, or a data provider that failed before the invocations it would have
* produced were known. A test result invented for such a failure carries a history id that no execution ever
* produces, so it distorts the run statistics and no retry can ever replace it.</p>
*
* <p>Write the result with {@link io.qameta.allure.AllureLifecycle#writeGlobals(Globals)} on the adapter's own
* lifecycle rather than through the {@link io.qameta.allure.Allure} facade, so that the error reaches the
* lifecycle the adapter reports to.</p>
*
* @param context the description of what failed, and of what did not run when the failure kept tests from
* running
* @param throwable the failure, may be null
* @return the globals artifact holding a single error
*/
public static Globals createGlobalError(final String context, final Throwable throwable) {
final GlobalError error = new GlobalError()
.setTimestamp(System.currentTimeMillis())
.setMessage(context);

getStatusDetails(throwable).ifPresent(
details -> error
.setMessage(context + ": " + details.getMessage())
.setTrace(details.getTrace())
.setActual(details.getActual())
.setExpected(details.getExpected())
);

final Globals globals = new Globals();
globals.getErrors().add(error);
return globals;
}

/**
* Returns the javadoc description.
*
Expand Down
Loading