Apply automation exclusions to Karate scenarios in both major versions

Author: calvinbayer

dd-java-agent/instrumentation/karate/karate-1.0/src/main/java/datadog/trace/instrumentation/karate/KarateTracingHook.java

@@ -115,7 +115,9 @@ public boolean beforeScenario(ScenarioRuntime sr) {
     if (Config.get().isCiVisibilityTestSkippingEnabled()
         || Config.get().isCiVisibilityTestManagementEnabled()) {
       TestIdentifier skippableTest = KarateUtils.toTestIdentifier(scenario);
-      SkipReason skipReason = TestEventsHandlerHolder.TEST_EVENTS_HANDLER.skipReason(skippableTest);
+      SkipReason skipReason =
+          TestEventsHandlerHolder.TEST_EVENTS_HANDLER.skipReason(
+              skippableTest, TestSourceData.UNKNOWN, categories);
 
       if (skipReason != null
           && !(skipReason == SkipReason.ITR

dd-java-agent/instrumentation/karate/karate-1.0/src/test/groovy/KarateTest.groovy

@@ -27,6 +27,33 @@ import static org.junit.platform.engine.discovery.DiscoverySelectors.selectClass
 @DisableTestTrace(reason = "avoid self-tracing")
 class KarateTest extends CiVisibilityInstrumentationTest {
 
+  def "test automation exclusions preserve a blocking failure and execute once"() {
+    given:
+    def id = new TestFQN("[org/example/test_automation_excluded] automation exclusions", "protected_failure")
+    givenDisabledTests([id])
+    givenQuarantinedTests([id])
+    givenAttemptToFixTests([id])
+    givenFlakyRetryEnabled(true)
+    givenFlakyTests([id])
+    givenEarlyFlakinessDetectionEnabled(true)
+    givenKnownTests([])
+
+    when:
+    runTests([TestAutomationExcludedKarate], false)
+
+    then:
+    TEST_WRITER.toList().flatten().count { it.getTag("test.name") == "protected_failure" } == 1
+    def testSpan = TEST_WRITER.toList().flatten().find { it.getTag("test.name") == "protected_failure" }
+    testSpan != null
+    testSpan.getTag("test.test_management.no_auto_retry") == true
+    testSpan.getTag("test.test_management.prevent_failure_suppression") == true
+    testSpan.getTag("test.test_management.is_test_disabled") == true
+    testSpan.getTag("test.test_management.is_quarantined") == true
+    testSpan.getTag("test.test_management.is_attempt_to_fix") == true
+    testSpan.getTag("test.test_management.attempt_to_fix_passed") == null
+  }
+
+
   def "test #testcaseName"() {
     Assumptions.assumeTrue(assumption)
 

dd-java-agent/instrumentation/karate/karate-1.0/src/test/java/org/example/TestAutomationExcludedKarate.java

@@ -0,0 +1,11 @@
+package org.example;
+
+import com.intuit.karate.junit5.Karate;
+
+public class TestAutomationExcludedKarate {
+
+  @Karate.Test
+  public Karate testFailed() {
+    return Karate.run("classpath:org/example/test_automation_excluded.feature");
+  }
+}

dd-java-agent/instrumentation/karate/karate-1.0/src/test/java/org/example/test_automation_excluded.feature

@@ -0,0 +1,5 @@
+Feature: automation exclusions
+
+  @datadog_no_auto_retry @datadog_prevent_failure_suppression
+  Scenario: protected_failure
+    * assert false

dd-java-agent/instrumentation/karate/karate-2.0/src/main/java21/datadog/trace/instrumentation/karate2/KarateTracingListener.java

@@ -124,7 +124,9 @@ private boolean beforeScenario(ScenarioRunEvent event) {
     if (Config.get().isCiVisibilityTestSkippingEnabled()
         || Config.get().isCiVisibilityTestManagementEnabled()) {
       TestIdentifier skippableTest = KarateUtils.toTestIdentifier(scenario);
-      SkipReason skipReason = TestEventsHandlerHolder.TEST_EVENTS_HANDLER.skipReason(skippableTest);
+      SkipReason skipReason =
+          TestEventsHandlerHolder.TEST_EVENTS_HANDLER.skipReason(
+              skippableTest, TestSourceData.UNKNOWN, categories);
 
       if (skipReason != null
           && !(skipReason == SkipReason.ITR

dd-java-agent/instrumentation/karate/karate-2.0/src/test/groovy/KarateV2Test.groovy

@@ -21,6 +21,33 @@ import static org.junit.platform.engine.discovery.DiscoverySelectors.selectClass
 @DisableTestTrace(reason = "avoid self-tracing")
 class KarateV2Test extends CiVisibilityInstrumentationTest {
 
+  def "test automation exclusions preserve a blocking failure and execute once"() {
+    given:
+    def id = new TestFQN("[org/example/test_automation_excluded] automation exclusions", "protected_failure")
+    givenDisabledTests([id])
+    givenQuarantinedTests([id])
+    givenAttemptToFixTests([id])
+    givenFlakyRetryEnabled(true)
+    givenFlakyTests([id])
+    givenEarlyFlakinessDetectionEnabled(true)
+    givenKnownTests([])
+
+    when:
+    runTests([TestAutomationExcludedKarate], false)
+
+    then:
+    TEST_WRITER.toList().flatten().count { it.getTag("test.name") == "protected_failure" } == 1
+    def testSpan = TEST_WRITER.toList().flatten().find { it.getTag("test.name") == "protected_failure" }
+    testSpan != null
+    testSpan.getTag("test.test_management.no_auto_retry") == true
+    testSpan.getTag("test.test_management.prevent_failure_suppression") == true
+    testSpan.getTag("test.test_management.is_test_disabled") == true
+    testSpan.getTag("test.test_management.is_quarantined") == true
+    testSpan.getTag("test.test_management.is_attempt_to_fix") == true
+    testSpan.getTag("test.test_management.attempt_to_fix_passed") == null
+  }
+
+
   def "test #testcaseName"() {
     runTests(tests, success)
 

dd-java-agent/instrumentation/karate/karate-2.0/src/test/java/org/example/TestAutomationExcludedKarate.java

@@ -0,0 +1,17 @@
+package org.example;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
+import com.intuit.karate.Results;
+import com.intuit.karate.Runner;
+import org.junit.jupiter.api.Test;
+
+public class TestAutomationExcludedKarate {
+
+  @Test
+  public void test() {
+    Results results =
+        Runner.path("classpath:org/example/test_automation_excluded.feature").parallel(1);
+    assertEquals(0, results.getFailCount(), results.getErrorMessages());
+  }
+}

dd-java-agent/instrumentation/karate/karate-2.0/src/test/java/org/example/test_automation_excluded.feature

@@ -0,0 +1,5 @@
+Feature: automation exclusions
+
+  @datadog_no_auto_retry @datadog_prevent_failure_suppression
+  Scenario: protected_failure
+    * assert false

docs/test-automation-exclusions.md

@@ -54,3 +54,10 @@ For either the JUnit4 or JUnit Platform Cucumber runner, tag a Scenario or
 Scenario Outline with `@datadog_no_auto_retry` and/or
 `@datadog_prevent_failure_suppression`. These names correspond to the public
 `TestAutomationExclusions` constants. Tags apply to every outline example.
+
+## Karate
+
+Tag the Scenario or Scenario Outline with `@datadog_no_auto_retry` and/or
+`@datadog_prevent_failure_suppression` (the names from the public
+`TestAutomationExclusions` constants). Both Karate 1.x and 2.x resolve these
+scenario tags before Datadog skipping and retry policy selection.