[test optimization] Fix cypress attempt to fix

integration-tests/cypress/cypress.spec.js

@@ -540,7 +540,7 @@ moduleTypes.forEach(({
           assert.ok(jsInvocationDetailsEvent, 'plain-js invocationDetails test event should exist')
           assert.strictEqual(
             jsInvocationDetailsEvent.content.metrics[TEST_SOURCE_START],
-            246,
+            255,
             'should keep invocationDetails line directly for plain JS specs without source maps'
           )
           assert.ok(
@@ -560,6 +560,9 @@ moduleTypes.forEach(({
         },
       })
 
+      childProcess.stdout?.pipe(process.stdout)
+      childProcess.stderr?.pipe(process.stderr)
+
       const [[exitCode]] = await Promise.all([once(childProcess, 'exit'), receiverPromise])
       assert.strictEqual(exitCode, 0, 'cypress process should exit successfully')
     })
@@ -2975,11 +2978,9 @@ moduleTypes.forEach(({
             testAssertionsPromise,
           ])
 
-          if (shouldAlwaysPass) {
+          if (shouldAlwaysPass || isAttemptToFix) {
             assert.strictEqual(exitCode, 0)
           } else {
-            // TODO: we need to figure out how to trick cypress into returning exit code 0
-            // even if there are failed tests
             assert.strictEqual(exitCode, 1)
           }
         }
@@ -3014,14 +3015,6 @@ moduleTypes.forEach(({
           await runAttemptToFixTest({ extraEnvVars: { DD_TEST_MANAGEMENT_ENABLED: '0' } })
         })
 
-        /**
-         * TODO:
-         * The spec says that quarantined tests that are not attempted to fix should be run and their result ignored.
-         * Cypress will skip the test instead.
-         *
-         * When a test is quarantined and attempted to fix, the spec is to run the test and ignore its result.
-         * Cypress will run the test, but it won't ignore its result.
-         */
         it('can mark tests as quarantined and tests are not skipped', async () => {
           receiver.setSettings({ test_management: { enabled: true, attempt_to_fix_retries: 3 } })
           receiver.setTestManagementTests({
@@ -3044,11 +3037,6 @@ moduleTypes.forEach(({
           await runAttemptToFixTest({ isAttemptToFix: true, isQuarantined: true })
         })
 
-        /**
-         * TODO:
-         * When a test is disabled and attempted to fix, the spec is to run the test and ignore its result.
-         * Cypress will run the test, but it won't ignore its result.
-         */
         it('can mark tests as disabled and tests are not skipped', async () => {
           receiver.setSettings({ test_management: { enabled: true, attempt_to_fix_retries: 3 } })
           receiver.setTestManagementTests({

packages/datadog-plugin-cypress/src/cypress-plugin.js

@@ -804,10 +804,10 @@ class CypressPlugin {
         if (cypressTest.displayError) {
           latestError = new Error(cypressTest.displayError)
         }
-        // Update test status - but NOT for quarantined tests where we intentionally
+        // Update test status - but NOT for quarantined or attempt-to-fix tests where we intentionally
         // report 'fail' to Datadog even though Cypress sees it as 'pass'
         const isQuarantinedTest = finishedTest.testSpan?.context()?._tags?.[TEST_MANAGEMENT_IS_QUARANTINED] === 'true'
-        if (cypressTestStatus !== finishedTest.testStatus && !isQuarantinedTest) {
+        if (cypressTestStatus !== finishedTest.testStatus && !isQuarantinedTest && !finishedTest.isAttemptToFix) {
           finishedTest.testSpan.setTag(TEST_STATUS, cypressTestStatus)
           finishedTest.testSpan.setTag('error', latestError)
         }

packages/datadog-plugin-cypress/src/support.js

@@ -17,6 +17,8 @@ let isTestIsolationEnabled = false
 const retryReasonsByTestName = new Map()
 // Track quarantined test errors - we catch them in Cypress.on('fail') but need to report to Datadog
 const quarantinedTestErrors = new Map()
+// Track attempt-to-fix test errors - suppress failures so Cypress exits 0, but report actual status to Datadog
+const attemptToFixTestErrors = new Map()
 
 // Track the most recently loaded window in the AUT. Updated via the 'window:load'
 // event so we always get the real app window (after cy.visit()), not the
@@ -72,7 +74,14 @@ Cypress.on('fail', (err, runnable) => {
     return
   }
 
-  // For all other tests (including attemptToFix), let the error propagate normally
+  // For attempt-to-fix tests, suppress the failure so Cypress exits with code 0
+  // The actual failure status is still reported to Datadog in afterEach
+  if (isAttemptToFix) {
+    attemptToFixTestErrors.set(testName, err)
+    return
+  }
+
+  // For all other tests, let the error propagate normally
   throw err
 })
 
@@ -247,23 +256,27 @@ afterEach(function () {
   const currentTest = Cypress.mocha.getRunner().suite.ctx.currentTest
   const testName = currentTest.fullTitle()
 
-  // Check if this was a quarantined test that we suppressed the failure for
+  // Check if this was a quarantined or attempt-to-fix test that we suppressed the failure for
   const quarantinedError = quarantinedTestErrors.get(testName)
   const isQuarantinedTestThatFailed = !!quarantinedError
-
-  // For quarantined tests, convert Error to a serializable format for cy.task
-  const errorToReport = isQuarantinedTestThatFailed
-    ? { message: quarantinedError.message, stack: quarantinedError.stack }
+  const attemptToFixError = attemptToFixTestErrors.get(testName)
+  const isAttemptToFixTestThatFailed = !!attemptToFixError
+  const hasSuppressedFailure = isQuarantinedTestThatFailed || isAttemptToFixTestThatFailed
+  const suppressedError = quarantinedError || attemptToFixError
+
+  // For tests with suppressed failures, convert Error to a serializable format for cy.task
+  const errorToReport = hasSuppressedFailure
+    ? { message: suppressedError.message, stack: suppressedError.stack }
     : currentTest.err
 
   const testInfo = {
     testName,
     testItTitle: currentTest.title,
     testSuite: Cypress.mocha.getRootSuite().file,
     testSuiteAbsolutePath: Cypress.spec && Cypress.spec.absolute,
-    // For quarantined tests, report the actual state (failed) to Datadog, not what Cypress thinks (passed)
-    state: isQuarantinedTestThatFailed ? 'failed' : currentTest.state,
-    // For quarantined tests, include the actual error that was suppressed
+    // For tests with suppressed failures, report the actual state (failed) to Datadog
+    state: hasSuppressedFailure ? 'failed' : currentTest.state,
+    // Include the actual error that was suppressed
     error: errorToReport,
     isNew: currentTest._ddIsNew,
     isEfdRetry: currentTest._ddIsEfdRetry,
@@ -294,10 +307,13 @@ afterEach(function () {
     // ignore error and continue
   }
 
-  // Clean up the quarantined error tracking
+  // Clean up suppressed error tracking
   if (isQuarantinedTestThatFailed) {
     quarantinedTestErrors.delete(testName)
   }
+  if (isAttemptToFixTestThatFailed) {
+    attemptToFixTestErrors.delete(testName)
+  }
 
   cy.task('dd:afterEach', { test: testInfo, coverage })
 })