From 6fb96683d8e95c41d4fc5763a5484c0f00f20e67 Mon Sep 17 00:00:00 2001 From: Juan Fernandez Date: Fri, 13 Mar 2026 12:08:17 +0100 Subject: [PATCH 1/2] fix(cypress): suppress attempt-to-fix test failures so Cypress exits with code 0 When a test is marked as attempt-to-fix by test management and fails, the spec requires exit code 0 (failures should be ignored for session outcome). This extends the existing quarantine error suppression pattern to also cover attempt-to-fix tests: errors are caught in Cypress.on('fail') and not re-thrown, while the actual failure status is still reported to Datadog via the afterEach hook. Also removes stale TODO comments for quarantine/disabled handling that were fixed in PR #7442. Co-Authored-By: Claude Opus 4.6 (1M context) --- integration-tests/cypress/cypress.spec.js | 17 +-------- .../src/cypress-plugin.js | 4 +-- .../datadog-plugin-cypress/src/support.js | 36 +++++++++++++------ 3 files changed, 29 insertions(+), 28 deletions(-) diff --git a/integration-tests/cypress/cypress.spec.js b/integration-tests/cypress/cypress.spec.js index c9cf3a9a7aa..35684e81381 100644 --- a/integration-tests/cypress/cypress.spec.js +++ b/integration-tests/cypress/cypress.spec.js @@ -2975,11 +2975,9 @@ moduleTypes.forEach(({ testAssertionsPromise, ]) - if (shouldAlwaysPass) { + if (shouldAlwaysPass || isAttemptToFix) { assert.strictEqual(exitCode, 0) } else { - // TODO: we need to figure out how to trick cypress into returning exit code 0 - // even if there are failed tests assert.strictEqual(exitCode, 1) } } @@ -3014,14 +3012,6 @@ moduleTypes.forEach(({ await runAttemptToFixTest({ extraEnvVars: { DD_TEST_MANAGEMENT_ENABLED: '0' } }) }) - /** - * TODO: - * The spec says that quarantined tests that are not attempted to fix should be run and their result ignored. - * Cypress will skip the test instead. - * - * When a test is quarantined and attempted to fix, the spec is to run the test and ignore its result. - * Cypress will run the test, but it won't ignore its result. - */ it('can mark tests as quarantined and tests are not skipped', async () => { receiver.setSettings({ test_management: { enabled: true, attempt_to_fix_retries: 3 } }) receiver.setTestManagementTests({ @@ -3044,11 +3034,6 @@ moduleTypes.forEach(({ await runAttemptToFixTest({ isAttemptToFix: true, isQuarantined: true }) }) - /** - * TODO: - * When a test is disabled and attempted to fix, the spec is to run the test and ignore its result. - * Cypress will run the test, but it won't ignore its result. - */ it('can mark tests as disabled and tests are not skipped', async () => { receiver.setSettings({ test_management: { enabled: true, attempt_to_fix_retries: 3 } }) receiver.setTestManagementTests({ diff --git a/packages/datadog-plugin-cypress/src/cypress-plugin.js b/packages/datadog-plugin-cypress/src/cypress-plugin.js index a29db608fa5..71e8c9166bb 100644 --- a/packages/datadog-plugin-cypress/src/cypress-plugin.js +++ b/packages/datadog-plugin-cypress/src/cypress-plugin.js @@ -804,10 +804,10 @@ class CypressPlugin { if (cypressTest.displayError) { latestError = new Error(cypressTest.displayError) } - // Update test status - but NOT for quarantined tests where we intentionally + // Update test status - but NOT for quarantined or attempt-to-fix tests where we intentionally // report 'fail' to Datadog even though Cypress sees it as 'pass' const isQuarantinedTest = finishedTest.testSpan?.context()?._tags?.[TEST_MANAGEMENT_IS_QUARANTINED] === 'true' - if (cypressTestStatus !== finishedTest.testStatus && !isQuarantinedTest) { + if (cypressTestStatus !== finishedTest.testStatus && !isQuarantinedTest && !finishedTest.isAttemptToFix) { finishedTest.testSpan.setTag(TEST_STATUS, cypressTestStatus) finishedTest.testSpan.setTag('error', latestError) } diff --git a/packages/datadog-plugin-cypress/src/support.js b/packages/datadog-plugin-cypress/src/support.js index fddfd5ad1fa..fe31ae311f4 100644 --- a/packages/datadog-plugin-cypress/src/support.js +++ b/packages/datadog-plugin-cypress/src/support.js @@ -17,6 +17,8 @@ let isTestIsolationEnabled = false const retryReasonsByTestName = new Map() // Track quarantined test errors - we catch them in Cypress.on('fail') but need to report to Datadog const quarantinedTestErrors = new Map() +// Track attempt-to-fix test errors - suppress failures so Cypress exits 0, but report actual status to Datadog +const attemptToFixTestErrors = new Map() // Track the most recently loaded window in the AUT. Updated via the 'window:load' // event so we always get the real app window (after cy.visit()), not the @@ -72,7 +74,14 @@ Cypress.on('fail', (err, runnable) => { return } - // For all other tests (including attemptToFix), let the error propagate normally + // For attempt-to-fix tests, suppress the failure so Cypress exits with code 0 + // The actual failure status is still reported to Datadog in afterEach + if (isAttemptToFix) { + attemptToFixTestErrors.set(testName, err) + return + } + + // For all other tests, let the error propagate normally throw err }) @@ -247,13 +256,17 @@ afterEach(function () { const currentTest = Cypress.mocha.getRunner().suite.ctx.currentTest const testName = currentTest.fullTitle() - // Check if this was a quarantined test that we suppressed the failure for + // Check if this was a quarantined or attempt-to-fix test that we suppressed the failure for const quarantinedError = quarantinedTestErrors.get(testName) const isQuarantinedTestThatFailed = !!quarantinedError - - // For quarantined tests, convert Error to a serializable format for cy.task - const errorToReport = isQuarantinedTestThatFailed - ? { message: quarantinedError.message, stack: quarantinedError.stack } + const attemptToFixError = attemptToFixTestErrors.get(testName) + const isAttemptToFixTestThatFailed = !!attemptToFixError + const hasSuppressedFailure = isQuarantinedTestThatFailed || isAttemptToFixTestThatFailed + const suppressedError = quarantinedError || attemptToFixError + + // For tests with suppressed failures, convert Error to a serializable format for cy.task + const errorToReport = hasSuppressedFailure + ? { message: suppressedError.message, stack: suppressedError.stack } : currentTest.err const testInfo = { @@ -261,9 +274,9 @@ afterEach(function () { testItTitle: currentTest.title, testSuite: Cypress.mocha.getRootSuite().file, testSuiteAbsolutePath: Cypress.spec && Cypress.spec.absolute, - // For quarantined tests, report the actual state (failed) to Datadog, not what Cypress thinks (passed) - state: isQuarantinedTestThatFailed ? 'failed' : currentTest.state, - // For quarantined tests, include the actual error that was suppressed + // For tests with suppressed failures, report the actual state (failed) to Datadog + state: hasSuppressedFailure ? 'failed' : currentTest.state, + // Include the actual error that was suppressed error: errorToReport, isNew: currentTest._ddIsNew, isEfdRetry: currentTest._ddIsEfdRetry, @@ -294,10 +307,13 @@ afterEach(function () { // ignore error and continue } - // Clean up the quarantined error tracking + // Clean up suppressed error tracking if (isQuarantinedTestThatFailed) { quarantinedTestErrors.delete(testName) } + if (isAttemptToFixTestThatFailed) { + attemptToFixTestErrors.delete(testName) + } cy.task('dd:afterEach', { test: testInfo, coverage }) }) From 1882fe823f83b827368ddb637edb820465115c34 Mon Sep 17 00:00:00 2001 From: Juan Fernandez Date: Fri, 13 Mar 2026 16:52:19 +0100 Subject: [PATCH 2/2] fix(cypress): update invocationDetails expected line after support.js changes Adding lines to support.js shifts line numbers in Cypress's bundled output. The invocationDetails test hardcodes the expected source line, which moved from 246 to 255. Co-Authored-By: Claude Opus 4.6 (1M context) --- integration-tests/cypress/cypress.spec.js | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/integration-tests/cypress/cypress.spec.js b/integration-tests/cypress/cypress.spec.js index 35684e81381..63fd3ce85ff 100644 --- a/integration-tests/cypress/cypress.spec.js +++ b/integration-tests/cypress/cypress.spec.js @@ -540,7 +540,7 @@ moduleTypes.forEach(({ assert.ok(jsInvocationDetailsEvent, 'plain-js invocationDetails test event should exist') assert.strictEqual( jsInvocationDetailsEvent.content.metrics[TEST_SOURCE_START], - 246, + 255, 'should keep invocationDetails line directly for plain JS specs without source maps' ) assert.ok( @@ -560,6 +560,9 @@ moduleTypes.forEach(({ }, }) + childProcess.stdout?.pipe(process.stdout) + childProcess.stderr?.pipe(process.stderr) + const [[exitCode]] = await Promise.all([once(childProcess, 'exit'), receiverPromise]) assert.strictEqual(exitCode, 0, 'cypress process should exit successfully') })