diff --git a/integration-tests/cypress/cypress.spec.js b/integration-tests/cypress/cypress.spec.js index c9cf3a9a7aa..63fd3ce85ff 100644 --- a/integration-tests/cypress/cypress.spec.js +++ b/integration-tests/cypress/cypress.spec.js @@ -540,7 +540,7 @@ moduleTypes.forEach(({ assert.ok(jsInvocationDetailsEvent, 'plain-js invocationDetails test event should exist') assert.strictEqual( jsInvocationDetailsEvent.content.metrics[TEST_SOURCE_START], - 246, + 255, 'should keep invocationDetails line directly for plain JS specs without source maps' ) assert.ok( @@ -560,6 +560,9 @@ moduleTypes.forEach(({ }, }) + childProcess.stdout?.pipe(process.stdout) + childProcess.stderr?.pipe(process.stderr) + const [[exitCode]] = await Promise.all([once(childProcess, 'exit'), receiverPromise]) assert.strictEqual(exitCode, 0, 'cypress process should exit successfully') }) @@ -2975,11 +2978,9 @@ moduleTypes.forEach(({ testAssertionsPromise, ]) - if (shouldAlwaysPass) { + if (shouldAlwaysPass || isAttemptToFix) { assert.strictEqual(exitCode, 0) } else { - // TODO: we need to figure out how to trick cypress into returning exit code 0 - // even if there are failed tests assert.strictEqual(exitCode, 1) } } @@ -3014,14 +3015,6 @@ moduleTypes.forEach(({ await runAttemptToFixTest({ extraEnvVars: { DD_TEST_MANAGEMENT_ENABLED: '0' } }) }) - /** - * TODO: - * The spec says that quarantined tests that are not attempted to fix should be run and their result ignored. - * Cypress will skip the test instead. - * - * When a test is quarantined and attempted to fix, the spec is to run the test and ignore its result. - * Cypress will run the test, but it won't ignore its result. - */ it('can mark tests as quarantined and tests are not skipped', async () => { receiver.setSettings({ test_management: { enabled: true, attempt_to_fix_retries: 3 } }) receiver.setTestManagementTests({ @@ -3044,11 +3037,6 @@ moduleTypes.forEach(({ await runAttemptToFixTest({ isAttemptToFix: true, isQuarantined: true }) }) - /** - * TODO: - * When a test is disabled and attempted to fix, the spec is to run the test and ignore its result. - * Cypress will run the test, but it won't ignore its result. - */ it('can mark tests as disabled and tests are not skipped', async () => { receiver.setSettings({ test_management: { enabled: true, attempt_to_fix_retries: 3 } }) receiver.setTestManagementTests({ diff --git a/packages/datadog-plugin-cypress/src/cypress-plugin.js b/packages/datadog-plugin-cypress/src/cypress-plugin.js index a29db608fa5..71e8c9166bb 100644 --- a/packages/datadog-plugin-cypress/src/cypress-plugin.js +++ b/packages/datadog-plugin-cypress/src/cypress-plugin.js @@ -804,10 +804,10 @@ class CypressPlugin { if (cypressTest.displayError) { latestError = new Error(cypressTest.displayError) } - // Update test status - but NOT for quarantined tests where we intentionally + // Update test status - but NOT for quarantined or attempt-to-fix tests where we intentionally // report 'fail' to Datadog even though Cypress sees it as 'pass' const isQuarantinedTest = finishedTest.testSpan?.context()?._tags?.[TEST_MANAGEMENT_IS_QUARANTINED] === 'true' - if (cypressTestStatus !== finishedTest.testStatus && !isQuarantinedTest) { + if (cypressTestStatus !== finishedTest.testStatus && !isQuarantinedTest && !finishedTest.isAttemptToFix) { finishedTest.testSpan.setTag(TEST_STATUS, cypressTestStatus) finishedTest.testSpan.setTag('error', latestError) } diff --git a/packages/datadog-plugin-cypress/src/support.js b/packages/datadog-plugin-cypress/src/support.js index fddfd5ad1fa..fe31ae311f4 100644 --- a/packages/datadog-plugin-cypress/src/support.js +++ b/packages/datadog-plugin-cypress/src/support.js @@ -17,6 +17,8 @@ let isTestIsolationEnabled = false const retryReasonsByTestName = new Map() // Track quarantined test errors - we catch them in Cypress.on('fail') but need to report to Datadog const quarantinedTestErrors = new Map() +// Track attempt-to-fix test errors - suppress failures so Cypress exits 0, but report actual status to Datadog +const attemptToFixTestErrors = new Map() // Track the most recently loaded window in the AUT. Updated via the 'window:load' // event so we always get the real app window (after cy.visit()), not the @@ -72,7 +74,14 @@ Cypress.on('fail', (err, runnable) => { return } - // For all other tests (including attemptToFix), let the error propagate normally + // For attempt-to-fix tests, suppress the failure so Cypress exits with code 0 + // The actual failure status is still reported to Datadog in afterEach + if (isAttemptToFix) { + attemptToFixTestErrors.set(testName, err) + return + } + + // For all other tests, let the error propagate normally throw err }) @@ -247,13 +256,17 @@ afterEach(function () { const currentTest = Cypress.mocha.getRunner().suite.ctx.currentTest const testName = currentTest.fullTitle() - // Check if this was a quarantined test that we suppressed the failure for + // Check if this was a quarantined or attempt-to-fix test that we suppressed the failure for const quarantinedError = quarantinedTestErrors.get(testName) const isQuarantinedTestThatFailed = !!quarantinedError - - // For quarantined tests, convert Error to a serializable format for cy.task - const errorToReport = isQuarantinedTestThatFailed - ? { message: quarantinedError.message, stack: quarantinedError.stack } + const attemptToFixError = attemptToFixTestErrors.get(testName) + const isAttemptToFixTestThatFailed = !!attemptToFixError + const hasSuppressedFailure = isQuarantinedTestThatFailed || isAttemptToFixTestThatFailed + const suppressedError = quarantinedError || attemptToFixError + + // For tests with suppressed failures, convert Error to a serializable format for cy.task + const errorToReport = hasSuppressedFailure + ? { message: suppressedError.message, stack: suppressedError.stack } : currentTest.err const testInfo = { @@ -261,9 +274,9 @@ afterEach(function () { testItTitle: currentTest.title, testSuite: Cypress.mocha.getRootSuite().file, testSuiteAbsolutePath: Cypress.spec && Cypress.spec.absolute, - // For quarantined tests, report the actual state (failed) to Datadog, not what Cypress thinks (passed) - state: isQuarantinedTestThatFailed ? 'failed' : currentTest.state, - // For quarantined tests, include the actual error that was suppressed + // For tests with suppressed failures, report the actual state (failed) to Datadog + state: hasSuppressedFailure ? 'failed' : currentTest.state, + // Include the actual error that was suppressed error: errorToReport, isNew: currentTest._ddIsNew, isEfdRetry: currentTest._ddIsEfdRetry, @@ -294,10 +307,13 @@ afterEach(function () { // ignore error and continue } - // Clean up the quarantined error tracking + // Clean up suppressed error tracking if (isQuarantinedTestThatFailed) { quarantinedTestErrors.delete(testName) } + if (isAttemptToFixTestThatFailed) { + attemptToFixTestErrors.delete(testName) + } cy.task('dd:afterEach', { test: testInfo, coverage }) })