diff --git a/integration-tests/ci-visibility/test-management/test-attempt-to-fix-parallel-1.js b/integration-tests/ci-visibility/test-management/test-attempt-to-fix-parallel-1.js new file mode 100644 index 00000000000..aa67382dd4e --- /dev/null +++ b/integration-tests/ci-visibility/test-management/test-attempt-to-fix-parallel-1.js @@ -0,0 +1,9 @@ +'use strict' + +const assert = require('assert') + +describe('attempt to fix parallel tests 1', () => { + it('can attempt to fix a test', () => { + assert.strictEqual(1 + 2, 4) + }) +}) diff --git a/integration-tests/ci-visibility/test-management/test-attempt-to-fix-parallel-2.js b/integration-tests/ci-visibility/test-management/test-attempt-to-fix-parallel-2.js new file mode 100644 index 00000000000..af3e88de361 --- /dev/null +++ b/integration-tests/ci-visibility/test-management/test-attempt-to-fix-parallel-2.js @@ -0,0 +1,9 @@ +'use strict' + +const assert = require('assert') + +describe('attempt to fix parallel tests 2', () => { + it('can attempt to fix a test', () => { + assert.strictEqual(1 + 2, 4) + }) +}) diff --git a/integration-tests/mocha/mocha.spec.js b/integration-tests/mocha/mocha.spec.js index 260267ca7b1..4b882ed8793 100644 --- a/integration-tests/mocha/mocha.spec.js +++ b/integration-tests/mocha/mocha.spec.js @@ -4012,6 +4012,86 @@ describe(`mocha@${MOCHA_VERSION}`, function () { runAttemptToFixTest(done, { isAttemptToFix: true, isDisabled: true }) }) + + onlyLatestIt('can attempt to fix in parallel mode', async () => { + const NUM_RETRIES = 3 + receiver.setSettings({ test_management: { enabled: true, attempt_to_fix_retries: NUM_RETRIES } }) + receiver.setTestManagementTests({ + mocha: { + suites: { + 'ci-visibility/test-management/test-attempt-to-fix-parallel-1.js': { + tests: { + 'attempt to fix parallel tests 1 can attempt to fix a test': { + properties: { attempt_to_fix: true }, + }, + }, + }, + 'ci-visibility/test-management/test-attempt-to-fix-parallel-2.js': { + tests: { + 'attempt to fix parallel tests 2 can attempt to fix a test': { + properties: { attempt_to_fix: true }, + }, + }, + }, + }, + }, + }) + + const eventsPromise = receiver + .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => { + const events = payloads.flatMap(({ payload }) => payload.events) + const tests = events.filter(event => event.type === 'test').map(event => event.content) + + const sessionEvent = events.find(event => event.type === 'test_session_end').content + assert.strictEqual(sessionEvent.meta[MOCHA_IS_PARALLEL], 'true') + assert.strictEqual(sessionEvent.meta[TEST_MANAGEMENT_ENABLED], 'true') + + // Each file: 1 initial attempt + NUM_RETRIES retries = (NUM_RETRIES + 1) per file, 2 files + assert.strictEqual(tests.length, (NUM_RETRIES + 1) * 2) + + // All attempts fail (tests always throw) + tests.forEach(test => { + assert.strictEqual(test.meta[TEST_STATUS], 'fail') + assert.strictEqual(test.meta[TEST_MANAGEMENT_IS_ATTEMPT_TO_FIX], 'true') + }) + + // Last attempt of each test should have failed-all-retries + const testsBySuite = {} + for (const test of tests) { + const suite = test.meta[TEST_SUITE] + if (!testsBySuite[suite]) testsBySuite[suite] = [] + testsBySuite[suite].push(test) + } + for (const suiteTests of Object.values(testsBySuite)) { + const lastAttempt = suiteTests[suiteTests.length - 1] + assert.strictEqual(lastAttempt.meta[TEST_HAS_FAILED_ALL_RETRIES], 'true') + assert.strictEqual(lastAttempt.meta[TEST_MANAGEMENT_ATTEMPT_TO_FIX_PASSED], 'false') + } + + // Verify separate worker processes + const firstTestPerSuite = Object.values(testsBySuite).map(t => t[0]) + assert.strictEqual(firstTestPerSuite.length, 2) + const runtimeIds = firstTestPerSuite.map(t => t.meta['runtime-id']) + assert.ok(runtimeIds[0]) + assert.ok(runtimeIds[1]) + assert.notStrictEqual(runtimeIds[0], runtimeIds[1]) + }) + + childProcess = exec( + 'node node_modules/mocha/bin/mocha --parallel --jobs 2' + + ' ./ci-visibility/test-management/test-attempt-to-fix-parallel-1.js' + + ' ./ci-visibility/test-management/test-attempt-to-fix-parallel-2.js', + { + cwd, + env: getCiVisAgentlessConfig(receiver.port), + } + ) + + await Promise.all([ + eventsPromise, + once(childProcess, 'exit'), + ]) + }) }) context('disabled', () => { diff --git a/packages/datadog-instrumentations/src/mocha/main.js b/packages/datadog-instrumentations/src/mocha/main.js index c6ba716def1..69cd4512c38 100644 --- a/packages/datadog-instrumentations/src/mocha/main.js +++ b/packages/datadog-instrumentations/src/mocha/main.js @@ -694,8 +694,7 @@ addHook({ if (config.isTestManagementTestsEnabled) { const testSuiteTestManagementTests = config.testManagementTests?.mocha?.suites?.[testPath] || {} newWorkerArgs._ddIsTestManagementTestsEnabled = true - // TODO: attempt to fix does not work in parallel mode yet - // newWorkerArgs._ddTestManagementAttemptToFixRetries = config.testManagementAttemptToFixRetries + newWorkerArgs._ddTestManagementAttemptToFixRetries = config.testManagementAttemptToFixRetries newWorkerArgs._ddTestManagementTests = { mocha: { suites: { diff --git a/packages/datadog-instrumentations/src/mocha/utils.js b/packages/datadog-instrumentations/src/mocha/utils.js index 6ba5a287d73..54644660bc9 100644 --- a/packages/datadog-instrumentations/src/mocha/utils.js +++ b/packages/datadog-instrumentations/src/mocha/utils.js @@ -140,7 +140,6 @@ function runnableWrapper (RunnablePackage, libraryConfig) { if (!testFinishCh.hasSubscribers) { return run.apply(this, arguments) } - // Flaky test retries does not work in parallel mode if (libraryConfig?.isFlakyTestRetriesEnabled) { this.retries(libraryConfig?.flakyTestRetriesCount) } diff --git a/packages/datadog-instrumentations/src/mocha/worker.js b/packages/datadog-instrumentations/src/mocha/worker.js index 224d5e32deb..b7de0006172 100644 --- a/packages/datadog-instrumentations/src/mocha/worker.js +++ b/packages/datadog-instrumentations/src/mocha/worker.js @@ -43,10 +43,10 @@ addHook({ } if (this.options._ddIsTestManagementTestsEnabled) { config.isTestManagementTestsEnabled = true - // TODO: attempt to fix does not work in parallel mode yet - // config.testManagementAttemptToFixRetries = this.options._ddTestManagementAttemptToFixRetries + config.testManagementAttemptToFixRetries = this.options._ddTestManagementAttemptToFixRetries config.testManagementTests = this.options._ddTestManagementTests delete this.options._ddIsTestManagementTestsEnabled + delete this.options._ddTestManagementAttemptToFixRetries delete this.options._ddTestManagementTests } if (this.options._ddIsFlakyTestRetriesEnabled) { diff --git a/packages/dd-trace/src/plugins/util/test.js b/packages/dd-trace/src/plugins/util/test.js index 23d940b1c41..ab5eca5c622 100644 --- a/packages/dd-trace/src/plugins/util/test.js +++ b/packages/dd-trace/src/plugins/util/test.js @@ -170,7 +170,7 @@ const MINIMUM_FRAMEWORK_VERSION_FOR_FAILED_TEST_REPLAY = { playwright: '>=1.38.0', } -const UNSUPPORTED_ATTEMPT_TO_FIX_FRAMEWORKS_PARALLEL_MODE = new Set(['mocha']) +const UNSUPPORTED_ATTEMPT_TO_FIX_FRAMEWORKS_PARALLEL_MODE = new Set([]) const NOT_SUPPORTED_GRANULARITY_IMPACTED_TESTS_FRAMEWORKS = new Set(['mocha', 'playwright', 'vitest']) const TEST_LEVEL_EVENT_TYPES = [