Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
'use strict'

const assert = require('assert')

describe('attempt to fix parallel tests 1', () => {
it('can attempt to fix a test', () => {
assert.strictEqual(1 + 2, 4)
})
})
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
'use strict'

const assert = require('assert')

describe('attempt to fix parallel tests 2', () => {
it('can attempt to fix a test', () => {
assert.strictEqual(1 + 2, 4)
})
})
80 changes: 80 additions & 0 deletions integration-tests/mocha/mocha.spec.js
Original file line number Diff line number Diff line change
Expand Up @@ -4012,6 +4012,86 @@ describe(`mocha@${MOCHA_VERSION}`, function () {

runAttemptToFixTest(done, { isAttemptToFix: true, isDisabled: true })
})

onlyLatestIt('can attempt to fix in parallel mode', async () => {
const NUM_RETRIES = 3
receiver.setSettings({ test_management: { enabled: true, attempt_to_fix_retries: NUM_RETRIES } })
receiver.setTestManagementTests({
mocha: {
suites: {
'ci-visibility/test-management/test-attempt-to-fix-parallel-1.js': {
tests: {
'attempt to fix parallel tests 1 can attempt to fix a test': {
properties: { attempt_to_fix: true },
},
},
},
'ci-visibility/test-management/test-attempt-to-fix-parallel-2.js': {
tests: {
'attempt to fix parallel tests 2 can attempt to fix a test': {
properties: { attempt_to_fix: true },
},
},
},
},
},
})

const eventsPromise = receiver
.gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => {
const events = payloads.flatMap(({ payload }) => payload.events)
const tests = events.filter(event => event.type === 'test').map(event => event.content)

const sessionEvent = events.find(event => event.type === 'test_session_end').content
assert.strictEqual(sessionEvent.meta[MOCHA_IS_PARALLEL], 'true')
assert.strictEqual(sessionEvent.meta[TEST_MANAGEMENT_ENABLED], 'true')

// Each file: 1 initial attempt + NUM_RETRIES retries = (NUM_RETRIES + 1) per file, 2 files
assert.strictEqual(tests.length, (NUM_RETRIES + 1) * 2)

// All attempts fail (tests always throw)
tests.forEach(test => {
assert.strictEqual(test.meta[TEST_STATUS], 'fail')
assert.strictEqual(test.meta[TEST_MANAGEMENT_IS_ATTEMPT_TO_FIX], 'true')
})

// Last attempt of each test should have failed-all-retries
const testsBySuite = {}
for (const test of tests) {
const suite = test.meta[TEST_SUITE]
if (!testsBySuite[suite]) testsBySuite[suite] = []
testsBySuite[suite].push(test)
}
for (const suiteTests of Object.values(testsBySuite)) {
const lastAttempt = suiteTests[suiteTests.length - 1]
assert.strictEqual(lastAttempt.meta[TEST_HAS_FAILED_ALL_RETRIES], 'true')
assert.strictEqual(lastAttempt.meta[TEST_MANAGEMENT_ATTEMPT_TO_FIX_PASSED], 'false')
}

// Verify separate worker processes
const firstTestPerSuite = Object.values(testsBySuite).map(t => t[0])
assert.strictEqual(firstTestPerSuite.length, 2)
const runtimeIds = firstTestPerSuite.map(t => t.meta['runtime-id'])
assert.ok(runtimeIds[0])
assert.ok(runtimeIds[1])
assert.notStrictEqual(runtimeIds[0], runtimeIds[1])
})

childProcess = exec(
'node node_modules/mocha/bin/mocha --parallel --jobs 2' +
' ./ci-visibility/test-management/test-attempt-to-fix-parallel-1.js' +
' ./ci-visibility/test-management/test-attempt-to-fix-parallel-2.js',
{
cwd,
env: getCiVisAgentlessConfig(receiver.port),
}
)

await Promise.all([
eventsPromise,
once(childProcess, 'exit'),
])
})
})

context('disabled', () => {
Expand Down
3 changes: 1 addition & 2 deletions packages/datadog-instrumentations/src/mocha/main.js
Original file line number Diff line number Diff line change
Expand Up @@ -694,8 +694,7 @@ addHook({
if (config.isTestManagementTestsEnabled) {
const testSuiteTestManagementTests = config.testManagementTests?.mocha?.suites?.[testPath] || {}
newWorkerArgs._ddIsTestManagementTestsEnabled = true
// TODO: attempt to fix does not work in parallel mode yet
// newWorkerArgs._ddTestManagementAttemptToFixRetries = config.testManagementAttemptToFixRetries
newWorkerArgs._ddTestManagementAttemptToFixRetries = config.testManagementAttemptToFixRetries
newWorkerArgs._ddTestManagementTests = {
mocha: {
suites: {
Expand Down
1 change: 0 additions & 1 deletion packages/datadog-instrumentations/src/mocha/utils.js
Original file line number Diff line number Diff line change
Expand Up @@ -140,7 +140,6 @@ function runnableWrapper (RunnablePackage, libraryConfig) {
if (!testFinishCh.hasSubscribers) {
return run.apply(this, arguments)
}
// Flaky test retries does not work in parallel mode
if (libraryConfig?.isFlakyTestRetriesEnabled) {
this.retries(libraryConfig?.flakyTestRetriesCount)
}
Expand Down
4 changes: 2 additions & 2 deletions packages/datadog-instrumentations/src/mocha/worker.js
Original file line number Diff line number Diff line change
Expand Up @@ -43,10 +43,10 @@ addHook({
}
if (this.options._ddIsTestManagementTestsEnabled) {
config.isTestManagementTestsEnabled = true
// TODO: attempt to fix does not work in parallel mode yet
// config.testManagementAttemptToFixRetries = this.options._ddTestManagementAttemptToFixRetries
config.testManagementAttemptToFixRetries = this.options._ddTestManagementAttemptToFixRetries
config.testManagementTests = this.options._ddTestManagementTests
delete this.options._ddIsTestManagementTestsEnabled
delete this.options._ddTestManagementAttemptToFixRetries
delete this.options._ddTestManagementTests
}
if (this.options._ddIsFlakyTestRetriesEnabled) {
Expand Down
2 changes: 1 addition & 1 deletion packages/dd-trace/src/plugins/util/test.js
Original file line number Diff line number Diff line change
Expand Up @@ -170,7 +170,7 @@
playwright: '>=1.38.0',
}

const UNSUPPORTED_ATTEMPT_TO_FIX_FRAMEWORKS_PARALLEL_MODE = new Set(['mocha'])
const UNSUPPORTED_ATTEMPT_TO_FIX_FRAMEWORKS_PARALLEL_MODE = new Set([])

Check failure on line 173 in packages/dd-trace/src/plugins/util/test.js

View workflow job for this annotation

GitHub Actions / lint

The empty array is useless
const NOT_SUPPORTED_GRANULARITY_IMPACTED_TESTS_FRAMEWORKS = new Set(['mocha', 'playwright', 'vitest'])

const TEST_LEVEL_EVENT_TYPES = [
Expand Down
Loading