From 03c0f8d8a60a8e8d141952f99d3aaa8aeda1641e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Wed, 11 Nov 2026 11:33:12 +0100 Subject: [PATCH] ci: run iOS and macOS smoke on independent Blacksmith runners --- .github/workflows/apple.yml | 60 +++++++++++----- scripts/__tests__/apple-ci-impact.test.ts | 85 ++++++++++++++++++++--- scripts/gate/workflows.test.ts | 12 +++- 3 files changed, 125 insertions(+), 32 deletions(-) diff --git a/.github/workflows/apple.yml b/.github/workflows/apple.yml index 4852c2c347..f2b77cef84 100644 --- a/.github/workflows/apple.yml +++ b/.github/workflows/apple.yml @@ -26,12 +26,15 @@ concurrency: group: ci-${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: - smoke-apple: - name: Smoke Tests + smoke-apple-platform: + name: Smoke Tests (${{ matrix.platform }}) + strategy: + fail-fast: false + matrix: + platform: [ios, macos] if: ${{ !github.event.pull_request.draft }} runs-on: blacksmith-6vcpu-macos-26 - # Keep each platform's former 80-minute ceiling, plus shared setup and cleanup. - timeout-minutes: 170 + timeout-minutes: 90 env: BASE_SHA: ${{ github.event.pull_request.base.sha }} IOS_STATE_DIR: ${{ github.workspace }}/.tmp/ios-agent-device-state @@ -55,18 +58,31 @@ jobs: - name: Select iOS smoke tests id: ios-impact shell: bash - run: node --experimental-strip-types scripts/apple-ci-impact.ts smoke + env: + APPLE_PLATFORM: ${{ matrix.platform }} + run: | + if [ "$APPLE_PLATFORM" = ios ]; then + node --experimental-strip-types scripts/apple-ci-impact.ts smoke + else + echo "run=false" >> "$GITHUB_OUTPUT" + fi - name: Select macOS smoke tests id: macos-impact shell: bash - run: node --experimental-strip-types scripts/apple-ci-impact.ts macos + env: + APPLE_PLATFORM: ${{ matrix.platform }} + run: | + if [ "$APPLE_PLATFORM" = macos ]; then + node --experimental-strip-types scripts/apple-ci-impact.ts macos + else + echo "run=false" >> "$GITHUB_OUTPUT" + fi - name: Build the shared CLI id: build if: steps.ios-impact.outputs.run != 'false' || steps.macos-impact.outputs.run != 'false' uses: ./.github/actions/run-gate with: gate: build - # The macOS clean-install proof may raise permission UI. Run the iOS focus proof first. - name: iOS smoke tests id: ios uses: ./.github/actions/smoke-ios-run @@ -82,9 +98,9 @@ jobs: AGENT_DEVICE_XCUITEST_ARCHS: arm64 AGENT_DEVICE_IOS_APP_EVENT_URL_TEMPLATE: agent-device-test-app:///automation?event={event}&payload={payload} if: ${{ !cancelled() && steps.build.outcome == 'success' && steps.ios-impact.outputs.run != 'false' }} - - name: Restore the Apple host after iOS + - name: Clean the iOS host id: ios-cleanup - if: ${{ always() && steps.build.outcome == 'success' }} + if: ${{ always() && steps.build.outcome == 'success' && steps.ios-impact.outputs.run != 'false' }} continue-on-error: true env: AGENT_DEVICE_STATE_DIR: ${{ env.IOS_STATE_DIR }} @@ -102,7 +118,7 @@ jobs: RESULT_BUNDLE_PATH: ${{ github.workspace }}/.tmp/xctest-host/RunnerTests.xcresult RESULT_SUMMARY_PATH: ${{ github.workspace }}/.tmp/xctest-host/test-results-summary.json XCTEST_LANE: host - if: ${{ !cancelled() && steps.build.outcome == 'success' && steps.ios-cleanup.outcome == 'success' && steps.macos-impact.outputs.run != 'false' }} + if: ${{ !cancelled() && steps.build.outcome == 'success' && steps.macos-impact.outputs.run != 'false' }} - name: Diagnose the Apple host after a phase failure if: ${{ always() && (steps.ios.outcome == 'failure' || steps.macos.outcome == 'failure' || steps.ios-cleanup.outcome == 'failure') }} continue-on-error: true @@ -113,7 +129,7 @@ jobs: osascript scripts/decline-hosted-compute-local-network.applescript diagnose - name: Clean macOS daemon and retained runners id: macos-cleanup - if: ${{ always() && steps.build.outcome == 'success' }} + if: ${{ always() && steps.build.outcome == 'success' && steps.macos-impact.outputs.run != 'false' }} continue-on-error: true env: AGENT_DEVICE_STATE_DIR: ${{ env.MACOS_STATE_DIR }} @@ -135,26 +151,23 @@ jobs: printf "### Apple smoke phases\n\n| Phase | Outcome |\n| --- | --- |\n| iOS | %s |\n| macOS | %s |\n" "$IOS_OUTCOME" "$MACOS_OUTCOME" >> "$GITHUB_STEP_SUMMARY" if [ "$IOS_SELECTED" = false ]; then test "$IOS_OUTCOME" = skipped + test "$IOS_CLEANUP_OUTCOME" = skipped else test "$IOS_OUTCOME" = success + test "$IOS_CLEANUP_OUTCOME" = success fi if [ "$MACOS_SELECTED" = false ]; then test "$MACOS_OUTCOME" = skipped - else - test "$MACOS_OUTCOME" = success - fi - if [ "$IOS_SELECTED" = false ] && [ "$MACOS_SELECTED" = false ]; then - test "$IOS_CLEANUP_OUTCOME" = skipped test "$MACOS_CLEANUP_OUTCOME" = skipped else - test "$IOS_CLEANUP_OUTCOME" = success + test "$MACOS_OUTCOME" = success test "$MACOS_CLEANUP_OUTCOME" = success fi - name: Upload Apple artifacts if: always() uses: ./.github/actions/upload-agent-device-artifacts with: - artifact-name: apple-artifacts + artifact-name: apple-artifacts-${{ matrix.platform }} agent-state-dir: ${{ env.IOS_STATE_DIR }} runner-derived-path: ${{ env.IOS_DERIVED_PATH }} additional-paths: | @@ -162,3 +175,14 @@ jobs: ${{ env.MACOS_STATE_DIR }}/sessions/** ${{ env.MACOS_DERIVED_PATH }}/.agent-device-runner-cache.json ${{ env.MACOS_DERIVED_PATH }}/Logs/** + + smoke-apple: + name: Smoke Tests + if: ${{ always() && !github.event.pull_request.draft }} + needs: smoke-apple-platform + runs-on: blacksmith-4vcpu-ubuntu-2404 + steps: + - name: Require both Apple jobs to succeed + env: + RESULT: ${{ needs.smoke-apple-platform.result }} + run: test "$RESULT" = success diff --git a/scripts/__tests__/apple-ci-impact.test.ts b/scripts/__tests__/apple-ci-impact.test.ts index 8c1d212e41..49e0e1aa7c 100644 --- a/scripts/__tests__/apple-ci-impact.test.ts +++ b/scripts/__tests__/apple-ci-impact.test.ts @@ -25,18 +25,31 @@ type AppleStep = { 'continue-on-error'?: boolean; }; -function appleJob() { +function appleWorkflow() { const doc = parse( fs.readFileSync(path.join(repoRoot, '.github/workflows/apple.yml'), 'utf8'), ) as { - jobs: Record; + jobs: Record< + string, + { + steps: AppleStep[]; + strategy?: { matrix: { platform: string[] }; 'fail-fast': boolean }; + needs?: string; + if?: string; + } + >; }; - expect(Object.keys(doc.jobs)).toHaveLength(1); - return Object.values(doc.jobs)[0]!; + expect(Object.keys(doc.jobs)).toHaveLength(2); + return doc; } -test('Apple smoke shares setup and build while keeping platform state and native products separate', () => { - const { steps } = appleJob(); +function appleJob() { + return appleWorkflow().jobs['smoke-apple-platform']!; +} + +test('Apple smoke gives each platform an independent runner and cleanup', () => { + const { steps, strategy } = appleJob(); + expect(strategy).toEqual({ 'fail-fast': false, matrix: { platform: ['ios', 'macos'] } }); expect(steps.filter((step) => step.uses?.startsWith('actions/checkout@'))).toHaveLength(1); expect(steps.filter((step) => step.uses === './.github/actions/setup-node-pnpm')).toHaveLength(1); expect(steps.filter((step) => step.with?.gate === 'build')).toHaveLength(1); @@ -44,8 +57,7 @@ test('Apple smoke shares setup and build while keeping platform state and native const macos = steps.find((step) => step.id === 'macos')!; expect(ios.uses).toBe('./.github/actions/smoke-ios-run'); expect(macos.uses).toBe('./.github/actions/smoke-macos'); - expect(steps.indexOf(ios)).toBeLessThan(steps.indexOf(macos)); - expect(macos.if).toContain("steps.ios-cleanup.outcome == 'success'"); + expect(macos.if).not.toContain('steps.ios-cleanup'); for (const variable of ['AGENT_DEVICE_STATE_DIR', 'AGENT_DEVICE_IOS_RUNNER_DERIVED_PATH']) { expect(ios.env?.[variable]).toBeTruthy(); expect(macos.env?.[variable]).toBeTruthy(); @@ -55,7 +67,7 @@ test('Apple smoke shares setup and build while keeping platform state and native expect(phase['continue-on-error']).toBe(true); const cleanup = steps.find((step) => step.id === `${phase.id}-cleanup`)!; expect(steps.indexOf(cleanup)).toBeGreaterThan(steps.indexOf(phase)); - if (phase === ios) expect(steps.indexOf(cleanup)).toBeLessThan(steps.indexOf(macos)); + expect(cleanup.if).toContain(`steps.${phase.id}-impact.outputs.run != 'false'`); expect(cleanup.if).toContain('always()'); expect(cleanup['continue-on-error']).toBe(true); expect(cleanup.env?.AGENT_DEVICE_STATE_DIR).toBe(phase.env?.AGENT_DEVICE_STATE_DIR); @@ -65,6 +77,49 @@ test('Apple smoke shares setup and build while keeping platform state and native } }); +test.each(['ios', 'macos'] as const)('the %s job selects only its own platform', (platform) => { + for (const target of ['ios', 'macos']) { + const step = appleJob().steps.find((step) => step.id === `${target}-impact`)!; + expect(step.env?.APPLE_PLATFORM).toBe('${{ matrix.platform }}'); + const output = path.join(mkdtempForTestSync('apple-platform-selection-'), 'output'); + const result = spawnSync( + '/bin/bash', + [ + '-c', + ` + set -euo pipefail + node() { echo 'run=true' >> "$GITHUB_OUTPUT"; } + ${step.run} + `, + ], + { + env: { ...process.env, APPLE_PLATFORM: platform, GITHUB_OUTPUT: output }, + encoding: 'utf8', + }, + ); + expect(result.status, result.stderr).toBe(0); + expect(fs.readFileSync(output, 'utf8').trim()).toBe(`run=${target === platform}`); + } +}); + +test.each([ + ['success', 0], + ['failure', 1], + ['cancelled', 1], + ['skipped', 1], +])('the stable Apple status requires matrix success: %s', (outcome, expected) => { + const job = appleWorkflow().jobs['smoke-apple']!; + expect(job.needs).toBe('smoke-apple-platform'); + expect(job.if).toBe('${{ always() && !github.event.pull_request.draft }}'); + const step = job.steps[0]!; + expect(step.env?.RESULT).toBe('${{ needs.smoke-apple-platform.result }}'); + const result = spawnSync('/bin/bash', ['-c', step.run!], { + env: { ...process.env, RESULT: outcome }, + encoding: 'utf8', + }); + expect(result.status, result.stderr).toBe(expected); +}); + test.runIf(process.platform === 'darwin')( 'hosted prompt cleanup refuses local execution before inspecting UI', () => { @@ -222,7 +277,11 @@ test('Apple uploads shared diagnostics once and retains both platform logs after test.each([ ['all successful', {}, 0], - ['iOS not selected', { IOS_SELECTED: 'false', IOS_OUTCOME: 'skipped' }, 0], + [ + 'iOS not selected', + { IOS_SELECTED: 'false', IOS_OUTCOME: 'skipped', IOS_CLEANUP_OUTCOME: 'skipped' }, + 0, + ], ['unselected iOS unexpectedly ran', { IOS_SELECTED: 'false' }, 1], ['selected iOS fails', { IOS_SELECTED: 'true', IOS_OUTCOME: 'failure' }, 1], ['iOS fails', { IOS_OUTCOME: 'failure' }, 1], @@ -231,7 +290,11 @@ test.each([ ['macOS silently skipped', { MACOS_OUTCOME: 'skipped' }, 1], ['iOS cleanup fails', { IOS_CLEANUP_OUTCOME: 'failure' }, 1], ['macOS cleanup fails', { MACOS_CLEANUP_OUTCOME: 'failure' }, 1], - ['macOS not selected', { MACOS_SELECTED: 'false', MACOS_OUTCOME: 'skipped' }, 0], + [ + 'macOS not selected', + { MACOS_SELECTED: 'false', MACOS_OUTCOME: 'skipped', MACOS_CLEANUP_OUTCOME: 'skipped' }, + 0, + ], ['unselected macOS unexpectedly ran', { MACOS_SELECTED: 'false' }, 1], [ 'no phase selected', diff --git a/scripts/gate/workflows.test.ts b/scripts/gate/workflows.test.ts index 02c25689d9..fad6ac94d0 100644 --- a/scripts/gate/workflows.test.ts +++ b/scripts/gate/workflows.test.ts @@ -322,19 +322,20 @@ test('device smoke lanes skip drafts and start when a draft is marked ready', () type Workflow = { name: string; on: { pull_request?: { types?: string[] } }; - jobs: Record; + jobs: Record; }; const smokeLanes = fs .readdirSync(directory) .map((file) => parse(fs.readFileSync(path.join(directory, file), 'utf8')) as Workflow) .flatMap((workflow) => Object.values(workflow.jobs) - .filter((job) => job.name === 'Smoke Tests' && workflow.on.pull_request) + .filter((job) => job.name?.startsWith('Smoke Tests') && workflow.on.pull_request) .map((job) => ({ workflow, job })), ); assert.deepEqual(smokeLanes.map(({ workflow }) => workflow.name).sort(), [ 'Android', 'Apple', + 'Apple', 'Linux', ]); for (const { workflow, job } of smokeLanes) { @@ -343,6 +344,11 @@ test('device smoke lanes skip drafts and start when a draft is marked ready', () ['opened', 'synchronize', 'reopened', 'ready_for_review', 'converted_to_draft'], workflow.name, ); - assert.equal(job.if, '${{ !github.event.pull_request.draft }}', workflow.name); + if (workflow.name === 'Apple' && job.name === 'Smoke Tests') { + assert.equal(job.needs, 'smoke-apple-platform'); + assert.equal(job.if, '${{ always() && !github.event.pull_request.draft }}'); + } else { + assert.equal(job.if, '${{ !github.event.pull_request.draft }}', workflow.name); + } } });