|
| 1 | +import { describe, expect, test } from 'bun:test' |
| 2 | +import { mkdtempSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs' |
| 3 | +import { tmpdir } from 'node:os' |
| 4 | +import { join } from 'node:path' |
| 5 | +import { |
| 6 | + LAB_ENVIRONMENTS, |
| 7 | + LAB_ENVIRONMENT_KEYS, |
| 8 | + labBatchInputSchema, |
| 9 | + labProbeCollided, |
| 10 | + nextLabBatchSlot, |
| 11 | + summarizeLabBatch, |
| 12 | + type LabBatch, |
| 13 | + type LabBatchRun, |
| 14 | + type LabRunGrade, |
| 15 | +} from './agentic-ad-lab-batch' |
| 16 | + |
| 17 | +const grade = ( |
| 18 | + tier: LabRunGrade['tier'], |
| 19 | + causeKey = tier === 'perfect' ? 'none' : 'missing_node', |
| 20 | + steps: [number, number] | null = [3, 3], |
| 21 | +): LabRunGrade => ({ |
| 22 | + tier, |
| 23 | + causeKey, |
| 24 | + cause: '', |
| 25 | + stepsCompleted: steps?.[0] ?? null, |
| 26 | + stepsTotal: steps?.[1] ?? null, |
| 27 | + grader: 'model', |
| 28 | + rubricVersion: 'v', |
| 29 | +}) |
| 30 | +const run = ( |
| 31 | + projectId: string, |
| 32 | + slot: number, |
| 33 | + extra: Partial<LabBatchRun> = {}, |
| 34 | +): LabBatchRun => ({ |
| 35 | + workspaceId: `${projectId}-${slot}`, |
| 36 | + projectId, |
| 37 | + slot, |
| 38 | + runId: `run-${projectId}-${slot}`, |
| 39 | + runStatus: 'completed', |
| 40 | + signup: 'idle', |
| 41 | + grade: null, |
| 42 | + probe: null, |
| 43 | + ...extra, |
| 44 | +}) |
| 45 | +const batch = ( |
| 46 | + runs: LabBatchRun[], |
| 47 | + extra: Partial<LabBatch> = {}, |
| 48 | +): LabBatch => ({ |
| 49 | + id: 'b', |
| 50 | + name: 'Archil', |
| 51 | + adName: 'Archil', |
| 52 | + procedureSha256: 'x', |
| 53 | + environment: 'default', |
| 54 | + signup: 'decline', |
| 55 | + repetitions: 10, |
| 56 | + repositories: [ |
| 57 | + { projectId: 'a', fullName: 'acme/a' }, |
| 58 | + { projectId: 'b', fullName: 'acme/b' }, |
| 59 | + { projectId: 'c', fullName: 'acme/c' }, |
| 60 | + ], |
| 61 | + state: 'running', |
| 62 | + createdAt: '2026-10-09T00:00:00.000Z', |
| 63 | + runs, |
| 64 | + ...extra, |
| 65 | +}) |
| 66 | + |
| 67 | +describe('summarizeLabBatch', () => { |
| 68 | + test('reports the perfect rate over graded runs with causes and steps reached', () => { |
| 69 | + const summary = summarizeLabBatch( |
| 70 | + batch([ |
| 71 | + run('a', 0, { grade: grade('perfect') }), |
| 72 | + run('a', 1, { grade: grade('partial', 'missing_node', [1, 3]) }), |
| 73 | + run('b', 0, { grade: grade('partial', 'missing_node', [1, 3]) }), |
| 74 | + run('b', 1, { grade: grade('handoff', 'needs_api_key', null) }), |
| 75 | + run('c', 0, { runStatus: 'running' }), |
| 76 | + ]), |
| 77 | + ) |
| 78 | + expect(summary).toEqual({ |
| 79 | + planned: 30, |
| 80 | + started: 5, |
| 81 | + graded: 4, |
| 82 | + perfect: 1, |
| 83 | + perfectRate: 0.25, |
| 84 | + byTier: { perfect: 1, handoff: 1, partial: 2, broken: 0, lost: 0 }, |
| 85 | + byCause: [ |
| 86 | + { causeKey: 'missing_node', count: 2 }, |
| 87 | + { causeKey: 'needs_api_key', count: 1 }, |
| 88 | + ], |
| 89 | + byStepReached: [ |
| 90 | + { reached: '1/3', count: 2 }, |
| 91 | + { reached: '3/3', count: 1 }, |
| 92 | + { reached: 'unknown', count: 1 }, |
| 93 | + ], |
| 94 | + collisions: null, |
| 95 | + }) |
| 96 | + }) |
| 97 | + |
| 98 | + test('has no rate before a run is graded', () => { |
| 99 | + expect(summarizeLabBatch(batch([])).perfectRate).toBeNull() |
| 100 | + }) |
| 101 | + |
| 102 | + test('counts package.json collisions only for runs the writer edited', () => { |
| 103 | + const probe = (lastEdit: number, markerInFile: number | null) => ({ |
| 104 | + kind: 'concurrent-package-json' as const, |
| 105 | + editsWritten: lastEdit, |
| 106 | + unreadable: 0, |
| 107 | + lastEdit: lastEdit || null, |
| 108 | + markerInFile, |
| 109 | + packageJsonValid: true, |
| 110 | + }) |
| 111 | + const summary = summarizeLabBatch( |
| 112 | + batch( |
| 113 | + [ |
| 114 | + run('a', 0, { probe: probe(5, 5) }), |
| 115 | + run('a', 1, { probe: probe(5, 4) }), |
| 116 | + run('a', 2, { probe: probe(0, null) }), |
| 117 | + run('a', 3), |
| 118 | + ], |
| 119 | + { environment: 'concurrent-package-json' }, |
| 120 | + ), |
| 121 | + ) |
| 122 | + expect(summary.collisions).toEqual({ measured: 2, collided: 1 }) |
| 123 | + }) |
| 124 | +}) |
| 125 | + |
| 126 | +test('labProbeCollided flags an invalid or unreadable file and a lost last edit', () => { |
| 127 | + const base = { |
| 128 | + kind: 'concurrent-package-json' as const, |
| 129 | + editsWritten: 3, |
| 130 | + unreadable: 0, |
| 131 | + lastEdit: 3, |
| 132 | + markerInFile: 3, |
| 133 | + packageJsonValid: true, |
| 134 | + } |
| 135 | + expect(labProbeCollided(base)).toBe(false) |
| 136 | + expect(labProbeCollided({ ...base, markerInFile: 2 })).toBe(true) |
| 137 | + expect(labProbeCollided({ ...base, packageJsonValid: false })).toBe(true) |
| 138 | + expect(labProbeCollided({ ...base, unreadable: 1 })).toBe(true) |
| 139 | + expect(labProbeCollided({ kind: 'setup-failed', error: 'x' })).toBeNull() |
| 140 | + expect(labProbeCollided(null)).toBeNull() |
| 141 | +}) |
| 142 | + |
| 143 | +test('nextLabBatchSlot fills each repetition across repositories before the next', () => { |
| 144 | + const b = batch([run('a', 0), run('b', 0)]) |
| 145 | + expect(nextLabBatchSlot(b)).toEqual({ projectId: 'c', slot: 0 }) |
| 146 | + expect(nextLabBatchSlot(b, new Set(['c']))).toEqual({ |
| 147 | + projectId: 'a', |
| 148 | + slot: 1, |
| 149 | + }) |
| 150 | + expect(nextLabBatchSlot({ ...b, repetitions: 1 }, new Set(['c']))).toBeNull() |
| 151 | +}) |
| 152 | + |
| 153 | +test('a batch input is bounded in runs and repositories', () => { |
| 154 | + const input = { |
| 155 | + name: 'Archil', |
| 156 | + selection: { kind: 'draft', id: crypto.randomUUID() }, |
| 157 | + projectIds: ['a', 'b', 'c'], |
| 158 | + repetitions: 10, |
| 159 | + environment: 'node-nvm', |
| 160 | + signup: 'decline', |
| 161 | + } |
| 162 | + expect(labBatchInputSchema.safeParse(input).success).toBe(true) |
| 163 | + expect( |
| 164 | + labBatchInputSchema.safeParse({ ...input, projectIds: ['a', 'a'] }).success, |
| 165 | + ).toBe(false) |
| 166 | + expect( |
| 167 | + labBatchInputSchema.safeParse({ |
| 168 | + ...input, |
| 169 | + projectIds: Array.from({ length: 6 }, (_, i) => `p${i}`), |
| 170 | + repetitions: 20, |
| 171 | + }).success, |
| 172 | + ).toBe(false) |
| 173 | + expect( |
| 174 | + labBatchInputSchema.safeParse({ ...input, environment: 'windows' }).success, |
| 175 | + ).toBe(false) |
| 176 | +}) |
| 177 | + |
| 178 | +describe('environment scripts', () => { |
| 179 | + const bash = (script: string, home: string, cwd = home, path = '') => |
| 180 | + Bun.spawnSync(['bash', '-c', script], { |
| 181 | + cwd, |
| 182 | + env: { |
| 183 | + HOME: home, |
| 184 | + PATH: `${path}${home}/.local/bin:${process.env.PATH}`, |
| 185 | + }, |
| 186 | + }) |
| 187 | + |
| 188 | + test.each(LAB_ENVIRONMENT_KEYS)('%s parses', (key) => { |
| 189 | + const { setup, probe } = LAB_ENVIRONMENTS[key] |
| 190 | + for (const script of [setup, probe]) |
| 191 | + if (script) |
| 192 | + expect(Bun.spawnSync(['bash', '-n', '-c', script]).exitCode).toBe(0) |
| 193 | + }) |
| 194 | + |
| 195 | + test('a toolchain case activates from the profile once and hides the system node', () => { |
| 196 | + const home = mkdtempSync(join(tmpdir(), 'lab-home-')) |
| 197 | + // A stand-in installer: curl prints a script that creates the mise binary. |
| 198 | + const bin = join(home, 'fake-bin') |
| 199 | + mkdirSync(bin) |
| 200 | + writeFileSync( |
| 201 | + join(bin, 'curl'), |
| 202 | + `#!/bin/sh\nprintf '%s\\n' 'mkdir -p "$(dirname "$MISE_INSTALL_PATH")"; printf "#!/bin/sh\\nexit 0\\n" > "$MISE_INSTALL_PATH"; chmod 755 "$MISE_INSTALL_PATH"'\n`, |
| 203 | + { mode: 0o755 }, |
| 204 | + ) |
| 205 | + const setup = LAB_ENVIRONMENTS['node-mise'].setup |
| 206 | + for (let i = 0; i < 2; i++) { |
| 207 | + const result = bash(setup, home, home, `${bin}:`) |
| 208 | + expect(result.stderr.toString()).toBe('') |
| 209 | + expect(result.exitCode).toBe(0) |
| 210 | + } |
| 211 | + const rc = readFileSync(join(home, '.bashrc'), 'utf8') |
| 212 | + expect(rc.match(/# freebuff-lab toolchain/g)).toHaveLength(1) |
| 213 | + expect(rc).toContain('eval "$("$MISE_DATA_DIR/bin/mise" activate bash)"') |
| 214 | + const node = bash('node --version', home) |
| 215 | + expect(node.exitCode).toBe(127) |
| 216 | + expect(node.stderr.toString()).toContain('node: not found') |
| 217 | + }) |
| 218 | + |
| 219 | + test('the collision probe reports the writer’s last edit against package.json', () => { |
| 220 | + const home = mkdtempSync(join(tmpdir(), 'lab-home-')) |
| 221 | + const repo = join(home, 'repo') |
| 222 | + mkdirSync(join(home, '.cache/freebuff-lab'), { recursive: true }) |
| 223 | + mkdirSync(repo) |
| 224 | + writeFileSync(join(home, '.cache/freebuff-lab/edits'), '1\n2\n3\n') |
| 225 | + writeFileSync( |
| 226 | + join(repo, 'package.json'), |
| 227 | + JSON.stringify({ name: 'x', freebuffLabConcurrentEdit: 2 }), |
| 228 | + ) |
| 229 | + const result = bash( |
| 230 | + LAB_ENVIRONMENTS['concurrent-package-json'].probe, |
| 231 | + home, |
| 232 | + repo, |
| 233 | + ) |
| 234 | + const probe = JSON.parse(result.stdout.toString().trim()) |
| 235 | + expect(probe).toEqual({ |
| 236 | + kind: 'concurrent-package-json', |
| 237 | + editsWritten: 3, |
| 238 | + unreadable: 0, |
| 239 | + lastEdit: 3, |
| 240 | + markerInFile: 2, |
| 241 | + packageJsonValid: true, |
| 242 | + }) |
| 243 | + expect(labProbeCollided(probe)).toBe(true) |
| 244 | + }) |
| 245 | + |
| 246 | + test('the collision case refuses a repository without package.json', () => { |
| 247 | + const home = mkdtempSync(join(tmpdir(), 'lab-home-')) |
| 248 | + const result = bash(LAB_ENVIRONMENTS['concurrent-package-json'].setup, home) |
| 249 | + expect(result.exitCode).toBe(3) |
| 250 | + }) |
| 251 | +}) |
0 commit comments