Skip to content

Commit c72fcd0

Browse files
Sync public snapshot from freebuff-private
Source: CodebuffAI/freebuff-private@062268582eb369cbb65774f26ebcd7fe7f9cdc96
1 parent 0e73b52 commit c72fcd0

2 files changed

Lines changed: 592 additions & 0 deletions

File tree

Lines changed: 251 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,251 @@
1+
import { describe, expect, test } from 'bun:test'
2+
import { mkdtempSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs'
3+
import { tmpdir } from 'node:os'
4+
import { join } from 'node:path'
5+
import {
6+
LAB_ENVIRONMENTS,
7+
LAB_ENVIRONMENT_KEYS,
8+
labBatchInputSchema,
9+
labProbeCollided,
10+
nextLabBatchSlot,
11+
summarizeLabBatch,
12+
type LabBatch,
13+
type LabBatchRun,
14+
type LabRunGrade,
15+
} from './agentic-ad-lab-batch'
16+
17+
const grade = (
18+
tier: LabRunGrade['tier'],
19+
causeKey = tier === 'perfect' ? 'none' : 'missing_node',
20+
steps: [number, number] | null = [3, 3],
21+
): LabRunGrade => ({
22+
tier,
23+
causeKey,
24+
cause: '',
25+
stepsCompleted: steps?.[0] ?? null,
26+
stepsTotal: steps?.[1] ?? null,
27+
grader: 'model',
28+
rubricVersion: 'v',
29+
})
30+
const run = (
31+
projectId: string,
32+
slot: number,
33+
extra: Partial<LabBatchRun> = {},
34+
): LabBatchRun => ({
35+
workspaceId: `${projectId}-${slot}`,
36+
projectId,
37+
slot,
38+
runId: `run-${projectId}-${slot}`,
39+
runStatus: 'completed',
40+
signup: 'idle',
41+
grade: null,
42+
probe: null,
43+
...extra,
44+
})
45+
const batch = (
46+
runs: LabBatchRun[],
47+
extra: Partial<LabBatch> = {},
48+
): LabBatch => ({
49+
id: 'b',
50+
name: 'Archil',
51+
adName: 'Archil',
52+
procedureSha256: 'x',
53+
environment: 'default',
54+
signup: 'decline',
55+
repetitions: 10,
56+
repositories: [
57+
{ projectId: 'a', fullName: 'acme/a' },
58+
{ projectId: 'b', fullName: 'acme/b' },
59+
{ projectId: 'c', fullName: 'acme/c' },
60+
],
61+
state: 'running',
62+
createdAt: '2026-10-09T00:00:00.000Z',
63+
runs,
64+
...extra,
65+
})
66+
67+
describe('summarizeLabBatch', () => {
68+
test('reports the perfect rate over graded runs with causes and steps reached', () => {
69+
const summary = summarizeLabBatch(
70+
batch([
71+
run('a', 0, { grade: grade('perfect') }),
72+
run('a', 1, { grade: grade('partial', 'missing_node', [1, 3]) }),
73+
run('b', 0, { grade: grade('partial', 'missing_node', [1, 3]) }),
74+
run('b', 1, { grade: grade('handoff', 'needs_api_key', null) }),
75+
run('c', 0, { runStatus: 'running' }),
76+
]),
77+
)
78+
expect(summary).toEqual({
79+
planned: 30,
80+
started: 5,
81+
graded: 4,
82+
perfect: 1,
83+
perfectRate: 0.25,
84+
byTier: { perfect: 1, handoff: 1, partial: 2, broken: 0, lost: 0 },
85+
byCause: [
86+
{ causeKey: 'missing_node', count: 2 },
87+
{ causeKey: 'needs_api_key', count: 1 },
88+
],
89+
byStepReached: [
90+
{ reached: '1/3', count: 2 },
91+
{ reached: '3/3', count: 1 },
92+
{ reached: 'unknown', count: 1 },
93+
],
94+
collisions: null,
95+
})
96+
})
97+
98+
test('has no rate before a run is graded', () => {
99+
expect(summarizeLabBatch(batch([])).perfectRate).toBeNull()
100+
})
101+
102+
test('counts package.json collisions only for runs the writer edited', () => {
103+
const probe = (lastEdit: number, markerInFile: number | null) => ({
104+
kind: 'concurrent-package-json' as const,
105+
editsWritten: lastEdit,
106+
unreadable: 0,
107+
lastEdit: lastEdit || null,
108+
markerInFile,
109+
packageJsonValid: true,
110+
})
111+
const summary = summarizeLabBatch(
112+
batch(
113+
[
114+
run('a', 0, { probe: probe(5, 5) }),
115+
run('a', 1, { probe: probe(5, 4) }),
116+
run('a', 2, { probe: probe(0, null) }),
117+
run('a', 3),
118+
],
119+
{ environment: 'concurrent-package-json' },
120+
),
121+
)
122+
expect(summary.collisions).toEqual({ measured: 2, collided: 1 })
123+
})
124+
})
125+
126+
test('labProbeCollided flags an invalid or unreadable file and a lost last edit', () => {
127+
const base = {
128+
kind: 'concurrent-package-json' as const,
129+
editsWritten: 3,
130+
unreadable: 0,
131+
lastEdit: 3,
132+
markerInFile: 3,
133+
packageJsonValid: true,
134+
}
135+
expect(labProbeCollided(base)).toBe(false)
136+
expect(labProbeCollided({ ...base, markerInFile: 2 })).toBe(true)
137+
expect(labProbeCollided({ ...base, packageJsonValid: false })).toBe(true)
138+
expect(labProbeCollided({ ...base, unreadable: 1 })).toBe(true)
139+
expect(labProbeCollided({ kind: 'setup-failed', error: 'x' })).toBeNull()
140+
expect(labProbeCollided(null)).toBeNull()
141+
})
142+
143+
test('nextLabBatchSlot fills each repetition across repositories before the next', () => {
144+
const b = batch([run('a', 0), run('b', 0)])
145+
expect(nextLabBatchSlot(b)).toEqual({ projectId: 'c', slot: 0 })
146+
expect(nextLabBatchSlot(b, new Set(['c']))).toEqual({
147+
projectId: 'a',
148+
slot: 1,
149+
})
150+
expect(nextLabBatchSlot({ ...b, repetitions: 1 }, new Set(['c']))).toBeNull()
151+
})
152+
153+
test('a batch input is bounded in runs and repositories', () => {
154+
const input = {
155+
name: 'Archil',
156+
selection: { kind: 'draft', id: crypto.randomUUID() },
157+
projectIds: ['a', 'b', 'c'],
158+
repetitions: 10,
159+
environment: 'node-nvm',
160+
signup: 'decline',
161+
}
162+
expect(labBatchInputSchema.safeParse(input).success).toBe(true)
163+
expect(
164+
labBatchInputSchema.safeParse({ ...input, projectIds: ['a', 'a'] }).success,
165+
).toBe(false)
166+
expect(
167+
labBatchInputSchema.safeParse({
168+
...input,
169+
projectIds: Array.from({ length: 6 }, (_, i) => `p${i}`),
170+
repetitions: 20,
171+
}).success,
172+
).toBe(false)
173+
expect(
174+
labBatchInputSchema.safeParse({ ...input, environment: 'windows' }).success,
175+
).toBe(false)
176+
})
177+
178+
describe('environment scripts', () => {
179+
const bash = (script: string, home: string, cwd = home, path = '') =>
180+
Bun.spawnSync(['bash', '-c', script], {
181+
cwd,
182+
env: {
183+
HOME: home,
184+
PATH: `${path}${home}/.local/bin:${process.env.PATH}`,
185+
},
186+
})
187+
188+
test.each(LAB_ENVIRONMENT_KEYS)('%s parses', (key) => {
189+
const { setup, probe } = LAB_ENVIRONMENTS[key]
190+
for (const script of [setup, probe])
191+
if (script)
192+
expect(Bun.spawnSync(['bash', '-n', '-c', script]).exitCode).toBe(0)
193+
})
194+
195+
test('a toolchain case activates from the profile once and hides the system node', () => {
196+
const home = mkdtempSync(join(tmpdir(), 'lab-home-'))
197+
// A stand-in installer: curl prints a script that creates the mise binary.
198+
const bin = join(home, 'fake-bin')
199+
mkdirSync(bin)
200+
writeFileSync(
201+
join(bin, 'curl'),
202+
`#!/bin/sh\nprintf '%s\\n' 'mkdir -p "$(dirname "$MISE_INSTALL_PATH")"; printf "#!/bin/sh\\nexit 0\\n" > "$MISE_INSTALL_PATH"; chmod 755 "$MISE_INSTALL_PATH"'\n`,
203+
{ mode: 0o755 },
204+
)
205+
const setup = LAB_ENVIRONMENTS['node-mise'].setup
206+
for (let i = 0; i < 2; i++) {
207+
const result = bash(setup, home, home, `${bin}:`)
208+
expect(result.stderr.toString()).toBe('')
209+
expect(result.exitCode).toBe(0)
210+
}
211+
const rc = readFileSync(join(home, '.bashrc'), 'utf8')
212+
expect(rc.match(/# freebuff-lab toolchain/g)).toHaveLength(1)
213+
expect(rc).toContain('eval "$("$MISE_DATA_DIR/bin/mise" activate bash)"')
214+
const node = bash('node --version', home)
215+
expect(node.exitCode).toBe(127)
216+
expect(node.stderr.toString()).toContain('node: not found')
217+
})
218+
219+
test('the collision probe reports the writer’s last edit against package.json', () => {
220+
const home = mkdtempSync(join(tmpdir(), 'lab-home-'))
221+
const repo = join(home, 'repo')
222+
mkdirSync(join(home, '.cache/freebuff-lab'), { recursive: true })
223+
mkdirSync(repo)
224+
writeFileSync(join(home, '.cache/freebuff-lab/edits'), '1\n2\n3\n')
225+
writeFileSync(
226+
join(repo, 'package.json'),
227+
JSON.stringify({ name: 'x', freebuffLabConcurrentEdit: 2 }),
228+
)
229+
const result = bash(
230+
LAB_ENVIRONMENTS['concurrent-package-json'].probe,
231+
home,
232+
repo,
233+
)
234+
const probe = JSON.parse(result.stdout.toString().trim())
235+
expect(probe).toEqual({
236+
kind: 'concurrent-package-json',
237+
editsWritten: 3,
238+
unreadable: 0,
239+
lastEdit: 3,
240+
markerInFile: 2,
241+
packageJsonValid: true,
242+
})
243+
expect(labProbeCollided(probe)).toBe(true)
244+
})
245+
246+
test('the collision case refuses a repository without package.json', () => {
247+
const home = mkdtempSync(join(tmpdir(), 'lab-home-'))
248+
const result = bash(LAB_ENVIRONMENTS['concurrent-package-json'].setup, home)
249+
expect(result.exitCode).toBe(3)
250+
})
251+
})

0 commit comments

Comments
 (0)