Skip to content

Commit e183cb6

Browse files
Sync public snapshot from freebuff-private
Source: CodebuffAI/freebuff-private@63a91364a9d5abfae2c750e225e0d184d58460b7
1 parent 3458612 commit e183cb6

2 files changed

Lines changed: 202 additions & 7 deletions

File tree

‎common/src/ads/sponsored-skill-layer2.test.ts‎

Lines changed: 150 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -9,10 +9,15 @@ import {
99
buildLayer2Skill,
1010
compileLayer2Skill,
1111
isLayer2Skill,
12+
LAYER2_MAX_VERIFY,
1213
LAYER2_SKILL_CHANGELOG,
1314
type SponsoredSkillLayer2,
15+
type SponsoredSkillLayer2Verify,
1416
} from './sponsored-skill-layer2'
15-
import { parseSponsoredProcedureSteps } from './sponsored-procedure-steps'
17+
import {
18+
parseSponsoredProcedureSteps,
19+
sponsoredStepNeedsUser,
20+
} from './sponsored-procedure-steps'
1621

1722
const SIGNUP =
1823
'https://app.posthog.com/signup?utm_source=freebuff&utm_medium=cpc&utm_campaign=analytics&bfcid={bfcid}'
@@ -30,6 +35,17 @@ const base: SponsoredSkillLayer2 = {
3035
},
3136
}
3237

38+
const VERIFY: SponsoredSkillLayer2Verify[] = [
39+
{
40+
label: 'a changed file imports posthog-js',
41+
pattern: `from ['"]posthog-js['"]|require\\(['"]posthog-js['"]\\)`,
42+
},
43+
{
44+
label: 'a changed file calls posthog.capture',
45+
pattern: 'posthog\\.capture\\(',
46+
},
47+
]
48+
3349
describe('buildLayer2Skill', () => {
3450
test('an npm skill with no keys compiles to a parsing format-2 skill', () => {
3551
const result = buildLayer2Skill(base, '1.0.0')
@@ -147,11 +163,129 @@ describe('buildLayer2Skill', () => {
147163
})
148164

149165
test('unknown fields are refused', () => {
150-
const result = buildLayer2Skill({ ...base, verify: [] }, '1.0.0')
166+
const result = buildLayer2Skill({ ...base, checks: [] }, '1.0.0')
151167
expect(result.ok).toBe(false)
152168
})
153169
})
154170

171+
describe('verify', () => {
172+
test('each item becomes a case-insensitive source check, before the auto checks', () => {
173+
const result = buildLayer2Skill(
174+
{ ...base, envKeys: ['POSTHOG_PROJECT_TOKEN'], verify: VERIFY },
175+
'1.0.0',
176+
)
177+
expect(result.ok).toBe(true)
178+
if (!result.ok) return
179+
const checks = result.skill.steps[0]!.checks
180+
expect(checks.map((check) => check.id)).toEqual([
181+
'verify-1',
182+
'verify-2',
183+
'package-1',
184+
'env-1',
185+
'no-secrets',
186+
])
187+
expect(checks[0]).toEqual({
188+
id: 'verify-1',
189+
kind: 'file_contains',
190+
label: 'a changed file imports posthog-js',
191+
path: '**/*.{ts,tsx,js,jsx,mjs,cjs,mts,cts,py}',
192+
pattern: VERIFY[0]!.pattern,
193+
flags: 'i',
194+
})
195+
})
196+
197+
test('omitted and empty compile to the same skill and hash', () => {
198+
const omitted = compileLayer2Skill(base, '1.0.0')
199+
const empty = compileLayer2Skill({ ...base, verify: [] }, '1.0.0')
200+
expect(empty).toEqual(omitted)
201+
expect(sponsoredSkillSha256(empty)).toBe(sponsoredSkillSha256(omitted))
202+
expect(JSON.stringify(omitted)).not.toContain('verify')
203+
})
204+
205+
test('verify moves the hash', () => {
206+
expect(
207+
sponsoredSkillSha256(
208+
compileLayer2Skill({ ...base, verify: VERIFY }, '1.0.0'),
209+
),
210+
).not.toBe(sponsoredSkillSha256(compileLayer2Skill(base, '1.0.0')))
211+
})
212+
213+
test('a pattern that is not a regex is refused', () => {
214+
const result = buildLayer2Skill(
215+
{ ...base, verify: [{ label: 'broken', pattern: 'posthog(' }] },
216+
'1.0.0',
217+
)
218+
expect(result.ok).toBe(false)
219+
if (result.ok) return
220+
expect(result.errors[0]!.path).toBe('verify.0.pattern')
221+
})
222+
223+
test(`more than ${LAYER2_MAX_VERIFY}, an empty label, a long pattern or an extra field is refused`, () => {
224+
const item = VERIFY[1]!
225+
for (const verify of [
226+
Array.from({ length: LAYER2_MAX_VERIFY + 1 }, () => item),
227+
[{ ...item, label: ' ' }],
228+
[{ ...item, label: 'x'.repeat(101) }],
229+
[{ ...item, pattern: 'a'.repeat(201) }],
230+
[{ ...item, flags: 'i' }],
231+
]) {
232+
expect(buildLayer2Skill({ ...base, verify }, '1.0.0').ok).toBe(false)
233+
}
234+
})
235+
})
236+
237+
describe('the agent step', () => {
238+
test('reads as guidance, and never as user work', () => {
239+
const skill = compileLayer2Skill(
240+
{ ...base, envKeys: ['POSTHOG_PROJECT_TOKEN'], verify: VERIFY },
241+
'1.0.0',
242+
)
243+
const integrate = skill.steps[0]!
244+
expect(integrate.title).toBe(
245+
'Build the Posthog integration in this project',
246+
)
247+
expect(integrate.instructions).toBe(
248+
`${base.goal}\nGuidelines:\n- Read the docs before writing code: ${base.docsUrl}\n- Use only the pinned packages; fit this project's framework and conventions, and change as little as possible.\n- Read POSTHOG_PROJECT_TOKEN from the environment only, and declare it with an empty value in the example env file.\nHow you do it is up to you; the step is done when its checks pass.`,
249+
)
250+
expect(
251+
sponsoredStepNeedsUser(`${integrate.title} ${integrate.instructions}`),
252+
).toBe(false)
253+
})
254+
255+
test('with no keys there is no env line', () => {
256+
const integrate = compileLayer2Skill(base, '1.0.0').steps[0]!
257+
expect(integrate.instructions).not.toContain('environment')
258+
expect(integrate.instructions).toEndWith(
259+
'change as little as possible.\nHow you do it is up to you; the step is done when its checks pass.',
260+
)
261+
})
262+
263+
test('two keys share one line', () => {
264+
const integrate = compileLayer2Skill(
265+
{
266+
...base,
267+
envKeys: ['POSTHOG_PROJECT_TOKEN', 'POSTHOG_PERSONAL_API_KEY'],
268+
},
269+
'1.0.0',
270+
).steps[0]!
271+
expect(integrate.instructions).toContain(
272+
'- Read POSTHOG_PROJECT_TOKEN and POSTHOG_PERSONAL_API_KEY from the environment only, and declare each',
273+
)
274+
})
275+
276+
test('the rendered steps with keys and verify still read as agent then user work', () => {
277+
const result = buildLayer2Skill(
278+
{ ...base, envKeys: ['POSTHOG_PROJECT_TOKEN'], verify: VERIFY },
279+
'1.0.0',
280+
)
281+
if (!result.ok) throw new Error(JSON.stringify(result.errors))
282+
const steps = parseSponsoredProcedureSteps(
283+
renderSponsoredSkillProcedure(result.skill),
284+
)
285+
expect(steps.map((step) => step.needsUser)).toEqual([false, true])
286+
})
287+
})
288+
155289
describe('compileLayer2Skill', () => {
156290
test('is deterministic: same source and version, same hash', () => {
157291
const a = sponsoredSkillSha256(compileLayer2Skill(base, '1.0.0'))
@@ -184,7 +318,20 @@ describe('compileLayer2Skill', () => {
184318

185319
test('renders far under the old long-form skills', () => {
186320
const rendered = renderSponsoredSkillProcedure(
187-
compileLayer2Skill({ ...base, envKeys: ['POSTHOG_PROJECT_TOKEN'] }, '1.0.0'),
321+
compileLayer2Skill(
322+
{
323+
...base,
324+
envKeys: ['POSTHOG_PROJECT_TOKEN'],
325+
verify: [
326+
...VERIFY,
327+
{
328+
label: 'a changed file calls posthog.init',
329+
pattern: 'posthog\\.init\\(',
330+
},
331+
],
332+
},
333+
'1.0.0',
334+
),
188335
)
189336
expect(rendered.length).toBeLessThan(2_500)
190337
})

‎common/src/ads/sponsored-skill-layer2.ts‎

Lines changed: 52 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -50,6 +50,8 @@ export function isLayer2Skill(
5050

5151
export const LAYER2_MAX_PACKAGES = 6
5252
export const LAYER2_MAX_ENV_KEYS = 2
53+
/** Advertiser checks that the integration is really in the code. */
54+
export const LAYER2_MAX_VERIFY = 3
5355

5456
const slug = z
5557
.string()
@@ -98,6 +100,28 @@ const layer2Package = z.discriminatedUnion('ecosystem', [
98100
.strict(),
99101
])
100102

103+
const layer2Verify = z
104+
.object({
105+
/** What the check proves, e.g. "a changed file imports disk". */
106+
label: z.string().trim().min(1).max(100),
107+
/** A case-insensitive regex over the changed source files. */
108+
pattern: z
109+
.string()
110+
.min(1)
111+
.max(200)
112+
.refine((pattern) => {
113+
try {
114+
new RegExp(pattern, 'i')
115+
return true
116+
} catch {
117+
return false
118+
}
119+
}, 'Not a valid regular expression.'),
120+
})
121+
.strict()
122+
123+
export type SponsoredSkillLayer2Verify = z.infer<typeof layer2Verify>
124+
101125
export const sponsoredSkillLayer2Schema = z
102126
.object({
103127
/** Stable slug, `<advertiser>-<goal>`; one id has many versions. */
@@ -125,6 +149,11 @@ export const sponsoredSkillLayer2Schema = z
125149
)
126150
.max(LAYER2_MAX_ENV_KEYS)
127151
.optional(),
152+
/**
153+
* Code checks that prove the integration landed (an import of the SDK, a
154+
* call to its client). Omitted or empty compiles to no checks.
155+
*/
156+
verify: z.array(layer2Verify).max(LAYER2_MAX_VERIFY).optional(),
128157
consent: z
129158
.object({
130159
title: z.string().trim().min(1).max(100),
@@ -173,6 +202,8 @@ function escapeRegex(text: string): string {
173202

174203
const ENV_EXAMPLE_PATH =
175204
'**/{.env.example,.env.local.example,.env.sample,.env.template}'
205+
/** The source files a verify check reads. */
206+
const SOURCE_FILES = '**/*.{ts,tsx,js,jsx,mjs,cjs,mts,cts,py}'
176207
const PYTHON_MANIFESTS =
177208
'**/{pyproject.toml,requirements*.txt,setup.py,setup.cfg,Pipfile}'
178209

@@ -239,17 +270,32 @@ export function compileLayer2Skill(
239270
key,
240271
}))
241272

273+
const verifyChecks: SponsoredSkillCheck[] = (source.verify ?? []).map(
274+
(item, index) => ({
275+
id: `verify-${index + 1}`,
276+
kind: 'file_contains',
277+
label: item.label,
278+
path: SOURCE_FILES,
279+
pattern: item.pattern,
280+
flags: 'i',
281+
}),
282+
)
283+
284+
// Guidance the agent adapts to the project, not a script; the checks are
285+
// what it must meet. No wording here may read as user work
286+
// (`NEEDS_USER_PATTERNS` in sponsored-procedure-steps.ts).
242287
const envLine = envKeys.length
243-
? `\nRead ${envKeys.join(' and ')} from the environment only, and declare ${
288+
? `- Read ${envKeys.join(' and ')} from the environment only, and declare ${
244289
envKeys.length === 1 ? 'it' : 'each'
245-
} with an empty value in the example env file.`
290+
} with an empty value in the example env file.\n`
246291
: ''
247292
const integrate: SponsoredSkillStep = {
248293
id: 'integrate',
249294
kind: 'agent',
250295
title: `Build the ${label} integration in this project`,
251-
instructions: `${source.goal}\nRead the docs before writing code: ${source.docsUrl}\nUse only the pinned packages, fit the project's framework and style, and change as little as possible.${envLine}`,
296+
instructions: `${source.goal}\nGuidelines:\n- Read the docs before writing code: ${source.docsUrl}\n- Use only the pinned packages; fit this project's framework and conventions, and change as little as possible.\n${envLine}How you do it is up to you; the step is done when its checks pass.`,
252297
checks: [
298+
...verifyChecks,
253299
...packageChecks,
254300
...envChecks,
255301
{
@@ -344,7 +390,9 @@ export function buildLayer2Skill(
344390
if (!sponsoredSkillTrackedLink(compiled.skill)) {
345391
return {
346392
ok: false,
347-
errors: [{ path: 'signupUrl', message: SPONSORED_SKILL_TRACKED_LINK_ERROR }],
393+
errors: [
394+
{ path: 'signupUrl', message: SPONSORED_SKILL_TRACKED_LINK_ERROR },
395+
],
348396
}
349397
}
350398
return { ok: true, source: parsed.data, skill: compiled.skill }

0 commit comments

Comments
 (0)