Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions .changeset/prompts-fetch-by-label.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
---
'@posthog/ai': minor
---

feat(ai): add a `label` option to `Prompts.get()` to fetch the prompt version a label (e.g. `production`) currently points to. Labeled fetches are cached separately, results carry the resolved `label`, and a warning is logged when the server does not resolve the requested label (older PostHog versions ignore the parameter and return the latest version).
78 changes: 59 additions & 19 deletions packages/ai/src/prompts.ts
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,9 @@ import type {

const DEFAULT_CACHE_TTL_SECONDS = 300 // 5 minutes
const DEFAULT_PROMPTS_HOST = 'https://us.posthog.com'
type PromptVersionCache = Map<number | undefined, CachedPrompt>
// Keyed by version number, label string, or undefined for the latest version.
// Version and label keys can't collide: one is always a number, the other a string.
type PromptVersionCache = Map<number | string | undefined, CachedPrompt>

function normalizeApiKey(value?: unknown): string {
return typeof value === 'string' ? value.trim() : ''
Expand All @@ -30,7 +32,12 @@ function isPromptApiResponse(data: unknown): data is PromptApiResponse {
return false
}
const record = data as Record<string, unknown>
return typeof record.prompt === 'string' && typeof record.name === 'string' && typeof record.version === 'number'
return (
typeof record.prompt === 'string' &&
typeof record.name === 'string' &&
typeof record.version === 'number' &&
(record.label === undefined || typeof record.label === 'string')
)
}

export interface PromptsWithPostHogOptions {
Expand Down Expand Up @@ -70,6 +77,11 @@ function isPromptsWithPostHog(options: PromptsOptions): options is PromptsWithPo
* version: 3,
* })
*
* // Or fetch the version a label currently points to
* const prod = await prompts.get('support-system-prompt', {
* label: 'production',
* })
*
* // Compile with variables
* const systemPrompt = prompts.compile(result.prompt, {
* company: 'Acme Corp',
Expand Down Expand Up @@ -114,8 +126,14 @@ export class Prompts {
return promptVersions
}

private getPromptLabel(name: string, version?: number): string {
return version === undefined ? `"${name}"` : `"${name}" version ${version}`
private getPromptReference(name: string, version?: number, label?: string): string {
if (version !== undefined) {
return `"${name}" version ${version}`
}
if (label !== undefined) {
return `"${name}" label "${label}"`
}
return `"${name}"`
}

/**
Expand All @@ -125,20 +143,25 @@ export class Prompts {
* `name`, and `version` metadata. Read `result.prompt` for the template string.
*/
async get(name: string, options?: GetPromptOptions): Promise<PromptResult> {
if (options?.version !== undefined && options?.label !== undefined) {
throw new Error('[PostHog Prompts] Pass either version or label, not both.')
}

try {
return await this.getInternal(name, options)
} catch (error) {
const fallback = options?.fallback

if (fallback !== undefined) {
const promptLabel = this.getPromptLabel(name, options?.version)
console.warn(`[PostHog Prompts] Failed to fetch prompt ${promptLabel}, using fallback:`, error)
const promptReference = this.getPromptReference(name, options?.version, options?.label)
console.warn(`[PostHog Prompts] Failed to fetch prompt ${promptReference}, using fallback:`, error)

return {
source: 'code_fallback',
prompt: fallback,
name: undefined,
version: undefined,
label: undefined,
} satisfies PromptCodeFallbackResult
}

Expand All @@ -153,10 +176,12 @@ export class Prompts {
private async getInternal(name: string, options?: GetPromptOptions): Promise<PromptRemoteResult> {
const cacheTtlSeconds = options?.cacheTtlSeconds ?? this.defaultCacheTtlSeconds
const version = options?.version
const promptLabel = this.getPromptLabel(name, version)
const label = options?.label
const promptReference = this.getPromptReference(name, version, label)
const cacheEntryKey = version ?? label

// Check cache first
const cached = this.getPromptCache(name)?.get(version)
const cached = this.getPromptCache(name)?.get(cacheEntryKey)
const now = Date.now()

if (cached) {
Expand All @@ -170,17 +195,27 @@ export class Prompts {

// Try to fetch from API
try {
const fetched = await this.fetchPromptFromApi(name, version)
const fetched = await this.fetchPromptFromApi(name, version, label)

// An older PostHog server ignores the label param and returns the latest
// version with no label field — surface that instead of failing silently.
if (label !== undefined && fetched.label !== label) {
console.warn(
`[PostHog Prompts] Requested label "${label}" for prompt "${name}" but the server resolved ` +
`${fetched.label === undefined ? 'no label' : `"${fetched.label}"`}. It may not support prompt ` +
'labels yet and returned the latest version instead.'
)
}

// Update cache
this.getOrCreatePromptCache(name).set(version, { ...fetched, fetchedAt: Date.now() })
this.getOrCreatePromptCache(name).set(cacheEntryKey, { ...fetched, fetchedAt: Date.now() })

return { source: 'api', ...fetched }
} catch (error) {
// Return stale cache (with warning)
if (cached) {
const { fetchedAt: _, ...cachedResult } = cached
console.warn(`[PostHog Prompts] Failed to fetch prompt ${promptLabel}, using stale cache:`, error)
console.warn(`[PostHog Prompts] Failed to fetch prompt ${promptReference}, using stale cache:`, error)
return { source: 'stale_cache', ...cachedResult }
}

Expand Down Expand Up @@ -237,7 +272,11 @@ export class Prompts {
}
}

private async fetchPromptFromApi(name: string, version?: number): Promise<Omit<PromptRemoteResult, 'source'>> {
private async fetchPromptFromApi(
name: string,
version?: number,
label?: string
): Promise<Omit<PromptRemoteResult, 'source'>> {
if (!this.personalApiKey) {
throw new Error(
'[PostHog Prompts] personalApiKey is required to fetch prompts. ' +
Expand All @@ -254,8 +293,9 @@ export class Prompts {
const encodedPromptName = encodeURIComponent(name)
const encodedProjectApiKey = encodeURIComponent(this.projectApiKey)
const versionQuery = version === undefined ? '' : `&version=${encodeURIComponent(String(version))}`
const promptLabel = this.getPromptLabel(name, version)
const url = `${this.host}/api/environments/@current/llm_prompts/name/${encodedPromptName}/?token=${encodedProjectApiKey}${versionQuery}`
const labelQuery = label === undefined ? '' : `&label=${encodeURIComponent(label)}`
Comment on lines 295 to +296

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

P2 Empty Label Becomes Ambiguous Request

When a configured label resolves to an empty string, this path sends label= and caches the response under the empty-string key. If the API treats an empty label as missing or rejects it, callers either receive the latest prompt instead of a labeled prompt or unexpectedly fall back; reject empty labels before building the request.

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

No, empty label will be rejected with a 400, and thus never cached

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Fair enough — a 400 will throw before any caching happens, so the failure is explicit rather than silent. My concern doesn't apply here.

Tip: You can customize Greptile's behavior for this repo with .greptile/rules.md and .greptile/config.json.

const promptReference = this.getPromptReference(name, version, label)
const url = `${this.host}/api/environments/@current/llm_prompts/name/${encodedPromptName}/?token=${encodedProjectApiKey}${versionQuery}${labelQuery}`

const response = await fetch(url, {
method: 'GET',
Expand All @@ -266,25 +306,25 @@ export class Prompts {

if (!response.ok) {
if (response.status === 404) {
throw new Error(`[PostHog Prompts] Prompt ${promptLabel} not found`)
throw new Error(`[PostHog Prompts] Prompt ${promptReference} not found`)
}

if (response.status === 403) {
throw new Error(
`[PostHog Prompts] Access denied for prompt ${promptLabel}. ` +
`[PostHog Prompts] Access denied for prompt ${promptReference}. ` +
'Check that your personalApiKey has the correct permissions and the LLM prompts feature is enabled.'
)
}

throw new Error(`[PostHog Prompts] Failed to fetch prompt ${promptLabel}: HTTP ${response.status}`)
throw new Error(`[PostHog Prompts] Failed to fetch prompt ${promptReference}: HTTP ${response.status}`)
}

const data: unknown = await response.json()

if (!isPromptApiResponse(data)) {
throw new Error(`[PostHog Prompts] Invalid response format for prompt ${promptLabel}`)
throw new Error(`[PostHog Prompts] Invalid response format for prompt ${promptReference}`)
}

return { prompt: data.prompt, name: data.name, version: data.version }
return { prompt: data.prompt, name: data.name, version: data.version, label: data.label }
Comment thread
jurajmajerik marked this conversation as resolved.
}
}
9 changes: 9 additions & 0 deletions packages/ai/src/types.ts
Original file line number Diff line number Diff line change
Expand Up @@ -105,7 +105,10 @@ export interface TokenUsage {
export interface GetPromptOptions {
cacheTtlSeconds?: number
fallback?: string
/** Specific prompt version to fetch. Mutually exclusive with label. */
version?: number
/** Fetch the version this label currently points to, e.g. 'production'. Mutually exclusive with version. */
label?: string
}

/**
Expand All @@ -115,6 +118,7 @@ export interface CachedPrompt {
prompt: string
name: string
version: number
label?: string
fetchedAt: number
}

Expand All @@ -126,6 +130,8 @@ export interface PromptApiResponse {
name: string
prompt: string
version: number
/** Present when the prompt was fetched by label. */
label?: string
created_by: string
created_at: string
updated_at: string
Expand All @@ -140,6 +146,8 @@ export interface PromptRemoteResult {
prompt: string
name: string
version: number
/** The label the prompt was fetched by, when fetching with the label option. */
label?: string
}

/**
Expand All @@ -152,6 +160,7 @@ export interface PromptCodeFallbackResult {
prompt: string
name: undefined
version: undefined
label: undefined
}

/**
Expand Down
83 changes: 83 additions & 0 deletions packages/ai/tests/prompts.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -119,6 +119,89 @@ describe('Prompts', () => {
)
})

it('should fetch by label and surface label metadata', async () => {
mockFetch.mockResolvedValueOnce({
ok: true,
status: 200,
json: () =>
Promise.resolve({ ...mockPromptResponse, version: 3, prompt: 'Production prompt', label: 'production' }),
})

const posthog = createMockPostHog()
const prompts = new Prompts({ posthog })

const result = await prompts.get('test-prompt', { label: 'production' })

expect(result.prompt).toBe('Production prompt')
expect(result.version).toBe(3)
expect(result.label).toBe('production')
expect(mockFetch).toHaveBeenCalledWith(
'https://us.posthog.com/api/environments/@current/llm_prompts/name/test-prompt/?token=phc_test_key&label=production',
{
method: 'GET',
headers: {
Authorization: 'Bearer phx_test_key',
},
}
)
})

it('should warn when the server does not resolve the requested label', async () => {
mockFetch.mockResolvedValueOnce({
ok: true,
status: 200,
json: () => Promise.resolve(mockPromptResponse), // no label field — old-server behavior
})

const prompts = new Prompts({ posthog: createMockPostHog() })
const result = await prompts.get('test-prompt', { label: 'production' })

expect(result.prompt).toBe(mockPromptResponse.prompt)
expect(result.label).toBeUndefined()
expect(consoleWarnSpy).toHaveBeenCalledWith(expect.stringContaining('may not support prompt labels'))
})

it('should reject version and label together', async () => {
const prompts = new Prompts({ posthog: createMockPostHog() })

await expect(prompts.get('test-prompt', { version: 1, label: 'production' })).rejects.toThrow(
'either version or label'
)
expect(mockFetch).not.toHaveBeenCalled()
})

it('should keep labeled and latest prompt caches separate', async () => {
mockFetch
.mockResolvedValueOnce({
ok: true,
status: 200,
json: () => Promise.resolve({ ...mockPromptResponse, version: 4, prompt: 'Latest prompt' }),
})
.mockResolvedValueOnce({
ok: true,
status: 200,
json: () =>
Promise.resolve({ ...mockPromptResponse, version: 2, prompt: 'Production prompt', label: 'production' }),
})

const posthog = createMockPostHog()
const prompts = new Prompts({ posthog })

// A labeled fetch after a latest fetch must not be served from the
// latest cache entry — that would silently return the wrong version.
await expect(prompts.get('test-prompt')).resolves.toHaveProperty('prompt', 'Latest prompt')
await expect(prompts.get('test-prompt', { label: 'production' })).resolves.toHaveProperty(
'prompt',
'Production prompt'
)
await expect(prompts.get('test-prompt')).resolves.toHaveProperty('prompt', 'Latest prompt')
await expect(prompts.get('test-prompt', { label: 'production' })).resolves.toHaveProperty(
'prompt',
'Production prompt'
)
expect(mockFetch).toHaveBeenCalledTimes(2)
})

it('should return cached prompt when fresh', async () => {
mockFetch.mockResolvedValueOnce({
ok: true,
Expand Down
Loading